Page MenuHomeFreeBSD
Differential D56482 Diff 176814 sys/netinet/in_pcb.c

Changeset View

Standalone View

View Options

sys/netinet/in_pcb.c

Show First 20 Lines • Show All 128 Lines • ▼ Show 20 Lines
*/ */
VNET_DEFINE(int, ipport_reservedhigh) = IPPORT_RESERVED - 1; /* 1023 */ VNET_DEFINE(int, ipport_reservedhigh) = IPPORT_RESERVED - 1; /* 1023 */
VNET_DEFINE(int, ipport_reservedlow); VNET_DEFINE(int, ipport_reservedlow);
/* Enable random ephemeral port allocation by default. */ /* Enable random ephemeral port allocation by default. */
VNET_DEFINE(int, ipport_randomized) = 1; VNET_DEFINE(int, ipport_randomized) = 1;
#ifdef INET #ifdef INET
static struct inpcb *in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, static struct inpcb *in_pcblookup_internal(struct inpcbinfo *pcbinfo,
struct in_addr faddr, u_int fport_arg, struct in_addr faddr, u_int fport_arg, struct in_addr laddr,
struct in_addr laddr, u_int lport_arg, u_int lport_arg, int lookupflags, uint8_t numa_domain, int fib);
int lookupflags, uint8_t numa_domain, int fib);
#define RANGECHK(var, min, max) \ #define RANGECHK(var, min, max) \
if ((var) < (min)) { (var) = (min); } \ if ((var) < (min)) { (var) = (min); } \
else if ((var) > (max)) { (var) = (max); } else if ((var) > (max)) { (var) = (max); }
static int static int
sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS) sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS)
{ {
▲ Show 20 Lines • Show All 655 Lines • ▼ Show 20 Lines do {
++*lastport; ++*lastport;
if (*lastport < first || *lastport > last) if (*lastport < first || *lastport > last)
*lastport = first; *lastport = first;
lport = htons(*lastport); lport = htons(*lastport);
if (fsa != NULL) { if (fsa != NULL) {
#ifdef INET #ifdef INET
if (lsa->sa_family == AF_INET) { if (lsa->sa_family == AF_INET) {
tmpinp = in_pcblookup_hash_locked(pcbinfo, tmpinp = in_pcblookup_internal(pcbinfo,
faddr, fport, laddr, lport, lookupflags, faddr, fport, laddr, lport, lookupflags,
M_NODOM, RT_ALL_FIBS); M_NODOM, RT_ALL_FIBS);
} }
#endif #endif
#ifdef INET6 #ifdef INET6
if (lsa->sa_family == AF_INET6) { if (lsa->sa_family == AF_INET6) {
tmpinp = in6_pcblookup_hash_locked(pcbinfo, tmpinp = in6_pcblookup_internal(pcbinfo,
faddr6, fport, laddr6, lport, lookupflags, faddr6, fport, laddr6, lport, lookupflags,
M_NODOM, RT_ALL_FIBS); M_NODOM, RT_ALL_FIBS);
} }
#endif #endif
} else { } else {
#ifdef INET6 #ifdef INET6
if ((inp->inp_vflag & INP_IPV6) != 0) { if ((inp->inp_vflag & INP_IPV6) != 0) {
tmpinp = in6_pcblookup_local(pcbinfo, tmpinp = in6_pcblookup_local(pcbinfo,
▲ Show 20 Lines • Show All 338 Lines • ▼ Show 20 Lines if (anonport) {
error = in_pcb_lport_dest(inp, (struct sockaddr *)&lsin, error = in_pcb_lport_dest(inp, (struct sockaddr *)&lsin,
&lport, (struct sockaddr *)&fsin, sin->sin_port, cred, &lport, (struct sockaddr *)&fsin, sin->sin_port, cred,
INPLOOKUP_WILDCARD); INPLOOKUP_WILDCARD);
if (__predict_false(error)) { if (__predict_false(error)) {
INP_HASH_WUNLOCK(inp->inp_pcbinfo); INP_HASH_WUNLOCK(inp->inp_pcbinfo);
return (error); return (error);
} }
} else if (in_pcblookup_hash_locked(inp->inp_pcbinfo, faddr, } else if (in_pcblookup_internal(inp->inp_pcbinfo, faddr,
sin->sin_port, laddr, inp->inp_lport, 0, M_NODOM, RT_ALL_FIBS) != sin->sin_port, laddr, inp->inp_lport, 0, M_NODOM, RT_ALL_FIBS) !=
NULL) { NULL) {
INP_HASH_WUNLOCK(inp->inp_pcbinfo); INP_HASH_WUNLOCK(inp->inp_pcbinfo);
return (EADDRINUSE); return (EADDRINUSE);
} else } else
lport = inp->inp_lport; lport = inp->inp_lport;
MPASS(!in_nullhost(inp->inp_laddr) || inp->inp_lport != 0 || MPASS(!in_nullhost(inp->inp_laddr) || inp->inp_lport != 0 ||
▲ Show 20 Lines • Show All 1,054 Lines • ▼ Show 20 Lines if (inp->inp_faddr.s_addr == faddr.s_addr &&
inp->inp_laddr.s_addr == laddr.s_addr && inp->inp_laddr.s_addr == laddr.s_addr &&
inp->inp_fport == fport && inp->inp_fport == fport &&
inp->inp_lport == lport) inp->inp_lport == lport)
return (true); return (true);
return (false); return (false);
} }
static struct inpcb * static struct inpcb *
in_pcblookup_hash_exact(struct inpcbinfo *pcbinfo, struct in_addr faddr, in_pcblookup_exact(struct inpcbinfo *pcbinfo, struct in_addr faddr,
u_short fport, struct in_addr laddr, u_short lport) u_short fport, struct in_addr laddr, u_short lport)
{ {
struct inpcbhead *head; struct inpcbhead *head;
struct inpcb *inp; struct inpcb *inp;
INP_HASH_LOCK_ASSERT(pcbinfo); INP_HASH_LOCK_ASSERT(pcbinfo);
head = &pcbinfo->ipi_hash_exact[INP_PCBHASH(&faddr, lport, fport, head = &pcbinfo->ipi_hash_exact[INP_PCBHASH(&faddr, lport, fport,
Show All 29 Lines#endif
if (inp->inp_laddr.s_addr == laddr.s_addr) if (inp->inp_laddr.s_addr == laddr.s_addr)
return (INPLOOKUP_MATCH_LADDR); return (INPLOOKUP_MATCH_LADDR);
return (INPLOOKUP_MATCH_NONE); return (INPLOOKUP_MATCH_NONE);
} }
#define INP_LOOKUP_AGAIN ((struct inpcb *)(uintptr_t)-1) #define INP_LOOKUP_AGAIN ((struct inpcb *)(uintptr_t)-1)
static struct inpcb * static struct inpcb *
in_pcblookup_hash_wild_smr(struct inpcbinfo *pcbinfo, struct in_addr laddr, in_pcblookup_wild_smr(struct inpcbinfo *pcbinfo, struct in_addr laddr,
u_short lport, int fib, const inp_lookup_t lockflags) u_short lport, int fib, const inp_lookup_t lockflags)
{ {
struct inpcbhead *head; struct inpcbhead *head;
struct inpcb *inp; struct inpcb *inp;
KASSERT(SMR_ENTERED(pcbinfo->ipi_smr), KASSERT(SMR_ENTERED(pcbinfo->ipi_smr),
("%s: not in SMR read section", __func__)); ("%s: not in SMR read section", __func__));
Show All 20 Lines CK_LIST_FOREACH(inp, head, inp_hash_wild) {
* to a serialized lookup. * to a serialized lookup.
*/ */
return (INP_LOOKUP_AGAIN); return (INP_LOOKUP_AGAIN);
} }
return (NULL); return (NULL);
} }
static struct inpcb * static struct inpcb *
in_pcblookup_hash_wild_locked(struct inpcbinfo *pcbinfo, struct in_addr laddr, in_pcblookup_wild_locked(struct inpcbinfo *pcbinfo, struct in_addr laddr,
u_short lport, int fib) u_short lport, int fib)
{ {
struct inpcbhead *head; struct inpcbhead *head;
struct inpcb *inp, *local_wild, *local_exact, *jail_wild; struct inpcb *inp, *local_wild, *local_exact, *jail_wild;
#ifdef INET6 #ifdef INET6
struct inpcb *local_wild_mapped; struct inpcb *local_wild_mapped;
#endif #endif
▲ Show 20 Lines • Show All 62 Lines • ▼ Show 20 Lines
/* /*
* Lookup PCB in hash list, using pcbinfo tables. This variation assumes * Lookup PCB in hash list, using pcbinfo tables. This variation assumes
* that the caller has either locked the hash list, which usually happens * that the caller has either locked the hash list, which usually happens
* for bind(2) operations, or is in SMR section, which happens when sorting * for bind(2) operations, or is in SMR section, which happens when sorting
* out incoming packets. * out incoming packets.
*/ */
static struct inpcb * static struct inpcb *
in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, struct in_addr faddr, in_pcblookup_internal(struct inpcbinfo *pcbinfo, struct in_addr faddr,
u_int fport_arg, struct in_addr laddr, u_int lport_arg, int lookupflags, u_int fport_arg, struct in_addr laddr, u_int lport_arg, int lookupflags,
uint8_t numa_domain, int fib) uint8_t numa_domain, int fib)
{ {
struct inpcb *inp; struct inpcb *inp;
const u_short fport = fport_arg, lport = lport_arg; const u_short fport = fport_arg, lport = lport_arg;
KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD | INPLOOKUP_FIB)) == 0, KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD | INPLOOKUP_FIB)) == 0,
("%s: invalid lookup flags %d", __func__, lookupflags)); ("%s: invalid lookup flags %d", __func__, lookupflags));
KASSERT(faddr.s_addr != INADDR_ANY, KASSERT(faddr.s_addr != INADDR_ANY,
("%s: invalid foreign address", __func__)); ("%s: invalid foreign address", __func__));
KASSERT(laddr.s_addr != INADDR_ANY, KASSERT(laddr.s_addr != INADDR_ANY,
("%s: invalid local address", __func__)); ("%s: invalid local address", __func__));
INP_HASH_WLOCK_ASSERT(pcbinfo); INP_HASH_WLOCK_ASSERT(pcbinfo);
inp = in_pcblookup_hash_exact(pcbinfo, faddr, fport, laddr, lport); inp = in_pcblookup_exact(pcbinfo, faddr, fport, laddr, lport);
if (inp != NULL) if (inp != NULL)
return (inp); return (inp);
if ((lookupflags & INPLOOKUP_WILDCARD) != 0) { if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
inp = in_pcblookup_lbgroup(pcbinfo, &faddr, fport, inp = in_pcblookup_lbgroup(pcbinfo, &faddr, fport,
&laddr, lport, numa_domain, fib); &laddr, lport, numa_domain, fib);
if (inp == NULL) { if (inp == NULL) {
inp = in_pcblookup_hash_wild_locked(pcbinfo, laddr, inp = in_pcblookup_wild_locked(pcbinfo, laddr,
lport, fib); lport, fib);
} }
} }
return (inp); return (inp);
} }
/*
* Lookup inpcb using locks. Used by in_pcblookup_smr() in case inp_smr_lock()
* failed.
*/
static struct inpcb * static struct inpcb *
in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr, in_pcblookup_with_lock(struct inpcbinfo *pcbinfo, struct in_addr faddr,
markjUnsubmitted
Not Done
Inline Actions

Conventionally, a lookup function named foo_locked() is expected to be called with a lock held, but here that's not the case. I'd rather not break that convention.

I think in_pcblookup_internal() should be called in_pcblookup_locked(). I'm not sure what to call this function though. Some ideas: in_pcblookup_with_lock(), in_pcblookup_nosmr(), in_pcblookup1().

markj: Conventionally, a lookup function named `foo_locked()` is expected to be called with a lock…
glebiusAuthorUnsubmitted
Done
Inline Actions

I had such a version, too :) What is currently called in_pcblookup_internal() were to be in_pcblookup_locked. This function was in_pcblookup_hard(), but I can go with in_pcblookup_withlock().

glebius: I had such a version, too :) What is currently called `in_pcblookup_internal()` were to be…
glebiusAuthorUnsubmitted
Done
Inline Actions

I was about to update to the mentioned version, but then looked up into my upcoming changes.

In my upcoming changes I have in_pcblookup_internal() to be called without a lock, thus it can't become in_pcblookup_locked(). The plan is that in_pcblookup_internal() will lookup the hash bucket itself, lock the bucket and return with the bucket locked. Later some consumers may grab the inpcb lock and drop bucket lock. That would be current in_pcblookup_locked(). Other consumers will see that inpcb is NULL and they are good to go with inserting their inpcb into the locked bucket.

So in_pcblookup_internal() definitely stays with this name. I can rename the _locked version into: _with_lock, _withlock or _hard.

glebius: I was about to update to the mentioned version, but then looked up into my upcoming changes.
u_int fport, struct in_addr laddr, u_int lport, int lookupflags, u_int fport, struct in_addr laddr, u_int lport, int lookupflags,
uint8_t numa_domain, int fib) uint8_t numa_domain, int fib)
{ {
struct inpcb *inp; struct inpcb *inp;
const inp_lookup_t lockflags = lookupflags & INPLOOKUP_LOCKMASK; const inp_lookup_t lockflags = lookupflags & INPLOOKUP_LOCKMASK;
KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0, KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0,
("%s: LOCKPCB not set", __func__)); ("%s: LOCKPCB not set", __func__));
INP_HASH_WLOCK(pcbinfo); INP_HASH_WLOCK(pcbinfo);
inp = in_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport, inp = in_pcblookup_internal(pcbinfo, faddr, fport, laddr, lport,
lookupflags & ~INPLOOKUP_LOCKMASK, numa_domain, fib); lookupflags & ~INPLOOKUP_LOCKMASK, numa_domain, fib);
if (inp != NULL && !inp_trylock(inp, lockflags)) { if (inp != NULL && !inp_trylock(inp, lockflags)) {
in_pcbref(inp); in_pcbref(inp);
INP_HASH_WUNLOCK(pcbinfo); INP_HASH_WUNLOCK(pcbinfo);
inp_lock(inp, lockflags); inp_lock(inp, lockflags);
if (in_pcbrele(inp, lockflags)) if (in_pcbrele(inp, lockflags))
/* XXX-MJ or retry until we get a negative match? */ /* XXX-MJ or retry until we get a negative match? */
inp = NULL; inp = NULL;
} else { } else {
INP_HASH_WUNLOCK(pcbinfo); INP_HASH_WUNLOCK(pcbinfo);
} }
return (inp); return (inp);
} }
static struct inpcb * static struct inpcb *
in_pcblookup_hash_smr(struct inpcbinfo *pcbinfo, struct in_addr faddr, in_pcblookup_smr(struct inpcbinfo *pcbinfo, struct in_addr faddr,
u_int fport_arg, struct in_addr laddr, u_int lport_arg, int lookupflags, u_int fport_arg, struct in_addr laddr, u_int lport_arg, int lookupflags,
uint8_t numa_domain, int fib) uint8_t numa_domain, int fib)
{ {
struct inpcb *inp; struct inpcb *inp;
const inp_lookup_t lockflags = lookupflags & INPLOOKUP_LOCKMASK; const inp_lookup_t lockflags = lookupflags & INPLOOKUP_LOCKMASK;
const u_short fport = fport_arg, lport = lport_arg; const u_short fport = fport_arg, lport = lport_arg;
KASSERT((lookupflags & ~INPLOOKUP_MASK) == 0, KASSERT((lookupflags & ~INPLOOKUP_MASK) == 0,
("%s: invalid lookup flags %d", __func__, lookupflags)); ("%s: invalid lookup flags %d", __func__, lookupflags));
KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0, KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0,
("%s: LOCKPCB not set", __func__)); ("%s: LOCKPCB not set", __func__));
smr_enter(pcbinfo->ipi_smr); smr_enter(pcbinfo->ipi_smr);
inp = in_pcblookup_hash_exact(pcbinfo, faddr, fport, laddr, lport); inp = in_pcblookup_exact(pcbinfo, faddr, fport, laddr, lport);
if (inp != NULL) { if (inp != NULL) {
if (__predict_true(inp_smr_lock(inp, lockflags))) { if (__predict_true(inp_smr_lock(inp, lockflags))) {
/* /*
* Revalidate the 4-tuple, the socket could have been * Revalidate the 4-tuple, the socket could have been
* disconnected. * disconnected.
*/ */
if (__predict_true(in_pcblookup_exact_match(inp, if (__predict_true(in_pcblookup_exact_match(inp,
faddr, fport, laddr, lport))) faddr, fport, laddr, lport)))
return (inp); return (inp);
inp_unlock(inp, lockflags); inp_unlock(inp, lockflags);
} }
/* /*
* We failed to lock the inpcb, or its connection state changed * We failed to lock the inpcb, or its connection state changed
* out from under us. Fall back to a precise search. * out from under us. Fall back to a precise search.
*/ */
return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport, return (in_pcblookup_with_lock(pcbinfo, faddr, fport, laddr,
lookupflags, numa_domain, fib)); lport, lookupflags, numa_domain, fib));
} }
if ((lookupflags & INPLOOKUP_WILDCARD) != 0) { if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
inp = in_pcblookup_lbgroup(pcbinfo, &faddr, fport, inp = in_pcblookup_lbgroup(pcbinfo, &faddr, fport,
&laddr, lport, numa_domain, fib); &laddr, lport, numa_domain, fib);
if (inp != NULL) { if (inp != NULL) {
if (__predict_true(inp_smr_lock(inp, lockflags))) { if (__predict_true(inp_smr_lock(inp, lockflags))) {
if (__predict_true(in_pcblookup_wild_match(inp, if (__predict_true(in_pcblookup_wild_match(inp,
laddr, lport, fib) != INPLOOKUP_MATCH_NONE)) laddr, lport, fib) != INPLOOKUP_MATCH_NONE))
return (inp); return (inp);
inp_unlock(inp, lockflags); inp_unlock(inp, lockflags);
} }
inp = INP_LOOKUP_AGAIN; inp = INP_LOOKUP_AGAIN;
} else { } else {
inp = in_pcblookup_hash_wild_smr(pcbinfo, laddr, lport, inp = in_pcblookup_wild_smr(pcbinfo, laddr, lport,
fib, lockflags); fib, lockflags);
} }
if (inp == INP_LOOKUP_AGAIN) { if (inp == INP_LOOKUP_AGAIN) {
return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, return (in_pcblookup_with_lock(pcbinfo, faddr, fport,
lport, lookupflags, numa_domain, fib)); laddr, lport, lookupflags, numa_domain, fib));
} }
} }
if (inp == NULL) if (inp == NULL)
smr_exit(pcbinfo->ipi_smr); smr_exit(pcbinfo->ipi_smr);
return (inp); return (inp);
} }
/* /*
* Public inpcb lookup routines, accepting a 4-tuple, and optionally, an mbuf * Public inpcb lookup routines, accepting a 4-tuple, and optionally, an mbuf
* from which a pre-calculated hash value may be extracted. * from which a pre-calculated hash value may be extracted.
*/ */
struct inpcb * struct inpcb *
in_pcblookup(struct inpcbinfo *pcbinfo, struct in_addr faddr, u_int fport, in_pcblookup(struct inpcbinfo *pcbinfo, struct in_addr faddr, u_int fport,
struct in_addr laddr, u_int lport, int lookupflags, struct in_addr laddr, u_int lport, int lookupflags,
struct ifnet *ifp) struct ifnet *ifp)
{ {
int fib; int fib;
fib = (lookupflags & INPLOOKUP_FIB) ? if_getfib(ifp) : RT_ALL_FIBS; fib = (lookupflags & INPLOOKUP_FIB) ? if_getfib(ifp) : RT_ALL_FIBS;
return (in_pcblookup_hash_smr(pcbinfo, faddr, fport, laddr, lport, return (in_pcblookup_smr(pcbinfo, faddr, fport, laddr, lport,
lookupflags, M_NODOM, fib)); lookupflags, M_NODOM, fib));
} }
struct inpcb * struct inpcb *
in_pcblookup_mbuf(struct inpcbinfo *pcbinfo, struct in_addr faddr, in_pcblookup_mbuf(struct inpcbinfo *pcbinfo, struct in_addr faddr,
u_int fport, struct in_addr laddr, u_int lport, int lookupflags, u_int fport, struct in_addr laddr, u_int lport, int lookupflags,
struct ifnet *ifp __unused, struct mbuf *m) struct ifnet *ifp __unused, struct mbuf *m)
{ {
int fib; int fib;
M_ASSERTPKTHDR(m); M_ASSERTPKTHDR(m);
fib = (lookupflags & INPLOOKUP_FIB) ? M_GETFIB(m) : RT_ALL_FIBS; fib = (lookupflags & INPLOOKUP_FIB) ? M_GETFIB(m) : RT_ALL_FIBS;
return (in_pcblookup_hash_smr(pcbinfo, faddr, fport, laddr, lport, return (in_pcblookup_smr(pcbinfo, faddr, fport, laddr, lport,
lookupflags, m->m_pkthdr.numa_domain, fib)); lookupflags, m->m_pkthdr.numa_domain, fib));
} }
#endif /* INET */ #endif /* INET */
static bool static bool
in_pcbjailed(const struct inpcb *inp, unsigned int flag) in_pcbjailed(const struct inpcb *inp, unsigned int flag)
{ {
return (prison_flag(inp->inp_cred, flag) != 0); return (prison_flag(inp->inp_cred, flag) != 0);
} }
/* /*
* Insert the PCB into a hash chain using ordering rules which ensure that * Insert the PCB into a hash chain using ordering rules which ensure that
* in_pcblookup_hash_wild_*() always encounter the highest-ranking PCB first. * in_pcblookup_wild_*() always encounter the highest-ranking PCB first.
* *
* Specifically, keep jailed PCBs in front of non-jailed PCBs, and keep PCBs * Specifically, keep jailed PCBs in front of non-jailed PCBs, and keep PCBs
* with exact local addresses ahead of wildcard PCBs. Unbound v4-mapped v6 PCBs * with exact local addresses ahead of wildcard PCBs. Unbound v4-mapped v6 PCBs
* always appear last no matter whether they are jailed. * always appear last no matter whether they are jailed.
*/ */
static void static void
_in_pcbinshash_wild(struct inpcbhead *pcbhash, struct inpcb *inp) _in_pcbinshash_wild(struct inpcbhead *pcbhash, struct inpcb *inp)
{ {
▲ Show 20 Lines • Show All 927 Lines • Show Last 20 Lines