Changeset View
Standalone View
sys/netinet/in_pcb.c
Show First 20 Lines • Show All 69 Lines • ▼ Show 20 Lines | |||||
#include <sys/kernel.h> | #include <sys/kernel.h> | ||||
#include <sys/sysctl.h> | #include <sys/sysctl.h> | ||||
#ifdef DDB | #ifdef DDB | ||||
#include <ddb/ddb.h> | #include <ddb/ddb.h> | ||||
#endif | #endif | ||||
#include <vm/uma.h> | #include <vm/uma.h> | ||||
#include <vm/vm.h> | |||||
#include <net/if.h> | #include <net/if.h> | ||||
#include <net/if_var.h> | #include <net/if_var.h> | ||||
#include <net/if_types.h> | #include <net/if_types.h> | ||||
#include <net/if_llatbl.h> | #include <net/if_llatbl.h> | ||||
#include <net/route.h> | #include <net/route.h> | ||||
#include <net/rss_config.h> | #include <net/rss_config.h> | ||||
#include <net/vnet.h> | #include <net/vnet.h> | ||||
▲ Show 20 Lines • Show All 59 Lines • ▼ Show 20 Lines | |||||
#define V_ipport_tcplastcount VNET(ipport_tcplastcount) | #define V_ipport_tcplastcount VNET(ipport_tcplastcount) | ||||
static void in_pcbremlists(struct inpcb *inp); | static void in_pcbremlists(struct inpcb *inp); | ||||
#ifdef INET | #ifdef INET | ||||
static struct inpcb *in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, | static struct inpcb *in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, | ||||
struct in_addr faddr, u_int fport_arg, | struct in_addr faddr, u_int fport_arg, | ||||
struct in_addr laddr, u_int lport_arg, | struct in_addr laddr, u_int lport_arg, | ||||
int lookupflags, struct ifnet *ifp); | int lookupflags, struct ifnet *ifp, | ||||
uint8_t numa_domain); | |||||
bz: il_numa_domain is a u_char (uint8_t), here (and possibly elsewhere) we are using an int? Is… | |||||
Done Inline ActionsChanged to uint8_t gallatin: Changed to uint8_t | |||||
#define RANGECHK(var, min, max) \ | #define RANGECHK(var, min, max) \ | ||||
if ((var) < (min)) { (var) = (min); } \ | if ((var) < (min)) { (var) = (min); } \ | ||||
else if ((var) > (max)) { (var) = (max); } | else if ((var) > (max)) { (var) = (max); } | ||||
static int | static int | ||||
sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS) | sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 81 Lines • ▼ Show 20 Lines | |||||
* | * | ||||
* NOTE: It is assumed that most of these functions will be called with | * NOTE: It is assumed that most of these functions will be called with | ||||
* the pcbinfo lock held, and often, the inpcb lock held, as these utility | * the pcbinfo lock held, and often, the inpcb lock held, as these utility | ||||
* functions often modify hash chains or addresses in pcbs. | * functions often modify hash chains or addresses in pcbs. | ||||
*/ | */ | ||||
static struct inpcblbgroup * | static struct inpcblbgroup * | ||||
in_pcblbgroup_alloc(struct inpcblbgrouphead *hdr, u_char vflag, | in_pcblbgroup_alloc(struct inpcblbgrouphead *hdr, u_char vflag, | ||||
uint16_t port, const union in_dependaddr *addr, int size) | uint16_t port, const union in_dependaddr *addr, int size, | ||||
uint8_t numa_domain) | |||||
{ | { | ||||
struct inpcblbgroup *grp; | struct inpcblbgroup *grp; | ||||
size_t bytes; | size_t bytes; | ||||
bytes = __offsetof(struct inpcblbgroup, il_inp[size]); | bytes = __offsetof(struct inpcblbgroup, il_inp[size]); | ||||
grp = malloc(bytes, M_PCB, M_ZERO | M_NOWAIT); | grp = malloc(bytes, M_PCB, M_ZERO | M_NOWAIT); | ||||
if (!grp) | if (!grp) | ||||
return (NULL); | return (NULL); | ||||
grp->il_vflag = vflag; | grp->il_vflag = vflag; | ||||
grp->il_lport = port; | grp->il_lport = port; | ||||
grp->il_numa_domain = numa_domain; | |||||
grp->il_dependladdr = *addr; | grp->il_dependladdr = *addr; | ||||
grp->il_inpsiz = size; | grp->il_inpsiz = size; | ||||
CK_LIST_INSERT_HEAD(hdr, grp, il_list); | CK_LIST_INSERT_HEAD(hdr, grp, il_list); | ||||
return (grp); | return (grp); | ||||
} | } | ||||
static void | static void | ||||
in_pcblbgroup_free_deferred(epoch_context_t ctx) | in_pcblbgroup_free_deferred(epoch_context_t ctx) | ||||
Show All 15 Lines | |||||
static struct inpcblbgroup * | static struct inpcblbgroup * | ||||
in_pcblbgroup_resize(struct inpcblbgrouphead *hdr, | in_pcblbgroup_resize(struct inpcblbgrouphead *hdr, | ||||
struct inpcblbgroup *old_grp, int size) | struct inpcblbgroup *old_grp, int size) | ||||
{ | { | ||||
struct inpcblbgroup *grp; | struct inpcblbgroup *grp; | ||||
int i; | int i; | ||||
grp = in_pcblbgroup_alloc(hdr, old_grp->il_vflag, | grp = in_pcblbgroup_alloc(hdr, old_grp->il_vflag, | ||||
old_grp->il_lport, &old_grp->il_dependladdr, size); | old_grp->il_lport, &old_grp->il_dependladdr, size, | ||||
old_grp->il_numa_domain); | |||||
if (grp == NULL) | if (grp == NULL) | ||||
return (NULL); | return (NULL); | ||||
KASSERT(old_grp->il_inpcnt < grp->il_inpsiz, | KASSERT(old_grp->il_inpcnt < grp->il_inpsiz, | ||||
("invalid new local group size %d and old local group count %d", | ("invalid new local group size %d and old local group count %d", | ||||
grp->il_inpsiz, old_grp->il_inpcnt)); | grp->il_inpsiz, old_grp->il_inpcnt)); | ||||
for (i = 0; i < old_grp->il_inpcnt; ++i) | for (i = 0; i < old_grp->il_inpcnt; ++i) | ||||
Show All 26 Lines | if (new_grp != NULL) | ||||
*grpp = new_grp; | *grpp = new_grp; | ||||
} | } | ||||
} | } | ||||
/* | /* | ||||
* Add PCB to load balance group for SO_REUSEPORT_LB option. | * Add PCB to load balance group for SO_REUSEPORT_LB option. | ||||
*/ | */ | ||||
static int | static int | ||||
in_pcbinslbgrouphash(struct inpcb *inp) | in_pcbinslbgrouphash(struct inpcb *inp, uint8_t numa_domain) | ||||
{ | { | ||||
const static struct timeval interval = { 60, 0 }; | const static struct timeval interval = { 60, 0 }; | ||||
static struct timeval lastprint; | static struct timeval lastprint; | ||||
struct inpcbinfo *pcbinfo; | struct inpcbinfo *pcbinfo; | ||||
struct inpcblbgrouphead *hdr; | struct inpcblbgrouphead *hdr; | ||||
struct inpcblbgroup *grp; | struct inpcblbgroup *grp; | ||||
uint32_t idx; | uint32_t idx; | ||||
Show All 19 Lines | #ifdef INET6 | ||||
} | } | ||||
#endif | #endif | ||||
idx = INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask); | idx = INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask); | ||||
hdr = &pcbinfo->ipi_lbgrouphashbase[idx]; | hdr = &pcbinfo->ipi_lbgrouphashbase[idx]; | ||||
CK_LIST_FOREACH(grp, hdr, il_list) { | CK_LIST_FOREACH(grp, hdr, il_list) { | ||||
if (grp->il_vflag == inp->inp_vflag && | if (grp->il_vflag == inp->inp_vflag && | ||||
grp->il_lport == inp->inp_lport && | grp->il_lport == inp->inp_lport && | ||||
grp->il_numa_domain == numa_domain && | |||||
memcmp(&grp->il_dependladdr, | memcmp(&grp->il_dependladdr, | ||||
&inp->inp_inc.inc_ie.ie_dependladdr, | &inp->inp_inc.inc_ie.ie_dependladdr, | ||||
sizeof(grp->il_dependladdr)) == 0) | sizeof(grp->il_dependladdr)) == 0) | ||||
break; | break; | ||||
} | } | ||||
if (grp == NULL) { | if (grp == NULL) { | ||||
/* Create new load balance group. */ | /* Create new load balance group. */ | ||||
grp = in_pcblbgroup_alloc(hdr, inp->inp_vflag, | grp = in_pcblbgroup_alloc(hdr, inp->inp_vflag, | ||||
inp->inp_lport, &inp->inp_inc.inc_ie.ie_dependladdr, | inp->inp_lport, &inp->inp_inc.inc_ie.ie_dependladdr, | ||||
INPCBLBGROUP_SIZMIN); | INPCBLBGROUP_SIZMIN, numa_domain); | ||||
if (grp == NULL) | if (grp == NULL) | ||||
return (ENOBUFS); | return (ENOBUFS); | ||||
} else if (grp->il_inpcnt == grp->il_inpsiz) { | } else if (grp->il_inpcnt == grp->il_inpsiz) { | ||||
Done Inline Actionsin_pcblgroup_alloc is a static function. Is there any reason not to pass numa_domain in to it as well given the function initializes other fields as well? bz: in_pcblgroup_alloc is a static function. Is there any reason not to pass numa_domain in to it… | |||||
if (grp->il_inpsiz >= INPCBLBGROUP_SIZMAX) { | if (grp->il_inpsiz >= INPCBLBGROUP_SIZMAX) { | ||||
if (ratecheck(&lastprint, &interval)) | if (ratecheck(&lastprint, &interval)) | ||||
printf("lb group port %d, limit reached\n", | printf("lb group port %d, limit reached\n", | ||||
ntohs(grp->il_lport)); | ntohs(grp->il_lport)); | ||||
return (0); | return (0); | ||||
} | } | ||||
/* Expand this local group. */ | /* Expand this local group. */ | ||||
▲ Show 20 Lines • Show All 41 Lines • ▼ Show 20 Lines | for (i = 0; i < grp->il_inpcnt; ++i) { | ||||
/* Pull up inpcbs, shrink group if possible. */ | /* Pull up inpcbs, shrink group if possible. */ | ||||
in_pcblbgroup_reorder(hdr, &grp, i); | in_pcblbgroup_reorder(hdr, &grp, i); | ||||
} | } | ||||
return; | return; | ||||
} | } | ||||
} | } | ||||
} | } | ||||
int | |||||
in_pcblbgroup_numa(struct inpcb *inp, int arg) | |||||
{ | |||||
struct inpcbinfo *pcbinfo; | |||||
struct inpcblbgrouphead *hdr; | |||||
struct inpcblbgroup *grp; | |||||
int err, i; | |||||
uint8_t numa_domain; | |||||
switch (arg) { | |||||
case TCP_REUSPORT_LB_NUMA_NODOM: | |||||
Done Inline ActionsNeither of those two need to be initialized here; there are possible return() calls before this, so should be moved down. Would probably also help to have a more consisten use of pcbinfo vs. inp->inp_pcbinfo in the code below. bz: Neither of those two need to be initialized here; there are possible return() calls before… | |||||
numa_domain = M_NODOM; | |||||
break; | |||||
case TCP_REUSPORT_LB_NUMA_CURDOM: | |||||
numa_domain = PCPU_GET(domain); | |||||
break; | |||||
default: | |||||
if (arg < 0 || arg >= vm_ndomains) | |||||
return (EINVAL); | |||||
else | |||||
jhbUnsubmitted Not Done Inline ActionsSuggestion: you could drop this else since you return in the if. jhb: Suggestion: you could drop this else since you return in the if. | |||||
numa_domain = arg; | |||||
} | |||||
err = 0; | |||||
pcbinfo = inp->inp_pcbinfo; | |||||
INP_WLOCK_ASSERT(inp); | |||||
INP_HASH_WLOCK(pcbinfo); | |||||
hdr = &pcbinfo->ipi_lbgrouphashbase[ | |||||
INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask)]; | |||||
CK_LIST_FOREACH(grp, hdr, il_list) { | |||||
Done Inline Actionsextra blank line. bz: extra blank line. | |||||
for (i = 0; i < grp->il_inpcnt; ++i) { | |||||
if (grp->il_inp[i] != inp) | |||||
continue; | |||||
if (grp->il_numa_domain == numa_domain) { | |||||
goto abort_with_hash_wlock; | |||||
} | |||||
Not Done Inline ActionsIf I am not mistaken err = 0; here and then there'll only be two goto calls, which I am fine with as writing it that way saves a level of indentation. The early err assignment saves a vertical block. bz: If I am not mistaken err = 0; here and then there'll only be two goto calls, which I am fine… | |||||
Done Inline ActionsYes, re-reading it, it is cleaner to reverse the logic. gallatin: Yes, re-reading it, it is cleaner to reverse the logic. | |||||
/* Remove it from the old group. */ | |||||
jhbUnsubmitted Not Done Inline ActionsBlank lines before comments is typical in the tree. jhb: Blank lines before comments is typical in the tree. | |||||
in_pcbremlbgrouphash(inp); | |||||
/* Add it to the new group based on numa domain. */ | |||||
in_pcbinslbgrouphash(inp, numa_domain); | |||||
goto abort_with_hash_wlock; | |||||
Done Inline ActionsComments start with upper case and end in punctuation. bz: Comments start with upper case and end in punctuation. | |||||
} | |||||
} | |||||
err = ENOENT; | |||||
abort_with_hash_wlock: | |||||
INP_HASH_WUNLOCK(pcbinfo); | |||||
return (err); | |||||
} | |||||
/* | /* | ||||
* Different protocols initialize their inpcbs differently - giving | * Different protocols initialize their inpcbs differently - giving | ||||
* different name to the lock. But they all are disposed the same. | * different name to the lock. But they all are disposed the same. | ||||
*/ | */ | ||||
static void | static void | ||||
inpcb_fini(void *mem, int size) | inpcb_fini(void *mem, int size) | ||||
{ | { | ||||
struct inpcb *inp = mem; | struct inpcb *inp = mem; | ||||
▲ Show 20 Lines • Show All 276 Lines • ▼ Show 20 Lines | if (*lastport < first || *lastport > last) | ||||
*lastport = first; | *lastport = first; | ||||
lport = htons(*lastport); | lport = htons(*lastport); | ||||
if (fsa != NULL) { | if (fsa != NULL) { | ||||
#ifdef INET | #ifdef INET | ||||
if (lsa->sa_family == AF_INET) { | if (lsa->sa_family == AF_INET) { | ||||
tmpinp = in_pcblookup_hash_locked(pcbinfo, | tmpinp = in_pcblookup_hash_locked(pcbinfo, | ||||
faddr, fport, laddr, lport, lookupflags, | faddr, fport, laddr, lport, lookupflags, | ||||
NULL); | NULL, M_NODOM); | ||||
} | } | ||||
#endif | #endif | ||||
#ifdef INET6 | #ifdef INET6 | ||||
if (lsa->sa_family == AF_INET6) { | if (lsa->sa_family == AF_INET6) { | ||||
tmpinp = in6_pcblookup_hash_locked(pcbinfo, | tmpinp = in6_pcblookup_hash_locked(pcbinfo, | ||||
faddr6, fport, laddr6, lport, lookupflags, | faddr6, fport, laddr6, lport, lookupflags, | ||||
NULL); | NULL, M_NODOM); | ||||
} | } | ||||
#endif | #endif | ||||
} else { | } else { | ||||
#ifdef INET6 | #ifdef INET6 | ||||
if ((inp->inp_vflag & INP_IPV6) != 0) | if ((inp->inp_vflag & INP_IPV6) != 0) | ||||
tmpinp = in6_pcblookup_local(pcbinfo, | tmpinp = in6_pcblookup_local(pcbinfo, | ||||
&inp->in6p_laddr, lport, lookupflags, cred); | &inp->in6p_laddr, lport, lookupflags, cred); | ||||
#endif | #endif | ||||
▲ Show 20 Lines • Show All 644 Lines • ▼ Show 20 Lines | if (IN_MULTICAST(ntohl(faddr.s_addr)) && | ||||
error = 0; | error = 0; | ||||
} | } | ||||
IN_IFADDR_RUNLOCK(&in_ifa_tracker); | IN_IFADDR_RUNLOCK(&in_ifa_tracker); | ||||
} | } | ||||
} | } | ||||
if (error) | if (error) | ||||
return (error); | return (error); | ||||
} | } | ||||
if (lport != 0) { | if (lport != 0) { | ||||
oinp = in_pcblookup_hash_locked(inp->inp_pcbinfo, faddr, | oinp = in_pcblookup_hash_locked(inp->inp_pcbinfo, faddr, | ||||
fport, laddr, lport, 0, NULL); | fport, laddr, lport, 0, NULL, M_NODOM); | ||||
if (oinp != NULL) { | if (oinp != NULL) { | ||||
if (oinpp != NULL) | if (oinpp != NULL) | ||||
*oinpp = oinp; | *oinpp = oinp; | ||||
return (EADDRINUSE); | return (EADDRINUSE); | ||||
} | } | ||||
} else { | } else { | ||||
struct sockaddr_in lsin, fsin; | struct sockaddr_in lsin, fsin; | ||||
▲ Show 20 Lines • Show All 601 Lines • ▼ Show 20 Lines | #endif | ||||
return (match); | return (match); | ||||
} | } | ||||
} | } | ||||
#undef INP_LOOKUP_MAPPED_PCB_COST | #undef INP_LOOKUP_MAPPED_PCB_COST | ||||
static struct inpcb * | static struct inpcb * | ||||
in_pcblookup_lbgroup(const struct inpcbinfo *pcbinfo, | in_pcblookup_lbgroup(const struct inpcbinfo *pcbinfo, | ||||
const struct in_addr *laddr, uint16_t lport, const struct in_addr *faddr, | const struct in_addr *laddr, uint16_t lport, const struct in_addr *faddr, | ||||
uint16_t fport, int lookupflags) | uint16_t fport, int lookupflags, int numa_domain) | ||||
{ | { | ||||
struct inpcb *local_wild; | struct inpcb *local_wild, *numa_wild; | ||||
const struct inpcblbgrouphead *hdr; | const struct inpcblbgrouphead *hdr; | ||||
Not Done Inline ActionsCould go on one line. bz: Could go on one line. | |||||
struct inpcblbgroup *grp; | struct inpcblbgroup *grp; | ||||
uint32_t idx; | uint32_t idx; | ||||
INP_HASH_LOCK_ASSERT(pcbinfo); | INP_HASH_LOCK_ASSERT(pcbinfo); | ||||
hdr = &pcbinfo->ipi_lbgrouphashbase[ | hdr = &pcbinfo->ipi_lbgrouphashbase[ | ||||
INP_PCBPORTHASH(lport, pcbinfo->ipi_lbgrouphashmask)]; | INP_PCBPORTHASH(lport, pcbinfo->ipi_lbgrouphashmask)]; | ||||
/* | /* | ||||
* Order of socket selection: | * Order of socket selection: | ||||
* 1. non-wild. | * 1. non-wild. | ||||
* 2. wild (if lookupflags contains INPLOOKUP_WILDCARD). | * 2. wild (if lookupflags contains INPLOOKUP_WILDCARD). | ||||
* | * | ||||
* NOTE: | * NOTE: | ||||
* - Load balanced group does not contain jailed sockets | * - Load balanced group does not contain jailed sockets | ||||
* - Load balanced group does not contain IPv4 mapped INET6 wild sockets | * - Load balanced group does not contain IPv4 mapped INET6 wild sockets | ||||
*/ | */ | ||||
local_wild = NULL; | local_wild = NULL; | ||||
numa_wild = NULL; | |||||
CK_LIST_FOREACH(grp, hdr, il_list) { | CK_LIST_FOREACH(grp, hdr, il_list) { | ||||
#ifdef INET6 | #ifdef INET6 | ||||
if (!(grp->il_vflag & INP_IPV4)) | if (!(grp->il_vflag & INP_IPV4)) | ||||
continue; | continue; | ||||
#endif | #endif | ||||
if (grp->il_lport != lport) | if (grp->il_lport != lport) | ||||
continue; | continue; | ||||
idx = INP_PCBLBGROUP_PKTHASH(faddr->s_addr, lport, fport) % | idx = INP_PCBLBGROUP_PKTHASH(faddr->s_addr, lport, fport) % | ||||
grp->il_inpcnt; | grp->il_inpcnt; | ||||
if (grp->il_laddr.s_addr == laddr->s_addr) | if (grp->il_laddr.s_addr == laddr->s_addr) { | ||||
if (numa_domain == M_NODOM || | |||||
grp->il_numa_domain == numa_domain) { | |||||
return (grp->il_inp[idx]); | return (grp->il_inp[idx]); | ||||
} else { | |||||
Done Inline ActionsOne one-line block has {} the other doesn't. bz: One one-line block has {} the other doesn't. | |||||
numa_wild = grp->il_inp[idx]; | |||||
} | |||||
} | |||||
if (grp->il_laddr.s_addr == INADDR_ANY && | if (grp->il_laddr.s_addr == INADDR_ANY && | ||||
(lookupflags & INPLOOKUP_WILDCARD) != 0) | (lookupflags & INPLOOKUP_WILDCARD) != 0 && | ||||
(local_wild == NULL || numa_domain == M_NODOM || | |||||
grp->il_numa_domain == numa_domain)) { | |||||
local_wild = grp->il_inp[idx]; | local_wild = grp->il_inp[idx]; | ||||
} | } | ||||
} | |||||
if (numa_wild != NULL) | |||||
Done Inline ActionsThis is just another || condition if the rest of the logic is right that it is local_wild both times? bz: This is just another || condition if the rest of the logic is right that it is local_wild both… | |||||
Done Inline ActionsIndeed, this can be simplified a bit. gallatin: Indeed, this can be simplified a bit. | |||||
return (numa_wild); | |||||
return (local_wild); | return (local_wild); | ||||
} | } | ||||
#ifdef PCBGROUP | #ifdef PCBGROUP | ||||
/* | /* | ||||
* Lookup PCB in hash list, using pcbgroup tables. | * Lookup PCB in hash list, using pcbgroup tables. | ||||
*/ | */ | ||||
static struct inpcb * | static struct inpcb * | ||||
▲ Show 20 Lines • Show All 230 Lines • ▼ Show 20 Lines | |||||
/* | /* | ||||
* Lookup PCB in hash list, using pcbinfo tables. This variation assumes | * Lookup PCB in hash list, using pcbinfo tables. This variation assumes | ||||
* that the caller has locked the hash list, and will not perform any further | * that the caller has locked the hash list, and will not perform any further | ||||
* locking or reference operations on either the hash list or the connection. | * locking or reference operations on either the hash list or the connection. | ||||
*/ | */ | ||||
static struct inpcb * | static struct inpcb * | ||||
in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, struct in_addr faddr, | in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, struct in_addr faddr, | ||||
u_int fport_arg, struct in_addr laddr, u_int lport_arg, int lookupflags, | u_int fport_arg, struct in_addr laddr, u_int lport_arg, int lookupflags, | ||||
struct ifnet *ifp) | struct ifnet *ifp, uint8_t numa_domain) | ||||
{ | { | ||||
struct inpcbhead *head; | struct inpcbhead *head; | ||||
struct inpcb *inp, *tmpinp; | struct inpcb *inp, *tmpinp; | ||||
u_short fport = fport_arg, lport = lport_arg; | u_short fport = fport_arg, lport = lport_arg; | ||||
KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD)) == 0, | KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD)) == 0, | ||||
("%s: invalid lookup flags %d", __func__, lookupflags)); | ("%s: invalid lookup flags %d", __func__, lookupflags)); | ||||
INP_HASH_LOCK_ASSERT(pcbinfo); | INP_HASH_LOCK_ASSERT(pcbinfo); | ||||
Show All 28 Lines | #endif | ||||
if (tmpinp != NULL) | if (tmpinp != NULL) | ||||
return (tmpinp); | return (tmpinp); | ||||
/* | /* | ||||
* Then look in lb group (for wildcard match). | * Then look in lb group (for wildcard match). | ||||
*/ | */ | ||||
if ((lookupflags & INPLOOKUP_WILDCARD) != 0) { | if ((lookupflags & INPLOOKUP_WILDCARD) != 0) { | ||||
inp = in_pcblookup_lbgroup(pcbinfo, &laddr, lport, &faddr, | inp = in_pcblookup_lbgroup(pcbinfo, &laddr, lport, &faddr, | ||||
fport, lookupflags); | fport, lookupflags, numa_domain); | ||||
if (inp != NULL) | if (inp != NULL) | ||||
return (inp); | return (inp); | ||||
} | } | ||||
/* | /* | ||||
* Then look for a wildcard match, if requested. | * Then look for a wildcard match, if requested. | ||||
*/ | */ | ||||
if ((lookupflags & INPLOOKUP_WILDCARD) != 0) { | if ((lookupflags & INPLOOKUP_WILDCARD) != 0) { | ||||
▲ Show 20 Lines • Show All 70 Lines • ▼ Show 20 Lines | |||||
/* | /* | ||||
* Lookup PCB in hash list, using pcbinfo tables. This variation locks the | * Lookup PCB in hash list, using pcbinfo tables. This variation locks the | ||||
* hash list lock, and will return the inpcb locked (i.e., requires | * hash list lock, and will return the inpcb locked (i.e., requires | ||||
* INPLOOKUP_LOCKPCB). | * INPLOOKUP_LOCKPCB). | ||||
*/ | */ | ||||
static struct inpcb * | static struct inpcb * | ||||
in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr, | in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr, | ||||
u_int fport, struct in_addr laddr, u_int lport, int lookupflags, | u_int fport, struct in_addr laddr, u_int lport, int lookupflags, | ||||
struct ifnet *ifp) | struct ifnet *ifp, uint8_t numa_domain) | ||||
{ | { | ||||
struct inpcb *inp; | struct inpcb *inp; | ||||
inp = in_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport, | inp = in_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport, | ||||
(lookupflags & ~(INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)), ifp); | (lookupflags & ~(INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)), ifp, | ||||
numa_domain); | |||||
if (inp != NULL) { | if (inp != NULL) { | ||||
if (lookupflags & INPLOOKUP_WLOCKPCB) { | if (lookupflags & INPLOOKUP_WLOCKPCB) { | ||||
INP_WLOCK(inp); | INP_WLOCK(inp); | ||||
if (__predict_false(inp->inp_flags2 & INP_FREED)) { | if (__predict_false(inp->inp_flags2 & INP_FREED)) { | ||||
INP_WUNLOCK(inp); | INP_WUNLOCK(inp); | ||||
inp = NULL; | inp = NULL; | ||||
} | } | ||||
} else if (lookupflags & INPLOOKUP_RLOCKPCB) { | } else if (lookupflags & INPLOOKUP_RLOCKPCB) { | ||||
▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Lines | #if defined(PCBGROUP) && !defined(RSS) | ||||
if (in_pcbgroup_enabled(pcbinfo)) { | if (in_pcbgroup_enabled(pcbinfo)) { | ||||
pcbgroup = in_pcbgroup_bytuple(pcbinfo, laddr, lport, faddr, | pcbgroup = in_pcbgroup_bytuple(pcbinfo, laddr, lport, faddr, | ||||
fport); | fport); | ||||
return (in_pcblookup_group(pcbinfo, pcbgroup, faddr, fport, | return (in_pcblookup_group(pcbinfo, pcbgroup, faddr, fport, | ||||
laddr, lport, lookupflags, ifp)); | laddr, lport, lookupflags, ifp)); | ||||
} | } | ||||
#endif | #endif | ||||
return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport, | return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport, | ||||
lookupflags, ifp)); | lookupflags, ifp, M_NODOM)); | ||||
} | } | ||||
struct inpcb * | struct inpcb * | ||||
in_pcblookup_mbuf(struct inpcbinfo *pcbinfo, struct in_addr faddr, | in_pcblookup_mbuf(struct inpcbinfo *pcbinfo, struct in_addr faddr, | ||||
u_int fport, struct in_addr laddr, u_int lport, int lookupflags, | u_int fport, struct in_addr laddr, u_int lport, int lookupflags, | ||||
struct ifnet *ifp, struct mbuf *m) | struct ifnet *ifp, struct mbuf *m) | ||||
{ | { | ||||
#ifdef PCBGROUP | #ifdef PCBGROUP | ||||
Show All 25 Lines | #ifndef RSS | ||||
pcbgroup = in_pcbgroup_bytuple(pcbinfo, laddr, lport, faddr, | pcbgroup = in_pcbgroup_bytuple(pcbinfo, laddr, lport, faddr, | ||||
fport); | fport); | ||||
return (in_pcblookup_group(pcbinfo, pcbgroup, faddr, fport, | return (in_pcblookup_group(pcbinfo, pcbgroup, faddr, fport, | ||||
laddr, lport, lookupflags, ifp)); | laddr, lport, lookupflags, ifp)); | ||||
#endif | #endif | ||||
} | } | ||||
#endif | #endif | ||||
return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport, | return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport, | ||||
lookupflags, ifp)); | lookupflags, ifp, m->m_pkthdr.numa_domain)); | ||||
} | } | ||||
#endif /* INET */ | #endif /* INET */ | ||||
/* | /* | ||||
* Insert PCB onto various hash lists. | * Insert PCB onto various hash lists. | ||||
*/ | */ | ||||
static int | static int | ||||
in_pcbinshash_internal(struct inpcb *inp, struct mbuf *m) | in_pcbinshash_internal(struct inpcb *inp, struct mbuf *m) | ||||
Show All 25 Lines | pcbporthash = &pcbinfo->ipi_porthashbase[ | ||||
INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_porthashmask)]; | INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_porthashmask)]; | ||||
/* | /* | ||||
* Add entry to load balance group. | * Add entry to load balance group. | ||||
* Only do this if SO_REUSEPORT_LB is set. | * Only do this if SO_REUSEPORT_LB is set. | ||||
*/ | */ | ||||
so_options = inp_so_options(inp); | so_options = inp_so_options(inp); | ||||
if (so_options & SO_REUSEPORT_LB) { | if (so_options & SO_REUSEPORT_LB) { | ||||
int ret = in_pcbinslbgrouphash(inp); | int ret = in_pcbinslbgrouphash(inp, M_NODOM); | ||||
if (ret) { | if (ret) { | ||||
/* pcb lb group malloc fail (ret=ENOBUFS). */ | /* pcb lb group malloc fail (ret=ENOBUFS). */ | ||||
return (ret); | return (ret); | ||||
} | } | ||||
} | } | ||||
/* | /* | ||||
* Go through port list and look for a head for this lport. | * Go through port list and look for a head for this lport. | ||||
▲ Show 20 Lines • Show All 919 Lines • Show Last 20 Lines |
il_numa_domain is a u_char (uint8_t), here (and possibly elsewhere) we are using an int? Is there a reason the types don't agree? I always found these things highly confusing in other old code when I had to deal with that. I understand that in other places the -1/-2 (TCP_REUSPORT_LB_NUMA_*) arguments are used outside the valid range.