Differential D21636 Diff 80965 head/sys/netinet/in_pcb.c

Changeset View

Standalone View

head/sys/netinet/in_pcb.c

Show First 20 Lines • Show All 69 Lines • ▼ Show 20 Lines
#include <sys/kernel.h>		#include <sys/kernel.h>
#include <sys/sysctl.h>		#include <sys/sysctl.h>

#ifdef DDB		#ifdef DDB
#include <ddb/ddb.h>		#include <ddb/ddb.h>
#endif		#endif

#include <vm/uma.h>		#include <vm/uma.h>
		#include <vm/vm.h>

#include <net/if.h>		#include <net/if.h>
#include <net/if_var.h>		#include <net/if_var.h>
#include <net/if_types.h>		#include <net/if_types.h>
#include <net/if_llatbl.h>		#include <net/if_llatbl.h>
#include <net/route.h>		#include <net/route.h>
#include <net/rss_config.h>		#include <net/rss_config.h>
#include <net/vnet.h>		#include <net/vnet.h>
▲ Show 20 Lines • Show All 59 Lines • ▼ Show 20 Lines

#define V_ipport_tcplastcount VNET(ipport_tcplastcount)		#define V_ipport_tcplastcount VNET(ipport_tcplastcount)

static void in_pcbremlists(struct inpcb *inp);		static void in_pcbremlists(struct inpcb *inp);
#ifdef INET		#ifdef INET
static struct inpcb in_pcblookup_hash_locked(struct inpcbinfo pcbinfo,		static struct inpcb in_pcblookup_hash_locked(struct inpcbinfo pcbinfo,
struct in_addr faddr, u_int fport_arg,		struct in_addr faddr, u_int fport_arg,
struct in_addr laddr, u_int lport_arg,		struct in_addr laddr, u_int lport_arg,
int lookupflags, struct ifnet *ifp);		int lookupflags, struct ifnet *ifp,
		uint8_t numa_domain);

#define RANGECHK(var, min, max) \		#define RANGECHK(var, min, max) \
if ((var) < (min)) { (var) = (min); } \		if ((var) < (min)) { (var) = (min); } \
else if ((var) > (max)) { (var) = (max); }		else if ((var) > (max)) { (var) = (max); }

static int		static int
sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS)		sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS)
{		{
▲ Show 20 Lines • Show All 81 Lines • ▼ Show 20 Lines
*		*
* NOTE: It is assumed that most of these functions will be called with		* NOTE: It is assumed that most of these functions will be called with
* the pcbinfo lock held, and often, the inpcb lock held, as these utility		* the pcbinfo lock held, and often, the inpcb lock held, as these utility
* functions often modify hash chains or addresses in pcbs.		* functions often modify hash chains or addresses in pcbs.
*/		*/

static struct inpcblbgroup *		static struct inpcblbgroup *
in_pcblbgroup_alloc(struct inpcblbgrouphead *hdr, u_char vflag,		in_pcblbgroup_alloc(struct inpcblbgrouphead *hdr, u_char vflag,
uint16_t port, const union in_dependaddr *addr, int size)		uint16_t port, const union in_dependaddr *addr, int size,
		uint8_t numa_domain)
{		{
struct inpcblbgroup *grp;		struct inpcblbgroup *grp;
size_t bytes;		size_t bytes;

bytes = __offsetof(struct inpcblbgroup, il_inp[size]);		bytes = __offsetof(struct inpcblbgroup, il_inp[size]);
grp = malloc(bytes, M_PCB, M_ZERO \| M_NOWAIT);		grp = malloc(bytes, M_PCB, M_ZERO \| M_NOWAIT);
if (!grp)		if (!grp)
return (NULL);		return (NULL);
grp->il_vflag = vflag;		grp->il_vflag = vflag;
grp->il_lport = port;		grp->il_lport = port;
		grp->il_numa_domain = numa_domain;
grp->il_dependladdr = *addr;		grp->il_dependladdr = *addr;
grp->il_inpsiz = size;		grp->il_inpsiz = size;
CK_LIST_INSERT_HEAD(hdr, grp, il_list);		CK_LIST_INSERT_HEAD(hdr, grp, il_list);
return (grp);		return (grp);
}		}

static void		static void
in_pcblbgroup_free_deferred(epoch_context_t ctx)		in_pcblbgroup_free_deferred(epoch_context_t ctx)
Show All 15 Lines
static struct inpcblbgroup *		static struct inpcblbgroup *
in_pcblbgroup_resize(struct inpcblbgrouphead *hdr,		in_pcblbgroup_resize(struct inpcblbgrouphead *hdr,
struct inpcblbgroup *old_grp, int size)		struct inpcblbgroup *old_grp, int size)
{		{
struct inpcblbgroup *grp;		struct inpcblbgroup *grp;
int i;		int i;

grp = in_pcblbgroup_alloc(hdr, old_grp->il_vflag,		grp = in_pcblbgroup_alloc(hdr, old_grp->il_vflag,
old_grp->il_lport, &old_grp->il_dependladdr, size);		old_grp->il_lport, &old_grp->il_dependladdr, size,
		old_grp->il_numa_domain);
if (grp == NULL)		if (grp == NULL)
return (NULL);		return (NULL);

KASSERT(old_grp->il_inpcnt < grp->il_inpsiz,		KASSERT(old_grp->il_inpcnt < grp->il_inpsiz,
("invalid new local group size %d and old local group count %d",		("invalid new local group size %d and old local group count %d",
grp->il_inpsiz, old_grp->il_inpcnt));		grp->il_inpsiz, old_grp->il_inpcnt));

for (i = 0; i < old_grp->il_inpcnt; ++i)		for (i = 0; i < old_grp->il_inpcnt; ++i)
Show All 26 Lines	if (new_grp != NULL)
*grpp = new_grp;		*grpp = new_grp;
}		}
}		}

/*		/*
* Add PCB to load balance group for SO_REUSEPORT_LB option.		* Add PCB to load balance group for SO_REUSEPORT_LB option.
*/		*/
static int		static int
in_pcbinslbgrouphash(struct inpcb *inp)		in_pcbinslbgrouphash(struct inpcb *inp, uint8_t numa_domain)
{		{
const static struct timeval interval = { 60, 0 };		const static struct timeval interval = { 60, 0 };
static struct timeval lastprint;		static struct timeval lastprint;
struct inpcbinfo *pcbinfo;		struct inpcbinfo *pcbinfo;
struct inpcblbgrouphead *hdr;		struct inpcblbgrouphead *hdr;
struct inpcblbgroup *grp;		struct inpcblbgroup *grp;
uint32_t idx;		uint32_t idx;

Show All 19 Lines	#ifdef INET6
}		}
#endif		#endif

idx = INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask);		idx = INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask);
hdr = &pcbinfo->ipi_lbgrouphashbase[idx];		hdr = &pcbinfo->ipi_lbgrouphashbase[idx];
CK_LIST_FOREACH(grp, hdr, il_list) {		CK_LIST_FOREACH(grp, hdr, il_list) {
if (grp->il_vflag == inp->inp_vflag &&		if (grp->il_vflag == inp->inp_vflag &&
grp->il_lport == inp->inp_lport &&		grp->il_lport == inp->inp_lport &&
		grp->il_numa_domain == numa_domain &&
memcmp(&grp->il_dependladdr,		memcmp(&grp->il_dependladdr,
&inp->inp_inc.inc_ie.ie_dependladdr,		&inp->inp_inc.inc_ie.ie_dependladdr,
sizeof(grp->il_dependladdr)) == 0)		sizeof(grp->il_dependladdr)) == 0)
break;		break;
}		}
if (grp == NULL) {		if (grp == NULL) {
/* Create new load balance group. */		/* Create new load balance group. */
grp = in_pcblbgroup_alloc(hdr, inp->inp_vflag,		grp = in_pcblbgroup_alloc(hdr, inp->inp_vflag,
inp->inp_lport, &inp->inp_inc.inc_ie.ie_dependladdr,		inp->inp_lport, &inp->inp_inc.inc_ie.ie_dependladdr,
INPCBLBGROUP_SIZMIN);		INPCBLBGROUP_SIZMIN, numa_domain);
if (grp == NULL)		if (grp == NULL)
return (ENOBUFS);		return (ENOBUFS);
} else if (grp->il_inpcnt == grp->il_inpsiz) {		} else if (grp->il_inpcnt == grp->il_inpsiz) {
if (grp->il_inpsiz >= INPCBLBGROUP_SIZMAX) {		if (grp->il_inpsiz >= INPCBLBGROUP_SIZMAX) {
if (ratecheck(&lastprint, &interval))		if (ratecheck(&lastprint, &interval))
printf("lb group port %d, limit reached\n",		printf("lb group port %d, limit reached\n",
ntohs(grp->il_lport));		ntohs(grp->il_lport));
return (0);		return (0);
▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines	for (i = 0; i < grp->il_inpcnt; ++i) {
/* Pull up inpcbs, shrink group if possible. */		/* Pull up inpcbs, shrink group if possible. */
in_pcblbgroup_reorder(hdr, &grp, i);		in_pcblbgroup_reorder(hdr, &grp, i);
}		}
return;		return;
}		}
}		}
}		}

		int
		in_pcblbgroup_numa(struct inpcb *inp, int arg)
		{
		struct inpcbinfo *pcbinfo;
		struct inpcblbgrouphead *hdr;
		struct inpcblbgroup *grp;
		int err, i;
		uint8_t numa_domain;

		switch (arg) {
		case TCP_REUSPORT_LB_NUMA_NODOM:
		numa_domain = M_NODOM;
		break;
		case TCP_REUSPORT_LB_NUMA_CURDOM:
		numa_domain = PCPU_GET(domain);
		break;
		default:
		if (arg < 0 \|\| arg >= vm_ndomains)
		return (EINVAL);
		numa_domain = arg;
		}

		err = 0;
		pcbinfo = inp->inp_pcbinfo;
		INP_WLOCK_ASSERT(inp);
		INP_HASH_WLOCK(pcbinfo);
		hdr = &pcbinfo->ipi_lbgrouphashbase[
		INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask)];
		CK_LIST_FOREACH(grp, hdr, il_list) {
		for (i = 0; i < grp->il_inpcnt; ++i) {
		if (grp->il_inp[i] != inp)
		continue;

		if (grp->il_numa_domain == numa_domain) {
		goto abort_with_hash_wlock;
		}

		/* Remove it from the old group. */
		in_pcbremlbgrouphash(inp);

		/* Add it to the new group based on numa domain. */
		in_pcbinslbgrouphash(inp, numa_domain);
		goto abort_with_hash_wlock;
		}
		}
		err = ENOENT;
		abort_with_hash_wlock:
		INP_HASH_WUNLOCK(pcbinfo);
		return (err);
		}

/*		/*
* Different protocols initialize their inpcbs differently - giving		* Different protocols initialize their inpcbs differently - giving
* different name to the lock. But they all are disposed the same.		* different name to the lock. But they all are disposed the same.
*/		*/
static void		static void
inpcb_fini(void *mem, int size)		inpcb_fini(void *mem, int size)
{		{
struct inpcb *inp = mem;		struct inpcb *inp = mem;
▲ Show 20 Lines • Show All 276 Lines • ▼ Show 20 Lines	if (lastport < first \|\| lastport > last)
*lastport = first;		*lastport = first;
lport = htons(*lastport);		lport = htons(*lastport);

if (fsa != NULL) {		if (fsa != NULL) {
#ifdef INET		#ifdef INET
if (lsa->sa_family == AF_INET) {		if (lsa->sa_family == AF_INET) {
tmpinp = in_pcblookup_hash_locked(pcbinfo,		tmpinp = in_pcblookup_hash_locked(pcbinfo,
faddr, fport, laddr, lport, lookupflags,		faddr, fport, laddr, lport, lookupflags,
NULL);		NULL, M_NODOM);
}		}
#endif		#endif
#ifdef INET6		#ifdef INET6
if (lsa->sa_family == AF_INET6) {		if (lsa->sa_family == AF_INET6) {
tmpinp = in6_pcblookup_hash_locked(pcbinfo,		tmpinp = in6_pcblookup_hash_locked(pcbinfo,
faddr6, fport, laddr6, lport, lookupflags,		faddr6, fport, laddr6, lport, lookupflags,
NULL);		NULL, M_NODOM);
}		}
#endif		#endif
} else {		} else {
#ifdef INET6		#ifdef INET6
if ((inp->inp_vflag & INP_IPV6) != 0)		if ((inp->inp_vflag & INP_IPV6) != 0)
tmpinp = in6_pcblookup_local(pcbinfo,		tmpinp = in6_pcblookup_local(pcbinfo,
&inp->in6p_laddr, lport, lookupflags, cred);		&inp->in6p_laddr, lport, lookupflags, cred);
#endif		#endif
▲ Show 20 Lines • Show All 644 Lines • ▼ Show 20 Lines	if (IN_MULTICAST(ntohl(faddr.s_addr)) &&
error = 0;		error = 0;
}		}
IN_IFADDR_RUNLOCK(&in_ifa_tracker);		IN_IFADDR_RUNLOCK(&in_ifa_tracker);
}		}
}		}
if (error)		if (error)
return (error);		return (error);
}		}

if (lport != 0) {		if (lport != 0) {
oinp = in_pcblookup_hash_locked(inp->inp_pcbinfo, faddr,		oinp = in_pcblookup_hash_locked(inp->inp_pcbinfo, faddr,
fport, laddr, lport, 0, NULL);		fport, laddr, lport, 0, NULL, M_NODOM);
if (oinp != NULL) {		if (oinp != NULL) {
if (oinpp != NULL)		if (oinpp != NULL)
*oinpp = oinp;		*oinpp = oinp;
return (EADDRINUSE);		return (EADDRINUSE);
}		}
} else {		} else {
struct sockaddr_in lsin, fsin;		struct sockaddr_in lsin, fsin;

▲ Show 20 Lines • Show All 601 Lines • ▼ Show 20 Lines	#endif
return (match);		return (match);
}		}
}		}
#undef INP_LOOKUP_MAPPED_PCB_COST		#undef INP_LOOKUP_MAPPED_PCB_COST

static struct inpcb *		static struct inpcb *
in_pcblookup_lbgroup(const struct inpcbinfo *pcbinfo,		in_pcblookup_lbgroup(const struct inpcbinfo *pcbinfo,
const struct in_addr laddr, uint16_t lport, const struct in_addr faddr,		const struct in_addr laddr, uint16_t lport, const struct in_addr faddr,
uint16_t fport, int lookupflags)		uint16_t fport, int lookupflags, int numa_domain)
{		{
struct inpcb *local_wild;		struct inpcb local_wild, numa_wild;
const struct inpcblbgrouphead *hdr;		const struct inpcblbgrouphead *hdr;
struct inpcblbgroup *grp;		struct inpcblbgroup *grp;
uint32_t idx;		uint32_t idx;

INP_HASH_LOCK_ASSERT(pcbinfo);		INP_HASH_LOCK_ASSERT(pcbinfo);

hdr = &pcbinfo->ipi_lbgrouphashbase[		hdr = &pcbinfo->ipi_lbgrouphashbase[
INP_PCBPORTHASH(lport, pcbinfo->ipi_lbgrouphashmask)];		INP_PCBPORTHASH(lport, pcbinfo->ipi_lbgrouphashmask)];

/*		/*
* Order of socket selection:		* Order of socket selection:
* 1. non-wild.		* 1. non-wild.
* 2. wild (if lookupflags contains INPLOOKUP_WILDCARD).		* 2. wild (if lookupflags contains INPLOOKUP_WILDCARD).
*		*
* NOTE:		* NOTE:
* - Load balanced group does not contain jailed sockets		* - Load balanced group does not contain jailed sockets
* - Load balanced group does not contain IPv4 mapped INET6 wild sockets		* - Load balanced group does not contain IPv4 mapped INET6 wild sockets
*/		*/
local_wild = NULL;		local_wild = NULL;
		numa_wild = NULL;
CK_LIST_FOREACH(grp, hdr, il_list) {		CK_LIST_FOREACH(grp, hdr, il_list) {
#ifdef INET6		#ifdef INET6
if (!(grp->il_vflag & INP_IPV4))		if (!(grp->il_vflag & INP_IPV4))
continue;		continue;
#endif		#endif
if (grp->il_lport != lport)		if (grp->il_lport != lport)
continue;		continue;

idx = INP_PCBLBGROUP_PKTHASH(faddr->s_addr, lport, fport) %		idx = INP_PCBLBGROUP_PKTHASH(faddr->s_addr, lport, fport) %
grp->il_inpcnt;		grp->il_inpcnt;
if (grp->il_laddr.s_addr == laddr->s_addr)		if (grp->il_laddr.s_addr == laddr->s_addr) {
		if (numa_domain == M_NODOM \|\|
		grp->il_numa_domain == numa_domain) {
return (grp->il_inp[idx]);		return (grp->il_inp[idx]);
		} else {
		numa_wild = grp->il_inp[idx];
		}
		}
if (grp->il_laddr.s_addr == INADDR_ANY &&		if (grp->il_laddr.s_addr == INADDR_ANY &&
(lookupflags & INPLOOKUP_WILDCARD) != 0)		(lookupflags & INPLOOKUP_WILDCARD) != 0 &&
		(local_wild == NULL \|\| numa_domain == M_NODOM \|\|
		grp->il_numa_domain == numa_domain)) {
local_wild = grp->il_inp[idx];		local_wild = grp->il_inp[idx];
}		}
		}
		if (numa_wild != NULL)
		return (numa_wild);

return (local_wild);		return (local_wild);
}		}

#ifdef PCBGROUP		#ifdef PCBGROUP
/*		/*
* Lookup PCB in hash list, using pcbgroup tables.		* Lookup PCB in hash list, using pcbgroup tables.
*/		*/
static struct inpcb *		static struct inpcb *
▲ Show 20 Lines • Show All 230 Lines • ▼ Show 20 Lines
/*		/*
* Lookup PCB in hash list, using pcbinfo tables. This variation assumes		* Lookup PCB in hash list, using pcbinfo tables. This variation assumes
* that the caller has locked the hash list, and will not perform any further		* that the caller has locked the hash list, and will not perform any further
* locking or reference operations on either the hash list or the connection.		* locking or reference operations on either the hash list or the connection.
*/		*/
static struct inpcb *		static struct inpcb *
in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, struct in_addr faddr,		in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, struct in_addr faddr,
u_int fport_arg, struct in_addr laddr, u_int lport_arg, int lookupflags,		u_int fport_arg, struct in_addr laddr, u_int lport_arg, int lookupflags,
struct ifnet *ifp)		struct ifnet *ifp, uint8_t numa_domain)
{		{
struct inpcbhead *head;		struct inpcbhead *head;
struct inpcb inp, tmpinp;		struct inpcb inp, tmpinp;
u_short fport = fport_arg, lport = lport_arg;		u_short fport = fport_arg, lport = lport_arg;

KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD)) == 0,		KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD)) == 0,
("%s: invalid lookup flags %d", __func__, lookupflags));		("%s: invalid lookup flags %d", __func__, lookupflags));
INP_HASH_LOCK_ASSERT(pcbinfo);		INP_HASH_LOCK_ASSERT(pcbinfo);
Show All 28 Lines	#endif
if (tmpinp != NULL)		if (tmpinp != NULL)
return (tmpinp);		return (tmpinp);

/*		/*
* Then look in lb group (for wildcard match).		* Then look in lb group (for wildcard match).
*/		*/
if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {		if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
inp = in_pcblookup_lbgroup(pcbinfo, &laddr, lport, &faddr,		inp = in_pcblookup_lbgroup(pcbinfo, &laddr, lport, &faddr,
fport, lookupflags);		fport, lookupflags, numa_domain);
if (inp != NULL)		if (inp != NULL)
return (inp);		return (inp);
}		}

/*		/*
* Then look for a wildcard match, if requested.		* Then look for a wildcard match, if requested.
*/		*/
if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {		if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
▲ Show 20 Lines • Show All 70 Lines • ▼ Show 20 Lines
/*		/*
* Lookup PCB in hash list, using pcbinfo tables. This variation locks the		* Lookup PCB in hash list, using pcbinfo tables. This variation locks the
* hash list lock, and will return the inpcb locked (i.e., requires		* hash list lock, and will return the inpcb locked (i.e., requires
* INPLOOKUP_LOCKPCB).		* INPLOOKUP_LOCKPCB).
*/		*/
static struct inpcb *		static struct inpcb *
in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr,		in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr,
u_int fport, struct in_addr laddr, u_int lport, int lookupflags,		u_int fport, struct in_addr laddr, u_int lport, int lookupflags,
struct ifnet *ifp)		struct ifnet *ifp, uint8_t numa_domain)
{		{
struct inpcb *inp;		struct inpcb *inp;

inp = in_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport,		inp = in_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport,
(lookupflags & ~(INPLOOKUP_RLOCKPCB \| INPLOOKUP_WLOCKPCB)), ifp);		(lookupflags & ~(INPLOOKUP_RLOCKPCB \| INPLOOKUP_WLOCKPCB)), ifp,
		numa_domain);
if (inp != NULL) {		if (inp != NULL) {
if (lookupflags & INPLOOKUP_WLOCKPCB) {		if (lookupflags & INPLOOKUP_WLOCKPCB) {
INP_WLOCK(inp);		INP_WLOCK(inp);
if (__predict_false(inp->inp_flags2 & INP_FREED)) {		if (__predict_false(inp->inp_flags2 & INP_FREED)) {
INP_WUNLOCK(inp);		INP_WUNLOCK(inp);
inp = NULL;		inp = NULL;
}		}
} else if (lookupflags & INPLOOKUP_RLOCKPCB) {		} else if (lookupflags & INPLOOKUP_RLOCKPCB) {
▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Lines	#if defined(PCBGROUP) && !defined(RSS)
if (in_pcbgroup_enabled(pcbinfo)) {		if (in_pcbgroup_enabled(pcbinfo)) {
pcbgroup = in_pcbgroup_bytuple(pcbinfo, laddr, lport, faddr,		pcbgroup = in_pcbgroup_bytuple(pcbinfo, laddr, lport, faddr,
fport);		fport);
return (in_pcblookup_group(pcbinfo, pcbgroup, faddr, fport,		return (in_pcblookup_group(pcbinfo, pcbgroup, faddr, fport,
laddr, lport, lookupflags, ifp));		laddr, lport, lookupflags, ifp));
}		}
#endif		#endif
return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,		return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,
lookupflags, ifp));		lookupflags, ifp, M_NODOM));
}		}

struct inpcb *		struct inpcb *
in_pcblookup_mbuf(struct inpcbinfo *pcbinfo, struct in_addr faddr,		in_pcblookup_mbuf(struct inpcbinfo *pcbinfo, struct in_addr faddr,
u_int fport, struct in_addr laddr, u_int lport, int lookupflags,		u_int fport, struct in_addr laddr, u_int lport, int lookupflags,
struct ifnet ifp, struct mbuf m)		struct ifnet ifp, struct mbuf m)
{		{
#ifdef PCBGROUP		#ifdef PCBGROUP
Show All 25 Lines	#ifndef RSS
pcbgroup = in_pcbgroup_bytuple(pcbinfo, laddr, lport, faddr,		pcbgroup = in_pcbgroup_bytuple(pcbinfo, laddr, lport, faddr,
fport);		fport);
return (in_pcblookup_group(pcbinfo, pcbgroup, faddr, fport,		return (in_pcblookup_group(pcbinfo, pcbgroup, faddr, fport,
laddr, lport, lookupflags, ifp));		laddr, lport, lookupflags, ifp));
#endif		#endif
}		}
#endif		#endif
return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,		return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,
lookupflags, ifp));		lookupflags, ifp, m->m_pkthdr.numa_domain));
}		}
#endif /* INET */		#endif /* INET */

/*		/*
* Insert PCB onto various hash lists.		* Insert PCB onto various hash lists.
*/		*/
static int		static int
in_pcbinshash_internal(struct inpcb inp, struct mbuf m)		in_pcbinshash_internal(struct inpcb inp, struct mbuf m)
Show All 25 Lines	pcbporthash = &pcbinfo->ipi_porthashbase[
INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_porthashmask)];		INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_porthashmask)];

/*		/*
* Add entry to load balance group.		* Add entry to load balance group.
* Only do this if SO_REUSEPORT_LB is set.		* Only do this if SO_REUSEPORT_LB is set.
*/		*/
so_options = inp_so_options(inp);		so_options = inp_so_options(inp);
if (so_options & SO_REUSEPORT_LB) {		if (so_options & SO_REUSEPORT_LB) {
int ret = in_pcbinslbgrouphash(inp);		int ret = in_pcbinslbgrouphash(inp, M_NODOM);
if (ret) {		if (ret) {
/* pcb lb group malloc fail (ret=ENOBUFS). */		/* pcb lb group malloc fail (ret=ENOBUFS). */
return (ret);		return (ret);
}		}
}		}

/*		/*
* Go through port list and look for a head for this lport.		* Go through port list and look for a head for this lport.
▲ Show 20 Lines • Show All 919 Lines • Show Last 20 Lines