D58131.diff
No OneTemporary
Actions

Size

74 KB

Referenced Files

None

Subscribers

None

D58131.diff
View Options

	diff --git a/sys/netinet/in_pcb.h b/sys/netinet/in_pcb.h
	--- a/sys/netinet/in_pcb.h
	+++ b/sys/netinet/in_pcb.h
	@@ -192,7 +192,7 @@
	#define IN6P_RTHDRDSTOPTS 0x00200000 /* receive dstoptions before rthdr */
	#define IN6P_TCLASS 0x00400000 /* receive traffic class value */
	#define IN6P_AUTOFLOWLABEL 0x00800000 /* attach flowlabel automatically */
	-/* INP_INLBGROUP 0x01000000 private to in_pcb.c */
	+/* INP_INLBGROUP 0x01000000 private to in_pcb.c/in6_pcb.c */
	#define INP_ONESBCAST 0x02000000 /* send all-ones broadcast */
	/* INP_UNCONNECTED 0x04000000 private to in_pcb.c/in6_pcb.c */
	#define INP_SOCKREF 0x08000000 /* strong socket reference */
	@@ -302,17 +302,6 @@
	#include <vm/uma.h>
	#include <sys/ck.h>

	-/*
	- * struct inpcb is the common protocol control block structure used in most
	- * IP transport protocols.
	- *
	- * Pointers to local and foreign host table entries, local and foreign socket
	- * numbers, and pointers up (to a socket structure) and down (to a
	- * protocol-specific control block) are stored here.
	- */
	-CK_LIST_HEAD(inpcbhead, inpcb);
	-CK_LIST_HEAD(inpcblbgrouphead, inpcblbgroup);
	-
	/*
	* struct inpcb captures the network layer state for TCP, UDP, and raw IPv4 and
	* IPv6 sockets. In the case of TCP and UDP, further per-connection state is
	@@ -428,11 +417,9 @@
	*
	* (c) Constant or nearly constant after initialisation
	* (e) Protected by SMR section
	- * (h) Locked by ipi_hash_lock
	+ * (h) Locked by ipi_list_unconn.lock
	*/
	struct inpcbinfo {
	- u_int ipi_count; /* (h) */
	-
	/*
	* Generation count -- incremented each time a connection is allocated
	* or freed.
	@@ -442,9 +429,11 @@
	/*
	* Fields associated with port lookup and allocation.
	*/
	- u_short ipi_lastport; /* (h) */
	- u_short ipi_lastlow; /* (h) */
	- u_short ipi_lasthi; /* (h) */
	+ u_int ipi_lastport; /* (h) */
	+ u_int ipi_lastlow; /* (h) */
	+ u_int ipi_lasthi; /* (h) */
	+
	+ u_int ipi_count; /* (h) */

	/*
	* UMA zone from which inpcbs are allocated for this protocol.
	@@ -458,10 +447,12 @@
	* port numbers. The "exact" hash holds PCBs connected to a foreign
	* address, and "wild" holds the rest.
	*/
	- struct mtx ipi_hash_lock;
	- struct inpcbhead ipi_list_unconn; /* (r:e/w:h) */
	- struct inpcbhead ipi_hash_exact; / (r:e/w:h) */
	- struct inpcbhead ipi_hash_wild; / (r:e/w:h) */
	+ struct inpbucket {
	+ CK_LIST_HEAD(, inpcb) head;
	+ struct mtx lock;
	+ } ipi_list_unconn; /* (r:e/w:h) */
	+ struct inpbucket ipi_hash_exact; / (r:e/w:h) */
	+ struct inpbucket ipi_hash_wild; / (r:e/w:h) */
	u_long ipi_hashmask; /* (c) */
	u_long ipi_porthashmask; /* (c) */
	u_long ipi_lbgrouphashmask; /* (c) */
	@@ -469,13 +460,16 @@
	/*
	* Global hash of inpcbs, hashed by only local port number.
	*/
	- struct inpcbhead ipi_porthashbase; / (h) */
	+ struct inpbucket ipi_porthash; / (h) */

	/*
	* Load balance groups used for the SO_REUSEPORT_LB option,
	* hashed by local port.
	*/
	- struct inpcblbgrouphead ipi_lbgrouphashbase; / (r:e/w:h) */
	+ struct lbgroupbucket {
	+ CK_LIST_HEAD(, inpcblbgroup) head;
	+ struct mtx lock;
	+ } ipi_lbgrouphashbase; / (r:e/w:h) */
	};

	/*
	@@ -551,13 +545,6 @@
	void inp_4tuple_get(struct inpcb inp, uint32_t laddr, uint16_t *lp,
	uint32_t faddr, uint16_t fp);

	-#define INP_HASH_WLOCK(ipi) mtx_lock(&(ipi)->ipi_hash_lock)
	-#define INP_HASH_WUNLOCK(ipi) mtx_unlock(&(ipi)->ipi_hash_lock)
	-#define INP_HASH_LOCK_ASSERT(ipi) MPASS(SMR_ENTERED((ipi)->ipi_smr) \|\| \
	- mtx_owned(&(ipi)->ipi_hash_lock))
	-#define INP_HASH_WLOCK_ASSERT(ipi) mtx_assert(&(ipi)->ipi_hash_lock, \
	- MA_OWNED)
	-
	VNET_DECLARE(uint32_t, in_pcbhashseed);
	#define V_in_pcbhashseed VNET(in_pcbhashseed)

	diff --git a/sys/netinet/in_pcb.c b/sys/netinet/in_pcb.c
	--- a/sys/netinet/in_pcb.c
	+++ b/sys/netinet/in_pcb.c
	@@ -65,6 +65,9 @@
	#include <sys/jail.h>
	#include <sys/kernel.h>
	#include <sys/sysctl.h>
	+#ifdef INVARIANTS
	+#include <machine/stack.h>
	+#endif

	#ifdef DDB
	#include <ddb/ddb.h>
	@@ -109,7 +112,6 @@
	#define INPCBLBGROUP_SIZMAX 256

	#define INP_FREED 0x00000200 /* Went through in_pcbfree(). */
	-#define INP_INLBGROUP 0x01000000 /* Inserted into inpcblbgroup. */

	/*
	* These configure the range of local port addresses assigned to
	@@ -134,9 +136,14 @@
	VNET_DEFINE(bool, ipport_randomized) = true;

	#ifdef INET
	-static struct inpcb in_pcblookup_internal(struct inpcbinfo pcbinfo,
	+static struct inpcb in_pcblookup_internal(struct inpcbinfo_ctx ipictx,
	struct in_addr faddr, u_int fport_arg, struct in_addr laddr,
	u_int lport_arg, int lookupflags, uint8_t numa_domain, int fib);
	+static struct inpcb in_pcblookup_exact(struct inpcbinfo pcbinfo,
	+ struct in_addr faddr, u_short fport, struct in_addr laddr, u_short lport,
	+ struct inpbucket **bucketp);
	+static struct inpcb in_pcblookup_wild_locked(struct inpcbinfo pcbinfo,
	+ struct in_addr laddr, u_short lport, int fib, struct inpbucket **bucketp);

	#define RANGECHK(var, min, max) \
	if ((var) < (min)) { (var) = (min); } \
	@@ -289,39 +296,40 @@
	NET_EPOCH_CALL(in_pcblbgroup_free_deferred, &grp->il_epoch_ctx);
	}

	-static struct inpcblbgroup *
	-in_pcblbgroup_find(struct inpcb *inp)
	+/*
	+ * Returns with locked bucket on success and not locked in case of failure.
	+ */
	+struct inpcblbgroup *
	+in_pcblbgroup_find(struct inpcb inp, struct lbgroupbucket *bucket)
	{
	- struct inpcbinfo *pcbinfo;
	+ const struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
	struct inpcblbgroup *grp;
	- struct inpcblbgrouphead *hdr;

	- INP_LOCK_ASSERT(inp);
	-
	- pcbinfo = inp->inp_pcbinfo;
	- INP_HASH_LOCK_ASSERT(pcbinfo);
	+ INP_WLOCK_ASSERT(inp);

	- hdr = &pcbinfo->ipi_lbgrouphashbase[
	+ *bucket = &pcbinfo->ipi_lbgrouphashbase[
	INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask)];
	- CK_LIST_FOREACH(grp, hdr, il_list) {
	+ INPBUCKET_LOCK(*bucket);
	+ CK_LIST_FOREACH(grp, &(*bucket)->head, il_list) {
	struct inpcb *inp1;

	for (unsigned int i = 0; i < grp->il_inpcnt; i++) {
	if (inp == grp->il_inp[i])
	- goto found;
	+ return (grp);
	}
	LIST_FOREACH(inp1, &grp->il_pending, inp_lbgroup_list) {
	if (inp == inp1)
	- goto found;
	+ return (grp);
	}
	}
	-found:
	- return (grp);
	+ INPBUCKET_UNLOCK(*bucket);
	+ return (NULL);
	}

	static void
	in_pcblbgroup_insert(struct inpcblbgroup grp, struct inpcb inp)
	{
	+ /* NB: bucket is locked */
	KASSERT(grp->il_inpcnt < grp->il_inpsiz,
	("invalid local group size %d and count %d", grp->il_inpsiz,
	grp->il_inpcnt));
	@@ -349,12 +357,14 @@
	}

	static struct inpcblbgroup *
	-in_pcblbgroup_resize(struct inpcblbgrouphead *hdr,
	+in_pcblbgroup_resize(struct lbgroupbucket *bucket,
	struct inpcblbgroup *old_grp, int size)
	{
	struct inpcblbgroup *grp;
	int i;

	+ INPBUCKET_LOCK_ASSERT(bucket);
	+
	grp = in_pcblbgroup_alloc(old_grp->il_cred, old_grp->il_vflag,
	old_grp->il_lport, &old_grp->il_dependladdr, size,
	old_grp->il_numa_domain, old_grp->il_fibnum);
	@@ -368,7 +378,7 @@
	for (i = 0; i < old_grp->il_inpcnt; ++i)
	grp->il_inp[i] = old_grp->il_inp[i];
	grp->il_inpcnt = old_grp->il_inpcnt;
	- CK_LIST_INSERT_HEAD(hdr, grp, il_list);
	+ CK_LIST_INSERT_HEAD(&bucket->head, grp, il_list);
	LIST_SWAP(&old_grp->il_pending, &grp->il_pending, inpcb,
	inp_lbgroup_list);
	grp->il_pendcnt = old_grp->il_pendcnt;
	@@ -385,16 +395,13 @@
	{
	const static struct timeval interval = { 60, 0 };
	static struct timeval lastprint;
	- struct inpcbinfo *pcbinfo;
	- struct inpcblbgrouphead *hdr;
	+ const struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
	+ struct lbgroupbucket *bucket;
	struct inpcblbgroup *grp;
	uint32_t idx;
	int fib;

	- pcbinfo = inp->inp_pcbinfo;
	-
	INP_WLOCK_ASSERT(inp);
	- INP_HASH_WLOCK_ASSERT(pcbinfo);

	fib = (inp->inp_flags & INP_BOUNDFIB) != 0 ?
	inp->inp_inc.inc_fibnum : RT_ALL_FIBS;
	@@ -411,8 +418,9 @@
	#endif

	idx = INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask);
	- hdr = &pcbinfo->ipi_lbgrouphashbase[idx];
	- CK_LIST_FOREACH(grp, hdr, il_list) {
	+ bucket = &pcbinfo->ipi_lbgrouphashbase[idx];
	+ INPBUCKET_LOCK(bucket);
	+ CK_LIST_FOREACH(grp, &bucket->head, il_list) {
	if (grp->il_cred->cr_prison == inp->inp_cred->cr_prison &&
	grp->il_vflag == inp->inp_vflag &&
	grp->il_lport == inp->inp_lport &&
	@@ -429,12 +437,15 @@
	grp = in_pcblbgroup_alloc(inp->inp_cred, inp->inp_vflag,
	inp->inp_lport, &inp->inp_inc.inc_ie.ie_dependladdr,
	INPCBLBGROUP_SIZMIN, numa_domain, fib);
	- if (grp == NULL)
	+ if (grp == NULL) {
	+ INPBUCKET_UNLOCK(bucket);
	return (ENOMEM);
	+ }
	in_pcblbgroup_insert(grp, inp);
	- CK_LIST_INSERT_HEAD(hdr, grp, il_list);
	+ CK_LIST_INSERT_HEAD(&bucket->head, grp, il_list);
	} else if (grp->il_inpcnt + grp->il_pendcnt == grp->il_inpsiz) {
	if (grp->il_inpsiz >= INPCBLBGROUP_SIZMAX) {
	+ INPBUCKET_UNLOCK(bucket);
	if (ratecheck(&lastprint, &interval))
	printf("lb group port %d, limit reached\n",
	ntohs(grp->il_lport));
	@@ -442,37 +453,37 @@
	}

	/* Expand this local group. */
	- grp = in_pcblbgroup_resize(hdr, grp, grp->il_inpsiz * 2);
	- if (grp == NULL)
	+ grp = in_pcblbgroup_resize(bucket, grp, grp->il_inpsiz * 2);
	+ if (grp == NULL) {
	+ INPBUCKET_UNLOCK(bucket);
	return (ENOMEM);
	+ }
	in_pcblbgroup_insert(grp, inp);
	} else {
	in_pcblbgroup_insert(grp, inp);
	}
	+ INPBUCKET_UNLOCK(bucket);
	return (0);
	}

	/*
	- * Remove PCB from load balance group.
	+ * Remove PCB from load balance group. Expects locked bucket.
	*/
	static void
	-in_pcbremlbgrouphash(struct inpcb *inp)
	+in_pcbremlbgrouphash(struct lbgroupbucket bucket, struct inpcb inp)
	{
	- struct inpcbinfo *pcbinfo;
	- struct inpcblbgrouphead *hdr;
	struct inpcblbgroup *grp;
	struct inpcb *inp1;
	int i;

	- pcbinfo = inp->inp_pcbinfo;
	-
	INP_WLOCK_ASSERT(inp);
	MPASS(inp->inp_flags & INP_INLBGROUP);
	- INP_HASH_WLOCK_ASSERT(pcbinfo);
	+ MPASS(bucket == &inp->inp_pcbinfo->ipi_lbgrouphashbase[
	+ INP_PCBPORTHASH(inp->inp_lport,
	+ inp->inp_pcbinfo->ipi_lbgrouphashmask)]);
	+ INPBUCKET_LOCK_ASSERT(bucket);

	- hdr = &pcbinfo->ipi_lbgrouphashbase[
	- INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask)];
	- CK_LIST_FOREACH(grp, hdr, il_list) {
	+ CK_LIST_FOREACH(grp, &bucket->head, il_list) {
	for (i = 0; i < grp->il_inpcnt; ++i) {
	if (grp->il_inp[i] != inp)
	continue;
	@@ -509,10 +520,11 @@
	int
	in_pcblbgroup_numa(struct inpcb *inp, int arg)
	{
	- struct inpcbinfo *pcbinfo;
	- int error;
	+ struct lbgroupbucket *bucket;
	uint8_t numa_domain;

	+ INP_WLOCK_ASSERT(inp);
	+
	switch (arg) {
	case TCP_REUSPORT_LB_NUMA_NODOM:
	numa_domain = M_NODOM;
	@@ -526,20 +538,15 @@
	numa_domain = arg;
	}

	- pcbinfo = inp->inp_pcbinfo;
	- INP_WLOCK_ASSERT(inp);
	- INP_HASH_WLOCK(pcbinfo);
	- if (in_pcblbgroup_find(inp) != NULL) {
	- /* Remove it from the old group. */
	- in_pcbremlbgrouphash(inp);
	- /* Add it to the new group based on numa domain. */
	- in_pcbinslbgrouphash(inp, numa_domain);
	- error = 0;
	- } else {
	- error = ENOENT;
	- }
	- INP_HASH_WUNLOCK(pcbinfo);
	- return (error);
	+ if (in_pcblbgroup_find(inp, &bucket) == NULL)
	+ return (ENOENT);
	+ /* Remove it from the old group. */
	+ in_pcbremlbgrouphash(bucket, inp);
	+ INPBUCKET_UNLOCK(bucket);
	+ /* Add it to the new group based on numa domain. */
	+ in_pcbinslbgrouphash(inp, numa_domain);
	+
	+ return (0);
	}

	/*
	@@ -553,26 +560,32 @@
	.mtype = M_PCB,
	.mflags = M_WAITOK,
	.head = HASH_HEAD_CK_LIST,
	+ .lock = HASH_LOCK_MTX,
	+ .lopts = MTX_DEF,
	};

	- mtx_init(&pcbinfo->ipi_hash_lock, pcbstor->ips_hashlock_name,
	+ mtx_init(&pcbinfo->ipi_list_unconn.lock, pcbstor->ips_hashlock_name,
	NULL, MTX_DEF);
	- CK_LIST_INIT(&pcbinfo->ipi_list_unconn);
	+ CK_LIST_INIT(&pcbinfo->ipi_list_unconn.head);
	pcbinfo->ipi_count = 0;

	ha.size = hash_nelements;
	+ ha.lname = "inpcb_exact_bucket",
	pcbinfo->ipi_hash_exact = hashalloc(&ha);
	+ ha.lname = "inpcb_wild_bucket",
	pcbinfo->ipi_hash_wild = hashalloc(&ha);
	pcbinfo->ipi_hashmask = ha.size - 1;

	if (porthash_nelements > 0) {
	ha.size = imin(porthash_nelements, IPPORT_MAX + 1);
	- pcbinfo->ipi_porthashbase = hashalloc(&ha);
	+ ha.lname = "inpcb_port_bucket",
	+ pcbinfo->ipi_porthash = hashalloc(&ha);
	pcbinfo->ipi_porthashmask = ha.size - 1;
	} else
	- pcbinfo->ipi_porthashbase = NULL;
	+ pcbinfo->ipi_porthash = NULL;
	if (lbgrouphash_nelements > 0) {
	ha.size = imin(lbgrouphash_nelements, IPPORT_MAX + 1);
	+ ha.lname = "inpcb_lbgroup_bucket",
	pcbinfo->ipi_lbgrouphashbase = hashalloc(&ha);
	pcbinfo->ipi_lbgrouphashmask = ha.size - 1;
	} else
	@@ -591,6 +604,7 @@
	struct hashalloc_args ha = {
	.mtype = M_PCB,
	.head = HASH_HEAD_CK_LIST,
	+ .lock = HASH_LOCK_MTX,
	};

	KASSERT(pcbinfo->ipi_count == 0,
	@@ -599,15 +613,15 @@
	ha.size = pcbinfo->ipi_hashmask + 1;
	hashfree(pcbinfo->ipi_hash_exact, &ha);
	hashfree(pcbinfo->ipi_hash_wild, &ha);
	- if (pcbinfo->ipi_porthashbase != NULL) {
	+ if (pcbinfo->ipi_porthash != NULL) {
	ha.size = pcbinfo->ipi_porthashmask + 1;
	- hashfree(pcbinfo->ipi_porthashbase, &ha);
	+ hashfree(pcbinfo->ipi_porthash, &ha);
	}
	if (pcbinfo->ipi_lbgrouphashbase != NULL) {
	ha.size = pcbinfo->ipi_lbgrouphashmask + 1;
	hashfree(pcbinfo->ipi_lbgrouphashbase, &ha);
	}
	- mtx_destroy(&pcbinfo->ipi_hash_lock);
	+ mtx_destroy(&pcbinfo->ipi_list_unconn.lock);
	}

	/*
	@@ -703,11 +717,12 @@
	refcount_init(&inp->inp_refcount, 1); /* Reference from socket. */
	inp->inp_flags \|= INP_UNCONNECTED;
	INP_WLOCK(inp);
	- INP_HASH_WLOCK(pcbinfo);
	+ IPI_LOCK(pcbinfo);
	pcbinfo->ipi_count++;
	inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
	- CK_LIST_INSERT_HEAD(&pcbinfo->ipi_list_unconn, inp, inp_unconn_list);
	- INP_HASH_WUNLOCK(pcbinfo);
	+ CK_LIST_INSERT_HEAD(&pcbinfo->ipi_list_unconn.head, inp,
	+ inp_unconn_list);
	+ IPI_UNLOCK(pcbinfo);
	so->so_pcb = inp;

	return (0);
	@@ -729,48 +744,40 @@
	* and a foreign address and port. If fsa is non-NULL, choose a local port
	* that is unused with those, otherwise one that is completely unused.
	* lsa can be NULL for IPv6.
	+ * On success returns with locked context, fills lportp.
	*/
	int
	-in_pcb_lport_dest(const struct inpcb inp, struct sockaddr lsa,
	- u_short lportp, struct sockaddr fsa, u_short fport, struct ucred *cred,
	- int lookupflags)
	+in_pcb_lport_dest(struct inpcbinfo_ctx ipictx, const struct inpcb inp,
	+ const struct sockaddr lsa, const struct sockaddr fsa, u_short fport,
	+ struct ucred cred, int lookupflags, u_short lportp)
	{
	- struct inpcbinfo *pcbinfo;
	- struct inpcb *tmpinp;
	- unsigned short *lastport;
	+ u_int first, last, port, *lastport;
	int count, error;
	- u_short aux, first, last, lport;
	+ u_short lport;
	#ifdef INET
	struct in_addr laddr, faddr;
	#endif
	#ifdef INET6
	- struct in6_addr laddr6, faddr6;
	+ const struct in6_addr laddr6, faddr6;
	#endif

	- pcbinfo = inp->inp_pcbinfo;
	-
	- /*
	- * Because no actual state changes occur here, a global write lock on
	- * the pcbinfo isn't required.
	- */
	INP_LOCK_ASSERT(inp);
	- INP_HASH_LOCK_ASSERT(pcbinfo);

	if (inp->inp_flags & INP_HIGHPORT) {
	first = V_ipport_hifirstauto; /* sysctl */
	last = V_ipport_hilastauto;
	- lastport = &pcbinfo->ipi_lasthi;
	+ lastport = &ipictx->pcbinfo->ipi_lasthi;
	} else if (inp->inp_flags & INP_LOWPORT) {
	error = priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT);
	if (error)
	return (error);
	first = V_ipport_lowfirstauto; /* 1023 */
	last = V_ipport_lowlastauto; /* 600 */
	- lastport = &pcbinfo->ipi_lastlow;
	+ lastport = &ipictx->pcbinfo->ipi_lastlow;
	} else {
	first = V_ipport_firstauto; /* sysctl */
	last = V_ipport_lastauto;
	- lastport = &pcbinfo->ipi_lastport;
	+ lastport = &ipictx->pcbinfo->ipi_lastport;
	}

	/*
	@@ -779,6 +786,8 @@
	* code path implementing all logic.
	*/
	if (first > last) {
	+ __typeof(first) aux;
	+
	aux = first;
	first = last;
	last = aux;
	@@ -788,47 +797,50 @@
	laddr.s_addr = INADDR_ANY; /* used by INET6+INET below too */
	if ((inp->inp_vflag & (INP_IPV4\|INP_IPV6)) == INP_IPV4) {
	if (lsa != NULL)
	- laddr = ((struct sockaddr_in *)lsa)->sin_addr;
	+ laddr = ((const struct sockaddr_in *)lsa)->sin_addr;
	if (fsa != NULL)
	- faddr = ((struct sockaddr_in *)fsa)->sin_addr;
	+ faddr = ((const struct sockaddr_in *)fsa)->sin_addr;
	}
	#endif
	#ifdef INET6
	laddr6 = NULL;
	if ((inp->inp_vflag & INP_IPV6) != 0) {
	if (lsa != NULL)
	- laddr6 = &((struct sockaddr_in6 *)lsa)->sin6_addr;
	+ laddr6 = &((const struct sockaddr_in6 *)lsa)->sin6_addr;
	if (fsa != NULL)
	- faddr6 = &((struct sockaddr_in6 *)fsa)->sin6_addr;
	+ faddr6 = &((const struct sockaddr_in6 *)fsa)->sin6_addr;
	}
	#endif

	- tmpinp = NULL;
	-
	- if (V_ipport_randomized)
	- *lastport = first + (arc4random() % (last - first));
	+ if (V_ipport_randomized) {
	+ port = first + (arc4random() % (last - first));
	+ } else {
	+ port = atomic_fetchadd_int(lastport, 1);
	+ if (__predict_false(port < first \|\| port > last)) {
	+ atomic_store_int(lastport, first);
	+ port = first;
	+ }
	+ }

	- count = last - first;
	+ for (count = last - first; count > 0; count--) {
	+ struct inpcbinfo_ctx tmpctx = {
	+ .pcbinfo = ipictx->pcbinfo
	+ };
	+ struct inpcb *tmpinp = NULL;

	- do {
	- if (count-- < 0) /* completely used? */
	- return (EADDRNOTAVAIL);
	- ++*lastport;
	- if (lastport < first \|\| lastport > last)
	- *lastport = first;
	- lport = htons(*lastport);
	+ lport = htons(port);

	if (fsa != NULL) {
	#ifdef INET
	if (lsa->sa_family == AF_INET) {
	- tmpinp = in_pcblookup_internal(pcbinfo,
	+ tmpinp = in_pcblookup_internal(&tmpctx,
	faddr, fport, laddr, lport, lookupflags,
	M_NODOM, RT_ALL_FIBS);
	}
	#endif
	#ifdef INET6
	if (lsa->sa_family == AF_INET6) {
	- tmpinp = in6_pcblookup_internal(pcbinfo,
	+ tmpinp = in6_pcblookup_internal(&tmpctx,
	faddr6, fport, laddr6, lport, lookupflags,
	M_NODOM, RT_ALL_FIBS);
	}
	@@ -836,13 +848,13 @@
	} else {
	#ifdef INET6
	if ((inp->inp_vflag & INP_IPV6) != 0) {
	- tmpinp = in6_pcblookup_local(pcbinfo,
	+ tmpinp = in6_pcblookup_local(&tmpctx,
	&inp->in6p_laddr, lport, RT_ALL_FIBS,
	lookupflags, cred);
	#ifdef INET
	if (tmpinp == NULL &&
	(inp->inp_vflag & INP_IPV4))
	- tmpinp = in_pcblookup_local(pcbinfo,
	+ tmpinp = in_pcblookup_local(&tmpctx,
	laddr, lport, RT_ALL_FIBS,
	lookupflags, cred);
	#endif
	@@ -852,11 +864,26 @@
	else
	#endif
	#ifdef INET
	- tmpinp = in_pcblookup_local(pcbinfo, laddr,
	+ tmpinp = in_pcblookup_local(&tmpctx, laddr,
	lport, RT_ALL_FIBS, lookupflags, cred);
	#endif
	}
	- } while (tmpinp != NULL);
	+ if (tmpinp == NULL) {
	+ MPASS(ipictx->ebucket == NULL);
	+ MPASS(ipictx->wbucket == NULL);
	+ MPASS(ipictx->pbucket == NULL);
	+ MPASS(ipictx->lbbucket == NULL);
	+ *ipictx = tmpctx;
	+ break;
	+ }
	+ inpcbinfo_ctx_release(&tmpctx);
	+ ++port;
	+ if (port < first \|\| port > last)
	+ port = first;
	+ }
	+
	+ if (count == 0) /* completely used? */
	+ return (EADDRNOTAVAIL);

	*lportp = lport;

	@@ -865,10 +892,12 @@

	/*
	* Select a local port (number) to use.
	+ * On success returns with locked context, fills lportp.
	*/
	int
	-in_pcb_lport(struct inpcb inp, struct in_addr laddrp, u_short *lportp,
	- struct ucred *cred, int lookupflags)
	+in_pcb_lport(struct inpcbinfo_ctx ipictx, struct inpcb inp,
	+ struct in_addr laddrp, u_short lportp, struct ucred *cred,
	+ int lookupflags)
	{
	struct sockaddr_in laddr;

	@@ -877,24 +906,25 @@
	laddr.sin_family = AF_INET;
	laddr.sin_addr = *laddrp;
	}
	- return (in_pcb_lport_dest(inp, laddrp ? (struct sockaddr *) &laddr :
	- NULL, lportp, NULL, 0, cred, lookupflags));
	+ return (in_pcb_lport_dest(ipictx, inp,
	+ laddrp ? (struct sockaddr *) &laddr : NULL,
	+ NULL, 0, cred, lookupflags, lportp));
	}
	#endif /* INET \|\| INET6 */

	#ifdef INET
	/*
	* Determine whether the inpcb can be bound to the specified address/port tuple.
	+ * May return with partially locked context.
	*/
	static int
	-in_pcbbind_avail(struct inpcb *inp, const struct in_addr laddr,
	- const u_short lport, const int fib, int sooptions, int lookupflags,
	- struct ucred *cred)
	+in_pcbbind_avail(struct inpcbinfo_ctx ipictx, struct inpcb inp,
	+ const struct in_addr laddr, const u_short lport, const int fib,
	+ int sooptions, int lookupflags, struct ucred *cred)
	{
	int reuseport, reuseport_lb;

	INP_LOCK_ASSERT(inp);
	- INP_HASH_LOCK_ASSERT(inp->inp_pcbinfo);

	reuseport = (sooptions & SO_REUSEPORT);
	reuseport_lb = (sooptions & SO_REUSEPORT_LB);
	@@ -952,7 +982,7 @@
	* However, we can share a port with a connected socket
	* which has a unique 4-tuple.
	*/
	- t = in_pcblookup_local(inp->inp_pcbinfo, laddr, lport,
	+ t = in_pcblookup_local(ipictx, laddr, lport,
	RT_ALL_FIBS, INPLOOKUP_WILDCARD, cred);
	if (t != NULL &&
	(inp->inp_socket->so_type != SOCK_STREAM \|\|
	@@ -960,7 +990,7 @@
	(inp->inp_cred->cr_uid != t->inp_cred->cr_uid))
	return (EADDRINUSE);
	}
	- t = in_pcblookup_local(inp->inp_pcbinfo, laddr, lport, fib,
	+ t = in_pcblookup_local(ipictx, laddr, lport, fib,
	lookupflags, cred);
	if (t != NULL && ((reuseport \| reuseport_lb) &
	t->inp_socket->so_options) == 0) {
	@@ -986,19 +1016,17 @@
	* On error, the values of laddrp and lportp are not changed.
	*/
	static int
	-in_pcbbind_setup_locked(struct inpcb inp, struct sockaddr_in sin,
	- in_addr_t laddrp, u_short lportp, int flags, struct ucred *cred)
	+in_pcbbind_setup_locked(struct inpcbinfo_ctx ipictx, struct inpcb inp,
	+ struct sockaddr_in sin, in_addr_t laddrp, u_short *lportp, int flags,
	+ struct ucred *cred)
	{
	struct socket *so = inp->inp_socket;
	struct in_addr laddr;
	u_short lport = 0;
	int error, fib, lookupflags, sooptions;

	- /*
	- * No state changes, so read locks are sufficient here.
	- */
	+ /* No state changes, so read lock is sufficient here. */
	INP_LOCK_ASSERT(inp);
	- INP_HASH_LOCK_ASSERT(inp->inp_pcbinfo);

	laddr.s_addr = *laddrp;
	if (sin != NULL && laddr.s_addr != INADDR_ANY)
	@@ -1032,15 +1060,16 @@
	RT_ALL_FIBS;

	/* See if this address/port combo is available. */
	- error = in_pcbbind_avail(inp, laddr, lport, fib, sooptions,
	- lookupflags, cred);
	+ error = in_pcbbind_avail(ipictx, inp, laddr, lport, fib,
	+ sooptions, lookupflags, cred);
	if (error != 0)
	return (error);
	}
	if (*lportp != 0)
	lport = *lportp;
	if (lport == 0) {
	- error = in_pcb_lport(inp, &laddr, &lport, cred, lookupflags);
	+ error = in_pcb_lport(ipictx, inp, &laddr, &lport, cred,
	+ lookupflags);
	if (error != 0)
	return (error);
	}
	@@ -1055,11 +1084,14 @@
	in_pcbbind_setup(struct inpcb inp, struct sockaddr_in sin, in_addr_t *laddrp,
	u_short lportp, int flags, struct ucred cred)
	{
	+ struct inpcbinfo_ctx ipictx = {
	+ .pcbinfo = inp->inp_pcbinfo
	+ };
	int error;

	- INP_HASH_WLOCK(inp->inp_pcbinfo);
	- error = in_pcbbind_setup_locked(inp, sin, laddrp, lportp, flags, cred);
	- INP_HASH_WUNLOCK(inp->inp_pcbinfo);
	+ error = in_pcbbind_setup_locked(&ipictx, inp, sin, laddrp, lportp,
	+ flags, cred);
	+ inpcbinfo_ctx_release(&ipictx);

	return (error);
	}
	@@ -1069,6 +1101,9 @@
	in_pcbbind(struct inpcb inp, struct sockaddr_in sin, int flags,
	struct ucred *cred)
	{
	+ struct inpcbinfo_ctx ipictx = {
	+ .pcbinfo = inp->inp_pcbinfo
	+ };
	int error;
	bool anonport;

	@@ -1082,22 +1117,21 @@
	return (EINVAL);
	anonport = sin == NULL \|\| sin->sin_port == 0;

	- INP_HASH_WLOCK(inp->inp_pcbinfo);
	- error = in_pcbbind_setup_locked(inp, sin, &inp->inp_laddr.s_addr,
	- &inp->inp_lport, flags, cred);
	+ error = in_pcbbind_setup_locked(&ipictx, inp, sin,
	+ &inp->inp_laddr.s_addr, &inp->inp_lport, flags, cred);
	if (error) {
	- INP_HASH_WUNLOCK(inp->inp_pcbinfo);
	+ inpcbinfo_ctx_release(&ipictx);
	return (error);
	}
	- if (__predict_false((error = in_pcbinshash(inp)) != 0)) {
	- INP_HASH_WUNLOCK(inp->inp_pcbinfo);
	+ if (__predict_false((error = in_pcbinshash(inp, &ipictx)) != 0)) {
	+ inpcbinfo_ctx_release(&ipictx);
	MPASS(inp->inp_socket->so_options & SO_REUSEPORT_LB);
	inp->inp_laddr.s_addr = INADDR_ANY;
	inp->inp_lport = 0;
	inp->inp_flags &= ~INP_BOUNDFIB;
	return (error);
	}
	- INP_HASH_WUNLOCK(inp->inp_pcbinfo);
	+ inpcbinfo_ctx_release(&ipictx);
	if (anonport)
	inp->inp_flags \|= INP_ANONPORT;
	return (0);
	@@ -1113,6 +1147,9 @@
	int
	in_pcbconnect(struct inpcb inp, struct sockaddr_in sin, struct ucred *cred)
	{
	+ struct inpcbinfo_ctx ipictx = {
	+ .pcbinfo = inp->inp_pcbinfo
	+ };
	struct in_addr laddr, faddr;
	u_short lport;
	int error;
	@@ -1156,16 +1193,26 @@
	} else
	faddr = sin->sin_addr;

	- INP_HASH_WLOCK(inp->inp_pcbinfo);
	if (in_nullhost(inp->inp_laddr)) {
	error = in_pcbladdr(inp, &faddr, &laddr, cred);
	if (__predict_false(error)) {
	- INP_HASH_WUNLOCK(inp->inp_pcbinfo);
	return (error);
	}
	} else
	laddr = inp->inp_laddr;

	+ /*
	+ * Depending on if the inpcb was bind(2)-ed to a local port before
	+ * connect(2), different actions are taken.
	+ * If the inpcb is newborn and lives just on the ipi_list_unconn, then
	+ * in_pcb_lport_dest() will find the exact hash slot and also port
	+ * slot (if applicable). Locking these two buckets is required for the
	+ * later call to in_pcbinshash().
	+ * If the inpcb is already bind(2)-ed, then it is already inserted in
	+ * the port hash (if applicable) and doesn't need to move within it.
	+ * But it needs to move from the wild hash to the exact hash. Both
	+ * buckets need to be locked for in_pcbrehash().
	+ */
	if (anonport) {
	struct sockaddr_in lsin = {
	.sin_family = AF_INET,
	@@ -1176,20 +1223,35 @@
	.sin_addr = faddr,
	};

	- error = in_pcb_lport_dest(inp, (struct sockaddr *)&lsin,
	- &lport, (struct sockaddr *)&fsin, sin->sin_port, cred,
	- INPLOOKUP_WILDCARD);
	- if (__predict_false(error)) {
	- INP_HASH_WUNLOCK(inp->inp_pcbinfo);
	+ MPASS(inp->inp_flags & INP_UNCONNECTED);
	+
	+ error = in_pcb_lport_dest(&ipictx, inp,
	+ (struct sockaddr )&lsin, (struct sockaddr )&fsin,
	+ sin->sin_port, cred, INPLOOKUP_WILDCARD, &lport);
	+ if (__predict_false(error))
	+ /* in_pcb_lport_dest() shouldn't obtained a lock */
	return (error);
	+ } else {
	+ if (in_pcblookup_exact(ipictx.pcbinfo, faddr, sin->sin_port,
	+ laddr, inp->inp_lport, &ipictx.ebucket) != NULL) {
	+ inpcbinfo_ctx_release(&ipictx);
	+ return (EADDRINUSE);
	+ }
	+ if ((inp->inp_flags & INP_UNCONNECTED) == 0) {
	+ struct inpcb *i __diagused;
	+ struct inpcblbgroup *g __diagused;
	+
	+ i = in_pcblookup_wild_locked(ipictx.pcbinfo,
	+ inp->inp_laddr, inp->inp_lport, RT_ALL_FIBS,
	+ &ipictx.wbucket);
	+ MPASS(i);
	+ if (inp->inp_flags & INP_INLBGROUP) {
	+ g = in_pcblbgroup_find(inp, &ipictx.lbbucket);
	+ MPASS(g);
	+ }
	}
	- } else if (in_pcblookup_internal(inp->inp_pcbinfo, faddr,
	- sin->sin_port, laddr, inp->inp_lport, 0, M_NODOM, RT_ALL_FIBS) !=
	- NULL) {
	- INP_HASH_WUNLOCK(inp->inp_pcbinfo);
	- return (EADDRINUSE);
	- } else
	lport = inp->inp_lport;
	+ }

	MPASS(!in_nullhost(inp->inp_laddr) \|\| inp->inp_lport != 0 \|\|
	(inp->inp_flags & INP_UNCONNECTED));
	@@ -1200,11 +1262,11 @@
	inp->inp_lport = lport;

	if (inp->inp_flags & INP_UNCONNECTED) {
	- error = in_pcbinshash(inp);
	+ error = in_pcbinshash(inp, &ipictx);
	MPASS(error == 0);
	} else
	- in_pcbrehash(inp);
	- INP_HASH_WUNLOCK(inp->inp_pcbinfo);
	+ in_pcbrehash(inp, &ipictx);
	+ inpcbinfo_ctx_release(&ipictx);

	if (V_fib_hash_outbound) {
	uint32_t hash_val, hash_type;
	@@ -1464,11 +1526,11 @@
	if (inp->inp_flags & INP_UNCONNECTED)
	return;

	- INP_HASH_WLOCK(inp->inp_pcbinfo);
	in_pcbremhash(inp);
	- CK_LIST_INSERT_HEAD(&inp->inp_pcbinfo->ipi_list_unconn, inp,
	+ IPI_LOCK(inp->inp_pcbinfo);
	+ CK_LIST_INSERT_HEAD(&inp->inp_pcbinfo->ipi_list_unconn.head, inp,
	inp_unconn_list);
	- INP_HASH_WUNLOCK(inp->inp_pcbinfo);
	+ IPI_UNLOCK(inp->inp_pcbinfo);
	inp->inp_flags \|= INP_UNCONNECTED;

	if ((inp->inp_socket->so_proto->pr_flags & PR_CONNREQUIRED) == 0) {
	@@ -1483,21 +1545,19 @@
	void
	in_pcblisten(struct inpcb *inp)
	{
	+ struct lbgroupbucket *bucket;
	struct inpcblbgroup *grp;

	INP_WLOCK_ASSERT(inp);

	- if ((inp->inp_flags & INP_INLBGROUP) != 0) {
	- struct inpcbinfo *pcbinfo;
	+ if ((inp->inp_flags & INP_INLBGROUP) == 0)
	+ return;

	- pcbinfo = inp->inp_pcbinfo;
	- INP_HASH_WLOCK(pcbinfo);
	- grp = in_pcblbgroup_find(inp);
	- LIST_REMOVE(inp, inp_lbgroup_list);
	- grp->il_pendcnt--;
	- in_pcblbgroup_insert(grp, inp);
	- INP_HASH_WUNLOCK(pcbinfo);
	- }
	+ grp = in_pcblbgroup_find(inp, &bucket);
	+ LIST_REMOVE(inp, inp_lbgroup_list);
	+ grp->il_pendcnt--;
	+ in_pcblbgroup_insert(grp, inp);
	+ INPBUCKET_UNLOCK(bucket);
	}

	/*
	@@ -1626,12 +1686,12 @@
	const int hash = ii->hash;

	if (hash < 0)
	- return (CK_LIST_FIRST(&ipi->ipi_list_unconn));
	+ return (CK_LIST_FIRST(&ipi->ipi_list_unconn.head));
	else if (hash <= ipi->ipi_hashmask)
	- return (CK_LIST_FIRST(&ipi->ipi_hash_wild[hash]));
	+ return (CK_LIST_FIRST(&ipi->ipi_hash_wild[hash].head));
	else
	return (CK_LIST_FIRST(
	- &ipi->ipi_hash_exact[hash - ipi->ipi_hashmask - 1]));
	+ &ipi->ipi_hash_exact[hash - ipi->ipi_hashmask - 1].head));
	}

	static inline struct inpcb *
	@@ -1874,14 +1934,16 @@
	* from the hash without acquiring inpcb lock, they rely on the hash
	* lock, thus in_pcbremhash() should be the first action.
	*/
	- INP_HASH_WLOCK(pcbinfo);
	- if (inp->inp_flags & INP_UNCONNECTED)
	+ if (inp->inp_flags & INP_UNCONNECTED) {
	+ IPI_LOCK(pcbinfo);
	CK_LIST_REMOVE(inp, inp_unconn_list);
	- else
	+ } else {
	in_pcbremhash(inp);
	+ IPI_LOCK(pcbinfo);
	+ }
	inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
	pcbinfo->ipi_count--;
	- INP_HASH_WUNLOCK(pcbinfo);
	+ IPI_UNLOCK(pcbinfo);

	#ifdef RATELIMIT
	if (inp->inp_snd_tag != NULL)
	@@ -2033,12 +2095,14 @@
	}

	/*
	- * Lookup a PCB based on the local address and port. Caller must hold the
	- * hash lock. No inpcb locks or references are acquired.
	+ * Lookup a PCB based on the local address and port.
	+ * No inpcb locks or references are acquired.
	+ *
	+ * Always returns with a locked context.
	*/
	#define INP_LOOKUP_MAPPED_PCB_COST 3
	struct inpcb *
	-in_pcblookup_local(struct inpcbinfo *pcbinfo, struct in_addr laddr,
	+in_pcblookup_local(struct inpcbinfo_ctx *ipictx, struct in_addr laddr,
	u_short lport, int fib, int lookupflags, struct ucred *cred)
	{
	struct inpcb *inp;
	@@ -2054,17 +2118,22 @@
	KASSERT(fib == RT_ALL_FIBS \|\| (fib >= 0 && fib < V_rt_numfibs),
	("%s: invalid fib %d", __func__, fib));

	- INP_HASH_LOCK_ASSERT(pcbinfo);
	+ /*
	+ * Lock the wildcard bucket regardless of lookupflags. It needs to be
	+ * locked before the ports bucket. Those callers that call
	+ * in_pcblookup_local() as a check before in_pcbinshash() require
	+ * wildcard bucket locked.
	+ * This function may be called repeatedly and inpcbinfo_ctx_wildlock()
	+ * takes care against lock recursion.
	+ */
	+ inpcbinfo_ctx_wildlock(ipictx, lport);

	if ((lookupflags & INPLOOKUP_WILDCARD) == 0) {
	- struct inpcbhead *head;
	/*
	* Look for an unconnected (wildcard foreign addr) PCB that
	* matches the local address and port we're looking for.
	*/
	- head = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport,
	- pcbinfo->ipi_hashmask)];
	- CK_LIST_FOREACH(inp, head, inp_hash_wild) {
	+ CK_LIST_FOREACH(inp, &ipictx->wbucket->head, inp_hash_wild) {
	#ifdef INET6
	/* XXX inp locking */
	if ((inp->inp_vflag & INP_IPV4) == 0)
	@@ -2086,16 +2155,13 @@
	*/
	return (NULL);
	} else {
	- struct inpcbhead *porthash;
	struct inpcb *match = NULL;

	/*
	- * Port is in use by one or more PCBs. Look for best
	- * fit.
	+ * Port may be in use by one or more PCBs. Look for best fit.
	*/
	- porthash = &pcbinfo->ipi_porthashbase[INP_PCBPORTHASH(lport,
	- pcbinfo->ipi_porthashmask)];
	- CK_LIST_FOREACH(inp, porthash, inp_portlist) {
	+ inpcbinfo_ctx_portlock(ipictx, lport);
	+ CK_LIST_FOREACH(inp, &ipictx->pbucket->head, inp_portlist) {
	if (inp->inp_lport != lport)
	continue;
	if (!prison_equal_ip4(inp->inp_cred->cr_prison,
	@@ -2155,19 +2221,16 @@
	static struct inpcb *
	in_pcblookup_lbgroup(const struct inpcbinfo *pcbinfo,
	const struct in_addr faddr, uint16_t fport, const struct in_addr laddr,
	- uint16_t lport, int domain, int fib)
	+ uint16_t lport, int domain, int fib, struct lbgroupbucket **bucketp)
	{
	- const struct inpcblbgrouphead *hdr;
	+ struct lbgroupbucket *bucket;
	struct inpcblbgroup *grp;
	struct inpcblbgroup jail_exact, jail_wild, local_exact, local_wild;
	struct inpcb *inp;
	u_int count;

	- INP_HASH_LOCK_ASSERT(pcbinfo);
	NET_EPOCH_ASSERT();
	-
	- hdr = &pcbinfo->ipi_lbgrouphashbase[
	- INP_PCBPORTHASH(lport, pcbinfo->ipi_lbgrouphashmask)];
	+ MPASS(bucketp != NULL \|\| SMR_ENTERED(pcbinfo->ipi_smr));

	/*
	* Search for an LB group match based on the following criteria:
	@@ -2176,7 +2239,14 @@
	* - prefer groups bound to the specified NUMA domain
	*/
	jail_exact = jail_wild = local_exact = local_wild = NULL;
	- CK_LIST_FOREACH(grp, hdr, il_list) {
	+ bucket = &pcbinfo->ipi_lbgrouphashbase[
	+ INP_PCBPORTHASH(lport, pcbinfo->ipi_lbgrouphashmask)];
	+ if (bucketp != NULL) {
	+ MPASS(*bucketp == NULL);
	+ *bucketp = bucket;
	+ INPBUCKET_LOCK(bucket);
	+ }
	+ CK_LIST_FOREACH(grp, &bucket->head, il_list) {
	bool injail;

	#ifdef INET6
	@@ -2253,18 +2323,30 @@
	return (false);
	}

	+/*
	+ * Look up inpcb in the exact hash.
	+ *
	+ * For packet processing called in the SMR section. For conlict checks the
	+ * bucket pointer shall be provided that will be returned locked.
	+ */
	static struct inpcb *
	in_pcblookup_exact(struct inpcbinfo *pcbinfo, struct in_addr faddr,
	- u_short fport, struct in_addr laddr, u_short lport)
	+ u_short fport, struct in_addr laddr, u_short lport,
	+ struct inpbucket **bucketp)
	{
	- struct inpcbhead *head;
	+ struct inpbucket *bucket;
	struct inpcb *inp;

	- INP_HASH_LOCK_ASSERT(pcbinfo);
	+ MPASS(bucketp != NULL \|\| SMR_ENTERED(pcbinfo->ipi_smr));

	- head = &pcbinfo->ipi_hash_exact[INP_PCBHASH(&faddr, lport, fport,
	+ bucket = &pcbinfo->ipi_hash_exact[INP_PCBHASH(&faddr, lport, fport,
	pcbinfo->ipi_hashmask)];
	- CK_LIST_FOREACH(inp, head, inp_hash_exact) {
	+ if (bucketp != NULL) {
	+ MPASS(*bucketp == NULL);
	+ *bucketp = bucket;
	+ INPBUCKET_LOCK(bucket);
	+ }
	+ CK_LIST_FOREACH(inp, &bucket->head, inp_hash_exact) {
	if (in_pcblookup_exact_match(inp, faddr, fport, laddr, lport))
	return (inp);
	}
	@@ -2303,15 +2385,15 @@
	in_pcblookup_wild_smr(struct inpcbinfo *pcbinfo, struct in_addr laddr,
	u_short lport, int fib, const inp_lookup_t lockflags)
	{
	- struct inpcbhead *head;
	+ struct inpbucket *bucket;
	struct inpcb *inp;

	KASSERT(SMR_ENTERED(pcbinfo->ipi_smr),
	("%s: not in SMR read section", __func__));

	- head = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport,
	+ bucket = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport,
	pcbinfo->ipi_hashmask)];
	- CK_LIST_FOREACH(inp, head, inp_hash_wild) {
	+ CK_LIST_FOREACH(inp, &bucket->head, inp_hash_wild) {
	inp_lookup_match_t match;

	match = in_pcblookup_wild_match(inp, laddr, lport, fib);
	@@ -2336,17 +2418,20 @@
	return (NULL);
	}

	+/*
	+ * Always returns with locked bucket.
	+ */
	static struct inpcb *
	in_pcblookup_wild_locked(struct inpcbinfo *pcbinfo, struct in_addr laddr,
	- u_short lport, int fib)
	+ u_short lport, int fib, struct inpbucket **bucketp)
	{
	- struct inpcbhead *head;
	+ struct inpbucket *bucket;
	struct inpcb inp, local_wild, local_exact, jail_wild;
	#ifdef INET6
	struct inpcb *local_wild_mapped;
	#endif

	- INP_HASH_LOCK_ASSERT(pcbinfo);
	+ MPASS(bucketp != NULL);

	/*
	* Order of socket selection - we always prefer jails.
	@@ -2355,13 +2440,16 @@
	* 3. non-jailed, non-wild.
	* 4. non-jailed, wild.
	*/
	- head = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport,
	- pcbinfo->ipi_hashmask)];
	local_wild = local_exact = jail_wild = NULL;
	#ifdef INET6
	local_wild_mapped = NULL;
	#endif
	- CK_LIST_FOREACH(inp, head, inp_hash_wild) {
	+ bucket = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport,
	+ pcbinfo->ipi_hashmask)];
	+ MPASS(*bucketp == NULL);
	+ *bucketp = bucket;
	+ INPBUCKET_LOCK(bucket);
	+ CK_LIST_FOREACH(inp, &bucket->head, inp_hash_wild) {
	inp_lookup_match_t match;
	bool injail;

	@@ -2410,18 +2498,16 @@
	}

	/*
	- * Lookup PCB in hash list, using pcbinfo tables. This variation assumes
	- * that the caller has either locked the hash list, which usually happens
	- * for bind(2) operations, or is in SMR section, which happens when sorting
	- * out incoming packets.
	+ * Returns with fully locked context in case of failure and partially
	+ * locked in case of successful lookup.
	*/
	static struct inpcb *
	-in_pcblookup_internal(struct inpcbinfo *pcbinfo, struct in_addr faddr,
	+in_pcblookup_internal(struct inpcbinfo_ctx *ipictx, struct in_addr faddr,
	u_int fport_arg, struct in_addr laddr, u_int lport_arg, int lookupflags,
	uint8_t numa_domain, int fib)
	{
	- struct inpcb *inp;
	const u_short fport = fport_arg, lport = lport_arg;
	+ struct inpcb *inp;

	KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD \| INPLOOKUP_FIB)) == 0,
	("%s: invalid lookup flags %d", __func__, lookupflags));
	@@ -2429,18 +2515,26 @@
	("%s: invalid foreign address", __func__));
	KASSERT(laddr.s_addr != INADDR_ANY,
	("%s: invalid local address", __func__));
	- INP_HASH_WLOCK_ASSERT(pcbinfo);

	- inp = in_pcblookup_exact(pcbinfo, faddr, fport, laddr, lport);
	+ inp = in_pcblookup_exact(ipictx->pcbinfo, faddr, fport, laddr, lport,
	+ &ipictx->ebucket);
	if (inp != NULL)
	return (inp);

	if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
	- inp = in_pcblookup_lbgroup(pcbinfo, &faddr, fport,
	- &laddr, lport, numa_domain, fib);
	+ inp = in_pcblookup_lbgroup(ipictx->pcbinfo, &faddr, fport,
	+ &laddr, lport, numa_domain, fib, &ipictx->lbbucket);
	if (inp == NULL) {
	- inp = in_pcblookup_wild_locked(pcbinfo, laddr,
	- lport, fib);
	+ /*
	+ * Unlock the lbbucket to avoid LOR. If the lookup
	+ * is made for insertion, it will be acquired later.
	+ * It is safe to do so, since load balance groups
	+ * by design have "conflicting" inpcbs.
	+ */
	+ INPBUCKET_UNLOCK(ipictx->lbbucket);
	+ ipictx->lbbucket = NULL;
	+ inp = in_pcblookup_wild_locked(ipictx->pcbinfo, laddr,
	+ lport, fib, &ipictx->wbucket);
	}
	}

	@@ -2450,31 +2544,33 @@
	/*
	* Lookup inpcb using locks. Used by in_pcblookup_smr() in case inp_smr_lock()
	* failed.
	+ * Returns with locked inpcb.
	*/
	static struct inpcb *
	in_pcblookup_with_lock(struct inpcbinfo *pcbinfo, struct in_addr faddr,
	u_int fport, struct in_addr laddr, u_int lport, int lookupflags,
	uint8_t numa_domain, int fib)
	{
	- struct inpcb *inp;
	+ struct inpcbinfo_ctx ipictx = {
	+ .pcbinfo = pcbinfo
	+ };
	const inp_lookup_t lockflags = lookupflags & INPLOOKUP_LOCKMASK;
	+ struct inpcb *inp;

	KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB \| INPLOOKUP_WLOCKPCB)) != 0,
	("%s: LOCKPCB not set", __func__));

	- INP_HASH_WLOCK(pcbinfo);
	- inp = in_pcblookup_internal(pcbinfo, faddr, fport, laddr, lport,
	+ inp = in_pcblookup_internal(&ipictx, faddr, fport, laddr, lport,
	lookupflags & ~INPLOOKUP_LOCKMASK, numa_domain, fib);
	if (inp != NULL && !inp_trylock(inp, lockflags)) {
	in_pcbref(inp);
	- INP_HASH_WUNLOCK(pcbinfo);
	+ inpcbinfo_ctx_release(&ipictx);
	inp_lock(inp, lockflags);
	if (in_pcbrele(inp, lockflags))
	/* XXX-MJ or retry until we get a negative match? */
	inp = NULL;
	- } else {
	- INP_HASH_WUNLOCK(pcbinfo);
	- }
	+ } else
	+ inpcbinfo_ctx_release(&ipictx);
	return (inp);
	}

	@@ -2493,7 +2589,7 @@
	("%s: LOCKPCB not set", __func__));

	smr_enter(pcbinfo->ipi_smr);
	- inp = in_pcblookup_exact(pcbinfo, faddr, fport, laddr, lport);
	+ inp = in_pcblookup_exact(pcbinfo, faddr, fport, laddr, lport, NULL);
	if (inp != NULL) {
	if (__predict_true(inp_smr_lock(inp, lockflags))) {
	/*
	@@ -2516,7 +2612,7 @@

	if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
	inp = in_pcblookup_lbgroup(pcbinfo, &faddr, fport,
	- &laddr, lport, numa_domain, fib);
	+ &laddr, lport, numa_domain, fib, NULL);
	if (inp != NULL) {
	if (__predict_true(inp_smr_lock(inp, lockflags))) {
	if (__predict_true(in_pcblookup_wild_match(inp,
	@@ -2586,30 +2682,30 @@
	* always appear last no matter whether they are jailed.
	*/
	static void
	-_in_pcbinshash_wild(struct inpcbhead pcbhash, struct inpcb inp)
	+_in_pcbinshash_wild(struct inpbucket bucket, struct inpcb inp)
	{
	struct inpcb *last;
	bool bound, injail;

	INP_LOCK_ASSERT(inp);
	- INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
	+ INPBUCKET_LOCK_ASSERT(bucket);

	last = NULL;
	bound = inp->inp_laddr.s_addr != INADDR_ANY;
	if (!bound && (inp->inp_vflag & INP_IPV6PROTO) != 0) {
	- CK_LIST_FOREACH(last, pcbhash, inp_hash_wild) {
	+ CK_LIST_FOREACH(last, &bucket->head, inp_hash_wild) {
	if (CK_LIST_NEXT(last, inp_hash_wild) == NULL) {
	CK_LIST_INSERT_AFTER(last, inp, inp_hash_wild);
	return;
	}
	}
	- CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_wild);
	+ CK_LIST_INSERT_HEAD(&bucket->head, inp, inp_hash_wild);
	return;
	}

	injail = in_pcbjailed(inp, PR_IP4);
	if (!injail) {
	- CK_LIST_FOREACH(last, pcbhash, inp_hash_wild) {
	+ CK_LIST_FOREACH(last, &bucket->head, inp_hash_wild) {
	if (!in_pcbjailed(last, PR_IP4))
	break;
	if (CK_LIST_NEXT(last, inp_hash_wild) == NULL) {
	@@ -2617,13 +2713,13 @@
	return;
	}
	}
	- } else if (!CK_LIST_EMPTY(pcbhash) &&
	- !in_pcbjailed(CK_LIST_FIRST(pcbhash), PR_IP4)) {
	- CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_wild);
	+ } else if (!CK_LIST_EMPTY(&bucket->head) &&
	+ !in_pcbjailed(CK_LIST_FIRST(&bucket->head), PR_IP4)) {
	+ CK_LIST_INSERT_HEAD(&bucket->head, inp, inp_hash_wild);
	return;
	}
	if (!bound) {
	- CK_LIST_FOREACH_FROM(last, pcbhash, inp_hash_wild) {
	+ CK_LIST_FOREACH_FROM(last, &bucket->head, inp_hash_wild) {
	if (last->inp_laddr.s_addr == INADDR_ANY)
	break;
	if (CK_LIST_NEXT(last, inp_hash_wild) == NULL) {
	@@ -2633,7 +2729,7 @@
	}
	}
	if (last == NULL)
	- CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_wild);
	+ CK_LIST_INSERT_HEAD(&bucket->head, inp, inp_hash_wild);
	else
	CK_LIST_INSERT_BEFORE(last, inp, inp_hash_wild);
	}
	@@ -2643,19 +2739,19 @@
	* See the comment above _in_pcbinshash_wild().
	*/
	static void
	-_in6_pcbinshash_wild(struct inpcbhead pcbhash, struct inpcb inp)
	+_in6_pcbinshash_wild(struct inpbucket bucket, struct inpcb inp)
	{
	struct inpcb *last;
	bool bound, injail;

	INP_LOCK_ASSERT(inp);
	- INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
	+ INPBUCKET_LOCK_ASSERT(bucket);

	last = NULL;
	bound = !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr);
	injail = in_pcbjailed(inp, PR_IP6);
	if (!injail) {
	- CK_LIST_FOREACH(last, pcbhash, inp_hash_wild) {
	+ CK_LIST_FOREACH(last, &bucket->head, inp_hash_wild) {
	if (!in_pcbjailed(last, PR_IP6))
	break;
	if (CK_LIST_NEXT(last, inp_hash_wild) == NULL) {
	@@ -2663,13 +2759,13 @@
	return;
	}
	}
	- } else if (!CK_LIST_EMPTY(pcbhash) &&
	- !in_pcbjailed(CK_LIST_FIRST(pcbhash), PR_IP6)) {
	- CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_wild);
	+ } else if (!CK_LIST_EMPTY(&bucket->head) &&
	+ !in_pcbjailed(CK_LIST_FIRST(&bucket->head), PR_IP6)) {
	+ CK_LIST_INSERT_HEAD(&bucket->head, inp, inp_hash_wild);
	return;
	}
	if (!bound) {
	- CK_LIST_FOREACH_FROM(last, pcbhash, inp_hash_wild) {
	+ CK_LIST_FOREACH_FROM(last, &bucket->head, inp_hash_wild) {
	if (IN6_IS_ADDR_UNSPECIFIED(&last->in6p_laddr))
	break;
	if (CK_LIST_NEXT(last, inp_hash_wild) == NULL) {
	@@ -2679,7 +2775,7 @@
	}
	}
	if (last == NULL)
	- CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_wild);
	+ CK_LIST_INSERT_HEAD(&bucket->head, inp, inp_hash_wild);
	else
	CK_LIST_INSERT_BEFORE(last, inp, inp_hash_wild);
	}
	@@ -2693,37 +2789,41 @@
	* that's the only condition when it can fail.
	*/
	int
	-in_pcbinshash(struct inpcb *inp)
	+in_pcbinshash(struct inpcb inp, struct inpcbinfo_ctx ipictx)
	{
	- struct inpcbhead pcbhash, pcbporthash;
	- struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
	- uint32_t hash;
	+ uint32_t hash __diagused;
	bool connected;

	INP_WLOCK_ASSERT(inp);
	- INP_HASH_WLOCK_ASSERT(pcbinfo);
	MPASS(inp->inp_flags & INP_UNCONNECTED);

	#ifdef INET6
	if (inp->inp_vflag & INP_IPV6) {
	+#ifdef INVARIANTS
	hash = INP6_PCBHASH(&inp->in6p_faddr, inp->inp_lport,
	- inp->inp_fport, pcbinfo->ipi_hashmask);
	+ inp->inp_fport, inp->inp_pcbinfo->ipi_hashmask);
	+#endif
	connected = !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr);
	} else
	#endif
	{
	+#ifdef INVARIANTS
	hash = INP_PCBHASH(&inp->inp_faddr, inp->inp_lport,
	- inp->inp_fport, pcbinfo->ipi_hashmask);
	+ inp->inp_fport, inp->inp_pcbinfo->ipi_hashmask);
	+#endif
	connected = !in_nullhost(inp->inp_faddr);
	}

	- if (connected)
	- pcbhash = &pcbinfo->ipi_hash_exact[hash];
	- else
	- pcbhash = &pcbinfo->ipi_hash_wild[hash];
	-
	- pcbporthash = &pcbinfo->ipi_porthashbase[
	- INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_porthashmask)];
	+#ifdef INVARIANTS
	+ if (connected) {
	+ MPASS(ipictx->ebucket ==
	+ &ipictx->pcbinfo->ipi_hash_exact[hash]);
	+ INPBUCKET_LOCK_ASSERT(ipictx->ebucket);
	+ } else {
	+ MPASS(ipictx->wbucket == &ipictx->pcbinfo->ipi_hash_wild[hash]);
	+ INPBUCKET_LOCK_ASSERT(ipictx->wbucket);
	+ }
	+#endif

	/*
	* Ignore SO_REUSEPORT_LB if the socket is connected. Really this case
	@@ -2744,23 +2844,32 @@
	* may be traversing this PCB to finish.
	*/
	if (inp->inp_smr != SMR_SEQ_INVALID) {
	- smr_wait(pcbinfo->ipi_smr, inp->inp_smr);
	+ smr_wait(ipictx->pcbinfo->ipi_smr, inp->inp_smr);
	inp->inp_smr = SMR_SEQ_INVALID;
	}

	+ IPI_LOCK(ipictx->pcbinfo);
	CK_LIST_REMOVE(inp, inp_unconn_list);
	+ IPI_UNLOCK(ipictx->pcbinfo);

	- if (connected)
	- CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_exact);
	- else {
	+ if (connected) {
	+ CK_LIST_INSERT_HEAD(&ipictx->ebucket->head, inp,
	+ inp_hash_exact);
	+ } else {
	#ifdef INET6
	if ((inp->inp_vflag & INP_IPV6) != 0)
	- _in6_pcbinshash_wild(pcbhash, inp);
	+ _in6_pcbinshash_wild(ipictx->wbucket, inp);
	else
	#endif
	- _in_pcbinshash_wild(pcbhash, inp);
	+ _in_pcbinshash_wild(ipictx->wbucket, inp);
	}
	- CK_LIST_INSERT_HEAD(pcbporthash, inp, inp_portlist);
	+ /*
	+ * In most code paths we arrive here without locked pbucket. This is
	+ * always true for connect(2) and also true for bind(2) when the socket
	+ * has SO_REUSEADDR set then port hash checks were not performed.
	+ */
	+ inpcbinfo_ctx_portlock(ipictx, inp->inp_lport);
	+ CK_LIST_INSERT_HEAD(&ipictx->pbucket->head, inp, inp_portlist);
	inp->inp_flags &= ~INP_UNCONNECTED;

	return (0);
	@@ -2769,28 +2878,53 @@
	void
	in_pcbremhash(struct inpcb *inp)
	{
	+ struct inpbucket *bucket;
	+ uint32_t hash;
	+ bool connected;

	INP_WLOCK_ASSERT(inp);
	- INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
	MPASS(!(inp->inp_flags & INP_UNCONNECTED));

	- if ((inp->inp_flags & INP_INLBGROUP) != 0)
	- in_pcbremlbgrouphash(inp);
	+ if ((inp->inp_flags & INP_INLBGROUP) != 0) {
	+ struct lbgroupbucket *lbbucket;
	+ struct inpcblbgroup *g __diagused;
	+
	+ g = in_pcblbgroup_find(inp, &lbbucket);
	+ MPASS(g);
	+ in_pcbremlbgrouphash(lbbucket, inp);
	+ INPBUCKET_UNLOCK(lbbucket);
	+ }
	+
	#ifdef INET6
	if (inp->inp_vflag & INP_IPV6) {
	- if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr))
	- CK_LIST_REMOVE(inp, inp_hash_wild);
	- else
	- CK_LIST_REMOVE(inp, inp_hash_exact);
	+ hash = INP6_PCBHASH(&inp->in6p_faddr, inp->inp_lport,
	+ inp->inp_fport, inp->inp_pcbinfo->ipi_hashmask);
	+ connected = !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr);
	} else
	#endif
	{
	- if (in_nullhost(inp->inp_faddr))
	- CK_LIST_REMOVE(inp, inp_hash_wild);
	- else
	- CK_LIST_REMOVE(inp, inp_hash_exact);
	+ hash = INP_PCBHASH(&inp->inp_faddr, inp->inp_lport,
	+ inp->inp_fport, inp->inp_pcbinfo->ipi_hashmask);
	+ connected = !in_nullhost(inp->inp_faddr);
	+ }
	+
	+ if (connected) {
	+ bucket = &inp->inp_pcbinfo->ipi_hash_exact[hash];
	+ INPBUCKET_LOCK(bucket);
	+ CK_LIST_REMOVE(inp, inp_hash_exact);
	+ } else {
	+ bucket = &inp->inp_pcbinfo->ipi_hash_wild[hash];
	+ INPBUCKET_LOCK(bucket);
	+ CK_LIST_REMOVE(inp, inp_hash_wild);
	}
	+ INPBUCKET_UNLOCK(bucket);
	+
	+ bucket = &inp->inp_pcbinfo->ipi_porthash[
	+ INP_PCBPORTHASH(inp->inp_lport,
	+ inp->inp_pcbinfo->ipi_porthashmask)];
	+ INPBUCKET_LOCK(bucket);
	CK_LIST_REMOVE(inp, inp_portlist);
	+ INPBUCKET_UNLOCK(bucket);
	}

	/*
	@@ -2800,38 +2934,40 @@
	* not change after in_pcbinshash() has been called.
	*/
	void
	-in_pcbrehash(struct inpcb *inp)
	+in_pcbrehash(struct inpcb inp, struct inpcbinfo_ctx ipictx)
	{
	- struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
	- struct inpcbhead *head;
	+#ifdef INVARIANTS
	uint32_t hash;

	INP_WLOCK_ASSERT(inp);
	- INP_HASH_WLOCK_ASSERT(pcbinfo);
	MPASS(!(inp->inp_flags & INP_UNCONNECTED));
	KASSERT(inp->inp_smr == SMR_SEQ_INVALID,
	("%s: inp was disconnected", __func__));
	-
	#ifdef INET6
	if (inp->inp_vflag & INP_IPV6) {
	MPASS(!IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr));
	hash = INP6_PCBHASH(&inp->in6p_faddr, inp->inp_lport,
	- inp->inp_fport, pcbinfo->ipi_hashmask);
	+ inp->inp_fport, ipictx->pcbinfo->ipi_hashmask);
	} else
	#endif
	{
	MPASS(!in_nullhost(inp->inp_faddr));
	hash = INP_PCBHASH(&inp->inp_faddr, inp->inp_lport,
	- inp->inp_fport, pcbinfo->ipi_hashmask);
	+ inp->inp_fport, ipictx->pcbinfo->ipi_hashmask);
	}
	+ MPASS(ipictx->ebucket == &ipictx->pcbinfo->ipi_hash_exact[hash]);
	+ INPBUCKET_LOCK_ASSERT(ipictx->ebucket);
	+ INPBUCKET_LOCK_ASSERT(ipictx->wbucket);
	+#endif

	/* See the comment in in_pcbinshash(). */
	- if ((inp->inp_flags & INP_INLBGROUP) != 0)
	- in_pcbremlbgrouphash(inp);
	+ if ((inp->inp_flags & INP_INLBGROUP) != 0) {
	+ INPBUCKET_LOCK_ASSERT(ipictx->lbbucket);
	+ in_pcbremlbgrouphash(ipictx->lbbucket, inp);
	+ }

	CK_LIST_REMOVE(inp, inp_hash_wild);
	- head = &pcbinfo->ipi_hash_exact[hash];
	- CK_LIST_INSERT_HEAD(head, inp, inp_hash_exact);
	+ CK_LIST_INSERT_HEAD(&ipictx->ebucket->head, inp, inp_hash_exact);
	}

	void
	@@ -2845,11 +2981,11 @@

	hash = RIPCB_HASH(inp) & pcbinfo->ipi_hashmask;

	- INP_HASH_WLOCK(pcbinfo);
	+ IPI_LOCK(pcbinfo);
	CK_LIST_REMOVE(inp, inp_unconn_list);
	- CK_LIST_INSERT_HEAD(&pcbinfo->ipi_hash_exact[hash], inp,
	+ CK_LIST_INSERT_HEAD(&pcbinfo->ipi_hash_exact[hash].head, inp,
	inp_hash_exact);
	- INP_HASH_WUNLOCK(pcbinfo);
	+ IPI_UNLOCK(pcbinfo);
	inp->inp_flags &= ~INP_UNCONNECTED;
	}

	@@ -2863,10 +2999,11 @@
	if (inp->inp_flags & INP_UNCONNECTED)
	return;

	- INP_HASH_WLOCK(pcbinfo);
	+ IPI_LOCK(pcbinfo);
	CK_LIST_REMOVE(inp, inp_hash_exact);
	- CK_LIST_INSERT_HEAD(&pcbinfo->ipi_list_unconn, inp, inp_unconn_list);
	- INP_HASH_WUNLOCK(pcbinfo);
	+ CK_LIST_INSERT_HEAD(&pcbinfo->ipi_list_unconn.head, inp,
	+ inp_unconn_list);
	+ IPI_UNLOCK(pcbinfo);
	inp->inp_flags \|= INP_UNCONNECTED;
	}

	diff --git a/sys/netinet/in_pcb_var.h b/sys/netinet/in_pcb_var.h
	--- a/sys/netinet/in_pcb_var.h
	+++ b/sys/netinet/in_pcb_var.h
	@@ -42,21 +42,98 @@
	*/

	#define INP_UNCONNECTED 0x04000000 /* Not inserted into hashes. */
	+#define INP_INLBGROUP 0x01000000 /* Inserted into inpcblbgroup. */
	+
	+/*
	+ * The inpcb database change context used by bind(2) and connect(2) operations.
	+ * XXX REVIEWERS: any better name for the struct and its instance common name?
	+ * Right now it looks like:
	+ struct inpcbinfo_ctx ipictx = {
	+ .pcbinfo = pcbinfo
	+ };
	+ */
	+struct inpcbinfo_ctx {
	+ struct inpcbinfo *pcbinfo;
	+ struct inpbucket ebucket, / Target exact bucket */
	+ wbucket, / Target(or old) wild bucket */
	+ pbucket; / Port bucket */
	+ struct lbgroupbucket lbbucket; / Load balancer bucket */
	+};
	+
	+#define INPBUCKET_LOCK(bucket) mtx_lock(&(bucket)->lock)
	+#define INPBUCKET_UNLOCK(bucket) mtx_unlock(&(bucket)->lock)
	+#define INPBUCKET_LOCK_ASSERT(bucket) mtx_assert(&(bucket)->lock, MA_OWNED)
	+
	+#define IPI_LOCK(ipi) INPBUCKET_LOCK(&(ipi)->ipi_list_unconn)
	+#define IPI_UNLOCK(ipi) INPBUCKET_UNLOCK(&(ipi)->ipi_list_unconn)
	+#define IPI_LOCK_ASSERT(ipi) INPBUCKET_LOCK_ASSERT(&(ipi)->ipi_list_unconn)
	+
	+static inline void
	+inpcbinfo_ctx_wildlock(struct inpcbinfo_ctx *ipictx, uint16_t lport)
	+{
	+ if (ipictx->wbucket == NULL) {
	+ ipictx->wbucket = &ipictx->pcbinfo->ipi_hash_wild[
	+ INP_PCBHASH_WILD(lport, ipictx->pcbinfo->ipi_hashmask)];
	+ INPBUCKET_LOCK(ipictx->wbucket);
	+ } else {
	+ MPASS(ipictx->wbucket == &ipictx->pcbinfo->ipi_hash_wild[
	+ INP_PCBHASH_WILD(lport, ipictx->pcbinfo->ipi_hashmask)]);
	+ INPBUCKET_LOCK_ASSERT(ipictx->wbucket);
	+ }
	+}
	+
	+static inline void
	+inpcbinfo_ctx_portlock(struct inpcbinfo_ctx *ipictx, uint16_t lport)
	+{
	+ if (ipictx->pbucket == NULL) {
	+ ipictx->pbucket = &ipictx->pcbinfo->ipi_porthash[
	+ INP_PCBPORTHASH(lport, ipictx->pcbinfo->ipi_porthashmask)];
	+ INPBUCKET_LOCK(ipictx->pbucket);
	+ } else {
	+ MPASS(ipictx->pbucket == &ipictx->pcbinfo->ipi_porthash[
	+ INP_PCBPORTHASH(lport, ipictx->pcbinfo->ipi_porthashmask)]);
	+ INPBUCKET_LOCK_ASSERT(ipictx->pbucket);
	+ }
	+}
	+
	+static inline void
	+inpcbinfo_ctx_release(struct inpcbinfo_ctx *ipictx)
	+{
	+ MPASS(kstack_contains(curthread, (vm_offset_t)ipictx, sizeof(*ipictx)));
	+
	+ if (ipictx->ebucket != NULL)
	+ INPBUCKET_UNLOCK(ipictx->ebucket);
	+ if (ipictx->wbucket != NULL)
	+ INPBUCKET_UNLOCK(ipictx->wbucket);
	+ if (ipictx->pbucket != NULL)
	+ INPBUCKET_UNLOCK(ipictx->pbucket);
	+ if (ipictx->lbbucket != NULL)
	+ INPBUCKET_UNLOCK(ipictx->lbbucket);
	+}

	void inp_lock(struct inpcb *inp, const inp_lookup_t lock);
	void inp_unlock(struct inpcb *inp, const inp_lookup_t lock);
	int inp_trylock(struct inpcb *inp, const inp_lookup_t lock);
	bool inp_smr_lock(struct inpcb *, const inp_lookup_t);
	-int in_pcb_lport(struct inpcb , struct in_addr , u_short *,
	- struct ucred *, int);
	-int in_pcb_lport_dest(const struct inpcb inp, struct sockaddr lsa,
	- u_short lportp, struct sockaddr fsa, u_short fport,
	- struct ucred *cred, int lookupflags);
	-struct inpcb in_pcblookup_local(struct inpcbinfo , struct in_addr, u_short,
	- int, int, struct ucred *);
	-int in_pcbinshash(struct inpcb *);
	-void in_pcbrehash(struct inpcb *);
	+int in_pcb_lport(struct inpcbinfo_ctx , struct inpcb , struct in_addr *,
	+ u_short , struct ucred , int);
	+int in_pcb_lport_dest(struct inpcbinfo_ctx *ipictx,
	+ const struct inpcb inp, const struct sockaddr lsa,
	+ const struct sockaddr fsa, u_short fport, struct ucred cred,
	+ int lookupflags, u_short *lportp);
	+struct inpcb in_pcblookup_local(struct inpcbinfo_ctx , struct in_addr,
	+ u_short, int, int, struct ucred *);
	+struct inpcb in6_pcblookup_local(struct inpcbinfo_ctx ,
	+ const struct in6_addr , u_short, int, int, struct ucred );
	+struct inpcb in6_pcblookup_internal(struct inpcbinfo_ctx ipictx,
	+ const struct in6_addr *faddr, u_int fport_arg,
	+ const struct in6_addr *laddr, u_int lport_arg,
	+ int lookupflags, uint8_t numa_domain, int fib);
	+int in_pcbinshash(struct inpcb , struct inpcbinfo_ctx );
	+void in_pcbrehash(struct inpcb , struct inpcbinfo_ctx );
	void in_pcbremhash(struct inpcb *);
	+struct inpcblbgroup in_pcblbgroup_find(struct inpcb inp,
	+ struct lbgroupbucket **bucket);

	/*
	* Load balance groups used for the SO_REUSEPORT_LB socket option. Each group
	diff --git a/sys/netinet/tcp_usrreq.c b/sys/netinet/tcp_usrreq.c
	--- a/sys/netinet/tcp_usrreq.c
	+++ b/sys/netinet/tcp_usrreq.c
	@@ -2939,16 +2939,24 @@
	show_inpcb = strchr(modif, 'i') != NULL;
	VNET_FOREACH(vnet_iter) {
	CURVNET_SET(vnet_iter);
	- for (u_int i = 0; i <= V_tcbinfo.ipi_porthashmask; i++)
	- CK_LIST_FOREACH(inp, &V_tcbinfo.ipi_porthashbase[i],
	- inp_portlist) {
	+ for (u_int i = 0; i <= V_tcbinfo.ipi_hashmask; i++)
	+ CK_LIST_FOREACH(inp, &V_tcbinfo.ipi_hash_exact[i].head,
	+ inp_hash_exact) {
	+ db_print_tcpcb(intotcpcb(inp), "tcpcb", 0,
	+ show_bblog, show_inpcb, only_locked);
	+ if (db_pager_quit)
	+ goto break_hash;
	+ }
	+ for (u_int i = 0; i <= V_tcbinfo.ipi_hashmask; i++)
	+ CK_LIST_FOREACH(inp, &V_tcbinfo.ipi_hash_wild[i].head,
	+ inp_hash_wild) {
	db_print_tcpcb(intotcpcb(inp), "tcpcb", 0,
	show_bblog, show_inpcb, only_locked);
	if (db_pager_quit)
	goto break_hash;
	}
	break_hash:
	- CK_LIST_FOREACH(inp, &V_tcbinfo.ipi_list_unconn,
	+ CK_LIST_FOREACH(inp, &V_tcbinfo.ipi_list_unconn.head,
	inp_unconn_list) {
	db_print_tcpcb(intotcpcb(inp), "tcpcb", 0,
	show_bblog, show_inpcb, only_locked);
	diff --git a/sys/netinet6/in6_pcb.h b/sys/netinet6/in6_pcb.h
	--- a/sys/netinet6/in6_pcb.h
	+++ b/sys/netinet6/in6_pcb.h
	@@ -74,12 +74,6 @@
	int in6_pcbconnect(struct inpcb , struct sockaddr_in6 , struct ucred *,
	bool);
	void in6_pcbdisconnect(struct inpcb *);
	-struct inpcb in6_pcblookup_local(struct inpcbinfo , const struct in6_addr *,
	- u_short, int, int, struct ucred *);
	-struct inpcb in6_pcblookup_internal(struct inpcbinfo pcbinfo,
	- const struct in6_addr *faddr, u_int fport_arg,
	- const struct in6_addr *laddr, u_int lport_arg,
	- int lookupflags, uint8_t numa_domain, int fib);
	struct inpcb in6_pcblookup(struct inpcbinfo , const struct in6_addr *, u_int,
	const struct in6_addr , u_int, int, struct ifnet );
	struct inpcb in6_pcblookup_mbuf(struct inpcbinfo , const struct in6_addr *,
	diff --git a/sys/netinet6/in6_pcb.c b/sys/netinet6/in6_pcb.c
	--- a/sys/netinet6/in6_pcb.c
	+++ b/sys/netinet6/in6_pcb.c
	@@ -87,6 +87,9 @@
	#include <sys/priv.h>
	#include <sys/proc.h>
	#include <sys/jail.h>
	+#ifdef INVARIANTS
	+#include <machine/stack.h>
	+#endif

	#include <vm/uma.h>

	@@ -112,16 +115,22 @@
	#include <netinet6/in6_fib.h>
	#include <netinet6/scope6_var.h>

	+static struct inpcb in6_pcblookup_exact(struct inpcbinfo pcbinfo,
	+ const struct in6_addr faddr, u_short fport, const struct in6_addr laddr,
	+ u_short lport, struct inpbucket **bucketp);
	+static struct inpcb in6_pcblookup_wild_locked(struct inpcbinfo pcbinfo,
	+ const struct in6_addr *laddr, u_short lport, int fib,
	+ struct inpbucket **bucketp);
	+
	static int
	-in6_pcbsetport_locked(struct in6_addr laddr, struct inpcb inp,
	- struct ucred *cred)
	+in6_pcbsetport_locked(struct inpcbinfo_ctx ipictx, struct in6_addr laddr,
	+ struct inpcb inp, struct ucred cred)
	{
	struct socket *so = inp->inp_socket;
	u_int16_t lport = 0;
	int error, lookupflags = 0;

	INP_WLOCK_ASSERT(inp);
	- INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);

	error = prison_local_ip6(cred, laddr,
	((inp->inp_flags & IN6P_IPV6_V6ONLY) != 0));
	@@ -132,30 +141,33 @@
	if ((so->so_options & (SO_REUSEADDR\|SO_REUSEPORT\|SO_REUSEPORT_LB)) == 0)
	lookupflags = INPLOOKUP_WILDCARD;

	- inp->inp_flags \|= INP_ANONPORT;
	-
	- error = in_pcb_lport(inp, NULL, &lport, cred, lookupflags);
	+ error = in_pcb_lport(ipictx, inp, NULL, &lport, cred, lookupflags);
	if (error != 0)
	return (error);

	inp->inp_lport = lport;
	- if (in_pcbinshash(inp) != 0) {
	+ if (__predict_false((error = in_pcbinshash(inp, ipictx)) != 0)) {
	+ MPASS(inp->inp_socket->so_options & SO_REUSEPORT_LB);
	inp->in6p_laddr = in6addr_any;
	inp->inp_lport = 0;
	- return (EAGAIN);
	+ return (error);
	}

	+ inp->inp_flags \|= INP_ANONPORT;
	+
	return (0);
	}

	int
	in6_pcbsetport(struct in6_addr laddr, struct inpcb inp, struct ucred *cred)
	{
	+ struct inpcbinfo_ctx ipictx = {
	+ .pcbinfo = inp->inp_pcbinfo
	+ };
	int error;

	- INP_HASH_WLOCK(inp->inp_pcbinfo);
	- error = in6_pcbsetport_locked(laddr, inp, cred);
	- INP_HASH_WUNLOCK(inp->inp_pcbinfo);
	+ error = in6_pcbsetport_locked(&ipictx, laddr, inp, cred);
	+ inpcbinfo_ctx_release(&ipictx);

	return (error);
	}
	@@ -164,15 +176,15 @@
	* Determine whether the inpcb can be bound to the specified address/port tuple.
	*/
	static int
	-in6_pcbbind_avail(struct inpcb inp, const struct sockaddr_in6 sin6, int fib,
	- int sooptions, int lookupflags, struct ucred *cred)
	+in6_pcbbind_avail(struct inpcbinfo_ctx ipictx, struct inpcb inp,
	+ const struct sockaddr_in6 *sin6, int fib, int sooptions, int lookupflags,
	+ struct ucred *cred)
	{
	const struct in6_addr *laddr;
	int reuseport, reuseport_lb;
	u_short lport;

	INP_LOCK_ASSERT(inp);
	- INP_HASH_LOCK_ASSERT(inp->inp_pcbinfo);

	laddr = &sin6->sin6_addr;
	lport = sin6->sin6_port;
	@@ -247,7 +259,7 @@
	* However, we can share a port with a connected socket
	* which has a unique 4-tuple.
	*/
	- t = in6_pcblookup_local(inp->inp_pcbinfo, laddr, lport,
	+ t = in6_pcblookup_local(ipictx, laddr, lport,
	RT_ALL_FIBS, INPLOOKUP_WILDCARD, cred);
	if (t != NULL &&
	(inp->inp_socket->so_type != SOCK_STREAM \|\|
	@@ -261,7 +273,7 @@
	struct sockaddr_in sin;

	in6_sin6_2_sin(&sin, sin6);
	- t = in_pcblookup_local(inp->inp_pcbinfo,
	+ t = in_pcblookup_local(ipictx,
	sin.sin_addr, lport, RT_ALL_FIBS,
	INPLOOKUP_WILDCARD, cred);
	if (t != NULL &&
	@@ -273,8 +285,8 @@
	}
	#endif
	}
	- t = in6_pcblookup_local(inp->inp_pcbinfo, laddr, lport,
	- fib, lookupflags, cred);
	+ t = in6_pcblookup_local(ipictx, laddr, lport, fib, lookupflags,
	+ cred);
	if (t != NULL && ((reuseport \| reuseport_lb) &
	t->inp_socket->so_options) == 0)
	return (EADDRINUSE);
	@@ -284,8 +296,8 @@
	struct sockaddr_in sin;

	in6_sin6_2_sin(&sin, sin6);
	- t = in_pcblookup_local(inp->inp_pcbinfo, sin.sin_addr,
	- lport, RT_ALL_FIBS, lookupflags, cred);
	+ t = in_pcblookup_local(ipictx, sin.sin_addr, lport,
	+ RT_ALL_FIBS, lookupflags, cred);
	if (t != NULL && ((reuseport \| reuseport_lb) &
	t->inp_socket->so_options) == 0 &&
	(!in_nullhost(t->inp_laddr) \|\|
	@@ -302,6 +314,9 @@
	in6_pcbbind(struct inpcb inp, struct sockaddr_in6 sin6, int flags,
	struct ucred *cred)
	{
	+ struct inpcbinfo_ctx ipictx = {
	+ .pcbinfo = inp->inp_pcbinfo
	+ };
	struct socket *so = inp->inp_socket;
	u_short lport = 0;
	int error, fib, lookupflags, sooptions;
	@@ -319,7 +334,6 @@
	if ((error = prison_local_ip6(cred, &inp->in6p_laddr,
	((inp->inp_flags & IN6P_IPV6_V6ONLY) != 0))) != 0)
	return (error);
	- INP_HASH_WLOCK(inp->inp_pcbinfo);
	} else {
	KASSERT(sin6->sin6_family == AF_INET6,
	("%s: invalid address family for %p", __func__, sin6));
	@@ -336,12 +350,11 @@
	fib = (flags & INPBIND_FIB) != 0 ? inp->inp_inc.inc_fibnum :
	RT_ALL_FIBS;

	- INP_HASH_WLOCK(inp->inp_pcbinfo);
	/* See if this address/port combo is available. */
	- error = in6_pcbbind_avail(inp, sin6, fib, sooptions,
	+ error = in6_pcbbind_avail(&ipictx, inp, sin6, fib, sooptions,
	lookupflags, cred);
	if (error != 0) {
	- INP_HASH_WUNLOCK(inp->inp_pcbinfo);
	+ inpcbinfo_ctx_release(&ipictx);
	return (error);
	}

	@@ -351,9 +364,10 @@
	if ((flags & INPBIND_FIB) != 0)
	inp->inp_flags \|= INP_BOUNDFIB;
	if (lport == 0) {
	- error = in6_pcbsetport_locked(&inp->in6p_laddr, inp, cred);
	+ error = in6_pcbsetport_locked(&ipictx, &inp->in6p_laddr, inp,
	+ cred);
	if (__predict_false(error != 0)) {
	- INP_HASH_WUNLOCK(inp->inp_pcbinfo);
	+ inpcbinfo_ctx_release(&ipictx);
	/* Undo an address bind that may have occurred. */
	inp->inp_flags &= ~INP_BOUNDFIB;
	inp->in6p_laddr = in6addr_any;
	@@ -361,15 +375,16 @@
	}
	} else {
	inp->inp_lport = lport;
	- if (in_pcbinshash(inp) != 0) {
	- INP_HASH_WUNLOCK(inp->inp_pcbinfo);
	+ error = in_pcbinshash(inp, &ipictx);
	+ if (__predict_false(error != 0)) {
	+ inpcbinfo_ctx_release(&ipictx);
	inp->inp_flags &= ~INP_BOUNDFIB;
	inp->in6p_laddr = in6addr_any;
	inp->inp_lport = 0;
	- return (EAGAIN);
	+ return (error);
	}
	}
	- INP_HASH_WUNLOCK(inp->inp_pcbinfo);
	+ inpcbinfo_ctx_release(&ipictx);

	return (0);
	}
	@@ -384,7 +399,6 @@

	NET_EPOCH_ASSERT();
	INP_WLOCK_ASSERT(inp);
	- INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo); /* XXXRW: why? */

	if (sin6->sin6_port == 0)
	return (EADDRNOTAVAIL);
	@@ -449,7 +463,9 @@
	in6_pcbconnect(struct inpcb inp, struct sockaddr_in6 sin6, struct ucred *cred,
	bool sas_required)
	{
	- struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
	+ struct inpcbinfo_ctx ipictx = {
	+ .pcbinfo = inp->inp_pcbinfo
	+ };
	struct sockaddr_in6 laddr6 = { .sin6_family = AF_INET6 };
	int error;
	bool anonport;
	@@ -465,7 +481,6 @@

	anonport = (inp->inp_lport == 0);

	- INP_HASH_WLOCK(pcbinfo);
	if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr)) {
	/*
	* Call inner routine, to assign local interface address.
	@@ -473,26 +488,39 @@
	*/
	error = in6_pcbladdr(inp, sin6, &laddr6.sin6_addr,
	sas_required);
	- if (__predict_false(error)) {
	- INP_HASH_WUNLOCK(pcbinfo);
	+ if (__predict_false(error))
	return (error);
	- }
	} else
	laddr6.sin6_addr = inp->in6p_laddr;

	if (anonport) {
	- error = in_pcb_lport_dest(inp, (struct sockaddr *) &laddr6,
	- &inp->inp_lport, (struct sockaddr *) sin6, sin6->sin6_port,
	- cred, INPLOOKUP_WILDCARD);
	+ error = in_pcb_lport_dest(&ipictx, inp,
	+ (struct sockaddr )&laddr6, (struct sockaddr )sin6,
	+ sin6->sin6_port, cred, INPLOOKUP_WILDCARD, &inp->inp_lport);
	if (__predict_false(error)) {
	- INP_HASH_WUNLOCK(pcbinfo);
	+ inpcbinfo_ctx_release(&ipictx);
	return (error);
	}
	- } else if (in6_pcblookup_internal(pcbinfo, &sin6->sin6_addr,
	- sin6->sin6_port, &laddr6.sin6_addr, inp->inp_lport, 0,
	- M_NODOM, RT_ALL_FIBS) != NULL) {
	- INP_HASH_WUNLOCK(pcbinfo);
	- return (EADDRINUSE);
	+ } else {
	+ if (in6_pcblookup_exact(ipictx.pcbinfo, &sin6->sin6_addr,
	+ sin6->sin6_port, &laddr6.sin6_addr, inp->inp_lport,
	+ &ipictx.ebucket) != NULL) {
	+ inpcbinfo_ctx_release(&ipictx);
	+ return (EADDRINUSE);
	+ }
	+ if ((inp->inp_flags & INP_UNCONNECTED) == 0) {
	+ struct inpcb *i __diagused;
	+ struct inpcblbgroup *g __diagused;
	+
	+ i = in6_pcblookup_wild_locked(ipictx.pcbinfo,
	+ &laddr6.sin6_addr, inp->inp_lport, RT_ALL_FIBS,
	+ &ipictx.wbucket);
	+ MPASS(i);
	+ if (inp->inp_flags & INP_INLBGROUP) {
	+ g = in_pcblbgroup_find(inp, &ipictx.lbbucket);
	+ MPASS(g);
	+ }
	+ }
	}

	MPASS(inp->inp_lport != 0);
	@@ -507,11 +535,11 @@
	(htonl(ip6_randomflowlabel()) & IPV6_FLOWLABEL_MASK);

	if (inp->inp_flags & INP_UNCONNECTED) {
	- error = in_pcbinshash(inp);
	+ error = in_pcbinshash(inp, &ipictx);
	MPASS(error == 0);
	} else
	- in_pcbrehash(inp);
	- INP_HASH_WUNLOCK(pcbinfo);
	+ in_pcbrehash(inp, &ipictx);
	+ inpcbinfo_ctx_release(&ipictx);

	if (V_fib_hash_outbound) {
	uint32_t hash_type, hash_val;
	@@ -539,12 +567,12 @@
	if (inp->inp_flags & INP_UNCONNECTED)
	return;

	- INP_HASH_WLOCK(inp->inp_pcbinfo);
	in_pcbremhash(inp);
	- inp->inp_flags \|= INP_UNCONNECTED;
	- CK_LIST_INSERT_HEAD(&inp->inp_pcbinfo->ipi_list_unconn, inp,
	+ IPI_LOCK(inp->inp_pcbinfo);
	+ CK_LIST_INSERT_HEAD(&inp->inp_pcbinfo->ipi_list_unconn.head, inp,
	inp_unconn_list);
	- INP_HASH_WUNLOCK(inp->inp_pcbinfo);
	+ IPI_UNLOCK(inp->inp_pcbinfo);
	+ inp->inp_flags \|= INP_UNCONNECTED;

	if ((inp->inp_socket->so_proto->pr_flags & PR_CONNREQUIRED) == 0) {
	/* See the comment in in_pcbinshash(). */
	@@ -733,7 +761,7 @@
	* hash lock. No inpcb locks or references are acquired.
	*/
	struct inpcb *
	-in6_pcblookup_local(struct inpcbinfo pcbinfo, const struct in6_addr laddr,
	+in6_pcblookup_local(struct inpcbinfo_ctx ipictx, const struct in6_addr laddr,
	u_short lport, int fib, int lookupflags, struct ucred *cred)
	{
	struct inpcb *inp;
	@@ -744,17 +772,15 @@
	KASSERT(fib == RT_ALL_FIBS \|\| (fib >= 0 && fib < V_rt_numfibs),
	("%s: invalid fib %d", __func__, fib));

	- INP_HASH_LOCK_ASSERT(pcbinfo);
	+ /* See comment in in_pcblookup_local(). */
	+ inpcbinfo_ctx_wildlock(ipictx, lport);

	if ((lookupflags & INPLOOKUP_WILDCARD) == 0) {
	- struct inpcbhead *head;
	/*
	* Look for an unconnected (wildcard foreign addr) PCB that
	* matches the local address and port we're looking for.
	*/
	- head = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport,
	- pcbinfo->ipi_hashmask)];
	- CK_LIST_FOREACH(inp, head, inp_hash_wild) {
	+ CK_LIST_FOREACH(inp, &ipictx->wbucket->head, inp_hash_wild) {
	/* XXX inp locking */
	if ((inp->inp_vflag & INP_IPV6) == 0)
	continue;
	@@ -772,16 +798,14 @@
	*/
	return (NULL);
	} else {
	- struct inpcbhead *porthash;
	struct inpcb *match = NULL;

	/*
	* Port is in use by one or more PCBs. Look for best
	* fit.
	*/
	- porthash = &pcbinfo->ipi_porthashbase[INP_PCBPORTHASH(lport,
	- pcbinfo->ipi_porthashmask)];
	- CK_LIST_FOREACH(inp, porthash, inp_portlist) {
	+ inpcbinfo_ctx_portlock(ipictx, lport);
	+ CK_LIST_FOREACH(inp, &ipictx->pbucket->head, inp_portlist) {
	if (inp->inp_lport != lport)
	continue;
	if (!prison_equal_ip6(cred->cr_prison,
	@@ -902,19 +926,16 @@
	static struct inpcb *
	in6_pcblookup_lbgroup(const struct inpcbinfo *pcbinfo,
	const struct in6_addr faddr, uint16_t fport, const struct in6_addr laddr,
	- uint16_t lport, uint8_t domain, int fib)
	+ uint16_t lport, uint8_t domain, int fib, struct lbgroupbucket **bucketp)
	{
	- const struct inpcblbgrouphead *hdr;
	+ struct lbgroupbucket *bucket;
	struct inpcblbgroup *grp;
	struct inpcblbgroup jail_exact, jail_wild, local_exact, local_wild;
	struct inpcb *inp;
	u_int count;

	- INP_HASH_LOCK_ASSERT(pcbinfo);
	NET_EPOCH_ASSERT();
	-
	- hdr = &pcbinfo->ipi_lbgrouphashbase[
	- INP_PCBPORTHASH(lport, pcbinfo->ipi_lbgrouphashmask)];
	+ MPASS(bucketp != NULL \|\| SMR_ENTERED(pcbinfo->ipi_smr));

	/*
	* Search for an LB group match based on the following criteria:
	@@ -923,7 +944,14 @@
	* - prefer groups bound to the specified NUMA domain
	*/
	jail_exact = jail_wild = local_exact = local_wild = NULL;
	- CK_LIST_FOREACH(grp, hdr, il_list) {
	+ bucket = &pcbinfo->ipi_lbgrouphashbase[
	+ INP_PCBPORTHASH(lport, pcbinfo->ipi_lbgrouphashmask)];
	+ if (bucketp != NULL) {
	+ MPASS(*bucketp == NULL);
	+ *bucketp = bucket;
	+ INPBUCKET_LOCK(bucket);
	+ }
	+ CK_LIST_FOREACH(grp, &bucket->head, il_list) {
	bool injail;

	#ifdef INET
	@@ -998,19 +1026,25 @@

	static struct inpcb *
	in6_pcblookup_exact(struct inpcbinfo pcbinfo, const struct in6_addr faddr,
	- u_short fport, const struct in6_addr *laddr, u_short lport)
	+ u_short fport, const struct in6_addr *laddr, u_short lport,
	+ struct inpbucket **bucketp)
	{
	- struct inpcbhead *head;
	+ struct inpbucket *bucket;
	struct inpcb *inp;

	- INP_HASH_LOCK_ASSERT(pcbinfo);
	+ MPASS(bucketp != NULL \|\| SMR_ENTERED(pcbinfo->ipi_smr));

	/*
	* First look for an exact match.
	*/
	- head = &pcbinfo->ipi_hash_exact[INP6_PCBHASH(faddr, lport, fport,
	+ bucket = &pcbinfo->ipi_hash_exact[INP6_PCBHASH(faddr, lport, fport,
	pcbinfo->ipi_hashmask)];
	- CK_LIST_FOREACH(inp, head, inp_hash_exact) {
	+ if (bucketp != NULL) {
	+ MPASS(*bucketp == NULL);
	+ *bucketp = bucket;
	+ INPBUCKET_LOCK(bucket);
	+ }
	+ CK_LIST_FOREACH(inp, &bucket->head, inp_hash_exact) {
	if (in6_pcblookup_exact_match(inp, faddr, fport, laddr, lport))
	return (inp);
	}
	@@ -1049,15 +1083,15 @@
	const struct in6_addr *laddr, u_short lport, int fib,
	const inp_lookup_t lockflags)
	{
	- struct inpcbhead *head;
	+ struct inpbucket *bucket;
	struct inpcb *inp;

	KASSERT(SMR_ENTERED(pcbinfo->ipi_smr),
	("%s: not in SMR read section", __func__));

	- head = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport,
	+ bucket = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport,
	pcbinfo->ipi_hashmask)];
	- CK_LIST_FOREACH(inp, head, inp_hash_wild) {
	+ CK_LIST_FOREACH(inp, &bucket->head, inp_hash_wild) {
	inp_lookup_match_t match;

	match = in6_pcblookup_wild_match(inp, laddr, lport, fib);
	@@ -1085,12 +1119,13 @@

	static struct inpcb *
	in6_pcblookup_wild_locked(struct inpcbinfo *pcbinfo,
	- const struct in6_addr *laddr, u_short lport, int fib)
	+ const struct in6_addr *laddr, u_short lport, int fib,
	+ struct inpbucket **bucketp)
	{
	- struct inpcbhead *head;
	+ struct inpbucket *bucket;
	struct inpcb inp, jail_wild, local_exact, local_wild;

	- INP_HASH_LOCK_ASSERT(pcbinfo);
	+ MPASS(bucketp != NULL);

	/*
	* Order of socket selection - we always prefer jails.
	@@ -1099,10 +1134,13 @@
	* 3. non-jailed, non-wild.
	* 4. non-jailed, wild.
	*/
	- head = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport,
	- pcbinfo->ipi_hashmask)];
	local_wild = local_exact = jail_wild = NULL;
	- CK_LIST_FOREACH(inp, head, inp_hash_wild) {
	+ bucket = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport,
	+ pcbinfo->ipi_hashmask)];
	+ MPASS(*bucketp == NULL);
	+ *bucketp = bucket;
	+ INPBUCKET_LOCK(bucket);
	+ CK_LIST_FOREACH(inp, &bucket->head, inp_hash_wild) {
	inp_lookup_match_t match;
	bool injail;

	@@ -1143,9 +1181,10 @@
	}

	struct inpcb *
	-in6_pcblookup_internal(struct inpcbinfo pcbinfo, const struct in6_addr faddr,
	- u_int fport_arg, const struct in6_addr *laddr, u_int lport_arg,
	- int lookupflags, uint8_t numa_domain, int fib)
	+in6_pcblookup_internal(struct inpcbinfo_ctx *ipictx,
	+ const struct in6_addr *faddr, u_int fport_arg,
	+ const struct in6_addr *laddr, u_int lport_arg, int lookupflags,
	+ uint8_t numa_domain, int fib)
	{
	struct inpcb *inp;
	u_short fport = fport_arg, lport = lport_arg;
	@@ -1156,18 +1195,21 @@
	("%s: invalid foreign address", __func__));
	KASSERT(!IN6_IS_ADDR_UNSPECIFIED(laddr),
	("%s: invalid local address", __func__));
	- INP_HASH_LOCK_ASSERT(pcbinfo);

	- inp = in6_pcblookup_exact(pcbinfo, faddr, fport, laddr, lport);
	+ inp = in6_pcblookup_exact(ipictx->pcbinfo, faddr, fport, laddr, lport,
	+ &ipictx->ebucket);
	if (inp != NULL)
	return (inp);

	if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
	- inp = in6_pcblookup_lbgroup(pcbinfo, faddr, fport, laddr,
	- lport, numa_domain, fib);
	+ inp = in6_pcblookup_lbgroup(ipictx->pcbinfo, faddr, fport,
	+ laddr, lport, numa_domain, fib, &ipictx->lbbucket);
	if (inp == NULL) {
	- inp = in6_pcblookup_wild_locked(pcbinfo, laddr, lport,
	- fib);
	+ /* See comment in in_pcblookup_internal(). */
	+ INPBUCKET_UNLOCK(ipictx->lbbucket);
	+ ipictx->lbbucket = NULL;
	+ inp = in6_pcblookup_wild_locked(ipictx->pcbinfo, laddr,
	+ lport, fib, &ipictx->wbucket);
	}
	}
	return (inp);
	@@ -1178,25 +1220,26 @@
	u_int fport, const struct in6_addr *laddr, u_int lport, int lookupflags,
	uint8_t numa_domain, int fib)
	{
	+ struct inpcbinfo_ctx ipictx = {
	+ .pcbinfo = pcbinfo
	+ };
	struct inpcb *inp;
	const inp_lookup_t lockflags = lookupflags & INPLOOKUP_LOCKMASK;

	KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB \| INPLOOKUP_WLOCKPCB)) != 0,
	("%s: LOCKPCB not set", __func__));

	- INP_HASH_WLOCK(pcbinfo);
	- inp = in6_pcblookup_internal(pcbinfo, faddr, fport, laddr, lport,
	+ inp = in6_pcblookup_internal(&ipictx, faddr, fport, laddr, lport,
	lookupflags & ~INPLOOKUP_LOCKMASK, numa_domain, fib);
	if (inp != NULL && !inp_trylock(inp, lockflags)) {
	in_pcbref(inp);
	- INP_HASH_WUNLOCK(pcbinfo);
	+ inpcbinfo_ctx_release(&ipictx);
	inp_lock(inp, lockflags);
	if (in_pcbrele(inp, lockflags))
	/* XXX-MJ or retry until we get a negative match? */
	inp = NULL;
	- } else {
	- INP_HASH_WUNLOCK(pcbinfo);
	- }
	+ } else
	+ inpcbinfo_ctx_release(&ipictx);
	return (inp);
	}

	@@ -1215,7 +1258,7 @@
	("%s: LOCKPCB not set", __func__));

	smr_enter(pcbinfo->ipi_smr);
	- inp = in6_pcblookup_exact(pcbinfo, faddr, fport, laddr, lport);
	+ inp = in6_pcblookup_exact(pcbinfo, faddr, fport, laddr, lport, NULL);
	if (inp != NULL) {
	if (__predict_true(inp_smr_lock(inp, lockflags))) {
	if (__predict_true(in6_pcblookup_exact_match(inp,
	@@ -1233,7 +1276,7 @@

	if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
	inp = in6_pcblookup_lbgroup(pcbinfo, faddr, fport,
	- laddr, lport, numa_domain, fib);
	+ laddr, lport, numa_domain, fib, NULL);
	if (inp != NULL) {
	if (__predict_true(inp_smr_lock(inp, lockflags))) {
	if (__predict_true(in6_pcblookup_wild_match(inp,

File Metadata

Mime Type: text/plain
Expires: Sun, Jul 12, 6:53 PM (3 h, 58 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 35007369
Default Alt Text: D58131.diff (74 KB)

D58131.diffNo OneTemporaryActions

D58131.diffView Options

File Metadata

Event Timeline

D58131.diff
No OneTemporary
Actions

D58131.diff
View Options