Index: sys/netinet/in_pcb.h =================================================================== --- sys/netinet/in_pcb.h +++ sys/netinet/in_pcb.h @@ -645,6 +645,7 @@ #define INP_INFO_WUNLOCK_ASSERT(ipi) \ mtx_assert(&(ipi)->ipi_lock, MA_NOTOWNED) #define INP_INFO_UNLOCK_ASSERT(ipi) MPASS(!in_epoch(net_epoch_preempt) && !mtx_owned(&(ipi)->ipi_lock)) +#define INP_INFO_WAIT(ipi) NET_EPOCH_WAIT() #define INP_LIST_LOCK_INIT(ipi, d) \ rw_init_flags(&(ipi)->ipi_list_lock, (d), 0) Index: sys/netinet/tcp_subr.c =================================================================== --- sys/netinet/tcp_subr.c +++ sys/netinet/tcp_subr.c @@ -942,6 +942,7 @@ if (force && blk->tfb_refcnt) { struct inpcb *inp; struct tcpcb *tp; + struct epoch_tracker et; VNET_ITERATOR_DECL(vnet_iter); rw_wunlock(&tcp_function_lock); @@ -949,7 +950,15 @@ VNET_LIST_RLOCK(); VNET_FOREACH(vnet_iter) { CURVNET_SET(vnet_iter); - INP_INFO_WLOCK(&V_tcbinfo); + + /* + * It is possible that another thread retrieved + * the stack before our change. We need to wait + * for all of them to complete their work before + * scanning the INP list for matching stacks. + */ + INP_INFO_WAIT(&V_tcbinfo); + INP_INFO_RLOCK_ET(&V_tcbinfo, et); CK_LIST_FOREACH(inp, V_tcbinfo.ipi_listhead, inp_list) { INP_WLOCK(inp); if (inp->inp_flags & INP_TIMEWAIT) { @@ -964,7 +973,7 @@ tcp_switch_back_to_default(tp); INP_WUNLOCK(inp); } - INP_INFO_WUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); CURVNET_RESTORE(); } VNET_LIST_RUNLOCK(); @@ -1703,6 +1712,7 @@ struct cc_algo *tmpalgo; struct inpcb *inp; struct tcpcb *tp; + struct epoch_tracker et; VNET_ITERATOR_DECL(vnet_iter); /* @@ -1713,14 +1723,15 @@ VNET_LIST_RLOCK(); VNET_FOREACH(vnet_iter) { CURVNET_SET(vnet_iter); - INP_INFO_WLOCK(&V_tcbinfo); /* * New connections already part way through being initialised - * with the CC algo we're removing will not race with this code - * because the INP_INFO_WLOCK is held during initialisation. We - * therefore don't enter the loop below until the connection - * list has stabilised. + * with the CC algo we're removing may race with this code. + * Therefore, we need to wait for them to complete their + * work before scanning the connection list for connections + * using the CC algorithm. */ + INP_INFO_WAIT(&V_tcbinfo); + INP_INFO_RLOCK_ET(&V_tcbinfo, et); CK_LIST_FOREACH(inp, &V_tcb, inp_list) { INP_WLOCK(inp); /* Important to skip tcptw structs. */ @@ -1751,7 +1762,7 @@ } INP_WUNLOCK(inp); } - INP_INFO_WUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); CURVNET_RESTORE(); } VNET_LIST_RUNLOCK();