diff --git a/sys/netinet/tcp_hpts.c b/sys/netinet/tcp_hpts.c --- a/sys/netinet/tcp_hpts.c +++ b/sys/netinet/tcp_hpts.c @@ -92,9 +92,8 @@ * * There is a common functions within the rack_bbr_common code * version i.e. ctf_do_queued_segments(). This function - * knows how to take the input queue of packets from - * tp->t_in_pkts and process them digging out - * all the arguments, calling any bpf tap and + * knows how to take the input queue of packets from tp->t_inqueue + * and process them digging out * all the arguments, calling any bpf tap and * calling into tfb_do_segment_nounlock(). The common * function (ctf_do_queued_segments()) requires that * you have defined the tfb_do_segment_nounlock() as @@ -1331,7 +1330,8 @@ kern_prefetch(tp->t_fb_ptr, &did_prefetch); did_prefetch = 1; } - if ((inp->inp_flags2 & INP_SUPPORTS_MBUFQ) && tp->t_in_pkt) { + if ((inp->inp_flags2 & INP_SUPPORTS_MBUFQ) && + !STAILQ_EMPTY(&tp->t_inqueue)) { error = (*tp->t_fb->tfb_do_queued_segments)(tp, 0); if (error) { /* The input killed the connection */ diff --git a/sys/netinet/tcp_lro.c b/sys/netinet/tcp_lro.c --- a/sys/netinet/tcp_lro.c +++ b/sys/netinet/tcp_lro.c @@ -1179,18 +1179,14 @@ #ifdef TCPHPTS static void -tcp_queue_pkts(struct inpcb *inp, struct tcpcb *tp, struct lro_entry *le) +tcp_queue_pkts(struct tcpcb *tp, struct lro_entry *le) { - INP_WLOCK_ASSERT(inp); - if (tp->t_in_pkt == NULL) { - /* Nothing yet there */ - tp->t_in_pkt = le->m_head; - tp->t_tail_pkt = le->m_last_mbuf; - } else { - /* Already some there */ - tp->t_tail_pkt->m_nextpkt = le->m_head; - tp->t_tail_pkt = le->m_last_mbuf; - } + + INP_WLOCK_ASSERT(tptoinpcb(tp)); + + STAILQ_HEAD(, mbuf) q = { le->m_head, + &STAILQ_NEXT(le->m_last_mbuf, m_stailqpkt) }; + STAILQ_CONCAT(&tp->t_inqueue, &q); le->m_head = NULL; le->m_last_mbuf = NULL; } @@ -1221,7 +1217,7 @@ /* Look at the last mbuf if any in queue */ if (can_append_old_cmp) { - m = tp->t_tail_pkt; + m = STAILQ_LAST(&tp->t_inqueue, mbuf, m_stailqpkt); if (m != NULL && (m->m_flags & M_ACKCMP) != 0) { if (M_TRAILINGSPACE(m) >= sizeof(struct tcp_ackent)) { tcp_lro_log(tp, lc, le, NULL, 23, 0, 0, 0, 0); @@ -1451,7 +1447,7 @@ if (le->m_head != NULL) { counter_u64_add(tcp_inp_lro_direct_queue, 1); tcp_lro_log(tp, lc, le, NULL, 22, 1, inp->inp_flags2, 0, 1); - tcp_queue_pkts(inp, tp, le); + tcp_queue_pkts(tp, le); } if (should_wake) { /* Wakeup */ diff --git a/sys/netinet/tcp_stacks/bbr.c b/sys/netinet/tcp_stacks/bbr.c --- a/sys/netinet/tcp_stacks/bbr.c +++ b/sys/netinet/tcp_stacks/bbr.c @@ -11597,7 +11597,7 @@ int retval; /* First lets see if we have old packets */ - if (tp->t_in_pkt) { + if (!STAILQ_EMPTY(&tp->t_inqueue)) { if (ctf_do_queued_segments(tp, 1)) { m_freem(m); return; diff --git a/sys/netinet/tcp_stacks/rack.c b/sys/netinet/tcp_stacks/rack.c --- a/sys/netinet/tcp_stacks/rack.c +++ b/sys/netinet/tcp_stacks/rack.c @@ -17065,7 +17065,7 @@ struct timeval tv; /* First lets see if we have old packets */ - if (tp->t_in_pkt) { + if (!STAILQ_EMPTY(&tp->t_inqueue)) { if (ctf_do_queued_segments(tp, 1)) { m_freem(m); return; diff --git a/sys/netinet/tcp_stacks/rack_bbr_common.c b/sys/netinet/tcp_stacks/rack_bbr_common.c --- a/sys/netinet/tcp_stacks/rack_bbr_common.c +++ b/sys/netinet/tcp_stacks/rack_bbr_common.c @@ -493,10 +493,8 @@ struct mbuf *m; /* First lets see if we have old packets */ - if (tp->t_in_pkt) { - m = tp->t_in_pkt; - tp->t_in_pkt = NULL; - tp->t_tail_pkt = NULL; + if ((m = STAILQ_FIRST(&tp->t_inqueue)) != NULL) { + STAILQ_INIT(&tp->t_inqueue); if (ctf_process_inbound_raw(tp, m, have_pkt)) { /* We lost the tcpcb (maybe a RST came in)? */ return(1); diff --git a/sys/netinet/tcp_subr.c b/sys/netinet/tcp_subr.c --- a/sys/netinet/tcp_subr.c +++ b/sys/netinet/tcp_subr.c @@ -2243,6 +2243,7 @@ #endif TAILQ_INIT(&tp->t_segq); + STAILQ_INIT(&tp->t_inqueue); tp->t_maxseg = #ifdef INET6 isipv6 ? V_tcp_v6mssdflt : @@ -2404,8 +2405,10 @@ tcp_log_tcpcbfini(tp); #endif TCPSTATES_DEC(tp->t_state); + if (tp->t_fb->tfb_tcp_fb_fini) (*tp->t_fb->tfb_tcp_fb_fini)(tp, 1); + MPASS(STAILQ_EMPTY(&tp->t_inqueue)); /* * If we got enough samples through the srtt filter, @@ -4179,7 +4182,7 @@ void tcp_handle_orphaned_packets(struct tcpcb *tp) { - struct mbuf *save, *m, *prev; + struct mbuf *m, *save, *prev; /* * Called when a stack switch is occuring from the fini() * of the old stack. We assue the init() as already been @@ -4191,7 +4194,8 @@ if (tptoinpcb(tp)->inp_flags2 & INP_MBUF_L_ACKS) return; - if ((tptoinpcb(tp)->inp_flags2 & INP_SUPPORTS_MBUFQ) == 0) { + if ((tptoinpcb(tp)->inp_flags2 & INP_SUPPORTS_MBUFQ) == 0 && + !STAILQ_EMPTY(&tp->t_inqueue)) { /* * It is unsafe to process the packets since a * reset may be lurking in them (its rare but it @@ -4202,44 +4206,27 @@ * This new stack does not do any fancy LRO features * so all we can do is toss the packets. */ - m = tp->t_in_pkt; - tp->t_in_pkt = NULL; - tp->t_tail_pkt = NULL; - while (m) { - save = m->m_nextpkt; - m->m_nextpkt = NULL; + m = STAILQ_FIRST(&tp->t_inqueue); + STAILQ_INIT(&tp->t_inqueue); + STAILQ_FOREACH_FROM_SAFE(m, &tp->t_inqueue, m_stailqpkt, save) m_freem(m); - m = save; - } } else { /* * Here we have a stack that does mbuf queuing but * does not support compressed ack's. We must * walk all the mbufs and discard any compressed acks. */ - m = tp->t_in_pkt; - prev = NULL; - while (m) { + STAILQ_FOREACH_SAFE(m, &tp->t_inqueue, m_stailqpkt, save) { if (m->m_flags & M_ACKCMP) { - /* We must toss this packet */ - if (tp->t_tail_pkt == m) - tp->t_tail_pkt = prev; - if (prev) - prev->m_nextpkt = m->m_nextpkt; + if (m == STAILQ_FIRST(&tp->t_inqueue)) + STAILQ_REMOVE_HEAD(&tp->t_inqueue, + m_stailqpkt); else - tp->t_in_pkt = m->m_nextpkt; - m->m_nextpkt = NULL; + STAILQ_REMOVE_AFTER(&tp->t_inqueue, + prev, m_stailqpkt); m_freem(m); - /* move forward */ - if (prev) - m = prev->m_nextpkt; - else - m = tp->t_in_pkt; - } else { - /* this one is ok */ + } else prev = m; - m = m->m_nextpkt; - } } } } diff --git a/sys/netinet/tcp_var.h b/sys/netinet/tcp_var.h --- a/sys/netinet/tcp_var.h +++ b/sys/netinet/tcp_var.h @@ -355,8 +355,7 @@ int t_segqlen; /* segment reassembly queue length */ uint32_t t_segqmbuflen; /* total reassembly queue byte length */ struct tsegqe_head t_segq; /* segment reassembly queue */ - struct mbuf *t_in_pkt; - struct mbuf *t_tail_pkt; + STAILQ_HEAD(, mbuf) t_inqueue; /* HPTS input queue */ uint32_t snd_ssthresh; /* snd_cwnd size threshold for * for slow start exponential to * linear switch