Index: sys/dev/al_eth/al_eth.c =================================================================== --- sys/dev/al_eth/al_eth.c +++ sys/dev/al_eth/al_eth.c @@ -1589,6 +1589,7 @@ uint32_t refill_required; uint32_t refill_actual; uint32_t do_if_input; + struct mbuf *m_head, *tail, *tmphead; if (napi != 0) { rx_ring->enqueue_is_running = 1; @@ -1675,10 +1676,15 @@ "%s: not filling rx queue %d\n", __func__, qid); } + m_head = tail = NULL; while (((queued = LIST_FIRST(&rx_ring->lro.lro_active)) != NULL)) { LIST_REMOVE(queued, next); - tcp_lro_flush(&rx_ring->lro, queued); + tmphead = tcp_lro_flush(&rx_ring->lro, queued, &tail); + if (m_head == NULL) + m_head = tmphead; } + if (m_head) + (rx_ring->lro->ifp->if_input)(rx_ring->lro->ifp, m_head); if (napi != 0) { rx_ring->enqueue_is_running = 0; Index: sys/dev/ixl/ixl_txrx.c =================================================================== --- sys/dev/ixl/ixl_txrx.c +++ sys/dev/ixl/ixl_txrx.c @@ -1757,10 +1757,17 @@ tcp_lro_flush_all(lro); #else struct lro_entry *queued; + struct mbuf *m_head, *tail, *tmphead; + + m_head = tail = NULL; while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) { SLIST_REMOVE_HEAD(&lro->lro_active, next); - tcp_lro_flush(lro, queued); + tmphead = tcp_lro_flush(lro, queued, &tail); + if (m_head == NULL) + m_head = tmphead; } + if (m_head) + (lro->ifp->if_input)(lro->ifp, m_head); #endif #endif /* defined(INET6) || defined(INET) */ Index: sys/dev/neta/if_mvneta.c =================================================================== --- sys/dev/neta/if_mvneta.c +++ sys/dev/neta/if_mvneta.c @@ -2982,6 +2982,7 @@ struct lro_entry *queued; void *pktbuf; int i, pktlen, processed, ndma; + struct mbuf *m_head, *tail, *tmphead; KASSERT_RX_MTX(sc, q); @@ -3094,10 +3095,15 @@ * Flush any outstanding LRO work */ lro = &rx->lro; + m_head = tail = NULL; while (__predict_false((queued = LIST_FIRST(&lro->lro_active)) != NULL)) { LIST_REMOVE(LIST_FIRST((&lro->lro_active)), next); - tcp_lro_flush(lro, queued); + tmphead = tcp_lro_flush(lro, queued, &tail); + if (m_head == NULL) + m_head = tmphead; } + if (m_head) + (lro->ifp->if_input)(lro->ifp, m_head); } STATIC void Index: sys/dev/qlnx/qlnxe/qlnx_os.c =================================================================== --- sys/dev/qlnx/qlnxe/qlnx_os.c +++ sys/dev/qlnx/qlnxe/qlnx_os.c @@ -486,12 +486,18 @@ #else struct lro_entry *queued; + struct mbuf *m_head, *tail, *tmphead; + m_head = tail = NULL; while ((!SLIST_EMPTY(&lro->lro_active))) { queued = SLIST_FIRST(&lro->lro_active); SLIST_REMOVE_HEAD(&lro->lro_active, next); - tcp_lro_flush(lro, queued); + tmphead = tcp_lro_flush(lro, queued, &tail); + if (m_head == NULL) + m_head = tmphead; } + if (m_head) + (lro->ifp->if_input)(lro->ifp, m_head); #endif /* #if (__FreeBSD_version >= 1100101) || (defined QLNX_QSORT_LRO) */ } } @@ -4681,13 +4687,19 @@ #else struct lro_entry *queued; + struct mbuf *m_head, *tail, *tmphead; + m_head = tail = NULL; while ((!SLIST_EMPTY(&lro->lro_active))) { queued = SLIST_FIRST(&lro->lro_active); SLIST_REMOVE_HEAD(&lro->lro_active, \ next); - tcp_lro_flush(lro, queued); + tmphead = tcp_lro_flush(lro, queued, &tail); + if (m_head == NULL) + m_head = tmphead; } + if (m_head) + (lro->ifp->if_input)(lro->ifp, m_head); #endif /* #if (__FreeBSD_version >= 1100101) || (defined QLNX_QSORT_LRO) */ } } @@ -7052,12 +7064,18 @@ #else struct lro_entry *queued; + struct mbuf *m_head, *tail, *tmphead; + m_head = tail = NULL; while ((!SLIST_EMPTY(&lro->lro_active))){ queued = SLIST_FIRST(&lro->lro_active); SLIST_REMOVE_HEAD(&lro->lro_active, next); - tcp_lro_flush(lro, queued); + tmphead = tcp_lro_flush(lro, queued, &tail); + if (m_head == NULL) + m_head = tmphead; } + if (m_head) + (lro->ifp->if_input)(lro->ifp, m_head); #endif /* #if (__FreeBSD_version >= 1100101) || (defined QLNX_QSORT_LRO) */ Index: sys/dev/qlxgbe/ql_hw.c =================================================================== --- sys/dev/qlxgbe/ql_hw.c +++ sys/dev/qlxgbe/ql_hw.c @@ -2623,12 +2623,18 @@ tcp_lro_flush_all(lro); #else struct lro_entry *queued; + struct mbuf *m_head, *tail, *tmphead; + m_head = tail = NULL; while ((!SLIST_EMPTY(&lro->lro_active))) { queued = SLIST_FIRST(&lro->lro_active); SLIST_REMOVE_HEAD(&lro->lro_active, next); - tcp_lro_flush(lro, queued); + tmphead = tcp_lro_flush(lro, queued, &tail); + if (m_head == NULL) + m_head = tmphead; } + if (m_head) + (lro->ifp->if_input)(lro->ifp, m_head); #endif /* #if (__FreeBSD_version >= 1100101) */ } Index: sys/dev/qlxgbe/ql_isr.c =================================================================== --- sys/dev/qlxgbe/ql_isr.c +++ sys/dev/qlxgbe/ql_isr.c @@ -739,12 +739,18 @@ #else struct lro_entry *queued; + struct mbuf *m_head, *tail, *tmphead; + m_head = tail = NULL; while ((!SLIST_EMPTY(&lro->lro_active))) { queued = SLIST_FIRST(&lro->lro_active); SLIST_REMOVE_HEAD(&lro->lro_active, next); - tcp_lro_flush(lro, queued); + tmphead = tcp_lro_flush(lro, queued, &tail); + if (m_head == NULL) + m_head = tmphead; } + if (m_head) + (lro->ifp->if_input)(lro->ifp, m_head); #endif /* #if (__FreeBSD_version >= 1100101) */ Index: sys/netinet/tcp_lro.h =================================================================== --- sys/netinet/tcp_lro.h +++ sys/netinet/tcp_lro.h @@ -109,7 +109,7 @@ int tcp_lro_init_args(struct lro_ctrl *, struct ifnet *, unsigned, unsigned); void tcp_lro_free(struct lro_ctrl *); void tcp_lro_flush_inactive(struct lro_ctrl *, const struct timeval *); -void tcp_lro_flush(struct lro_ctrl *, struct lro_entry *); +struct mbuf *tcp_lro_flush(struct lro_ctrl *, struct lro_entry *, struct mbuf **tail); void tcp_lro_flush_all(struct lro_ctrl *); int tcp_lro_rx(struct lro_ctrl *, struct mbuf *, uint32_t); void tcp_lro_queue_mbuf(struct lro_ctrl *, struct mbuf *); Index: sys/netinet/tcp_lro.c =================================================================== --- sys/netinet/tcp_lro.c +++ sys/netinet/tcp_lro.c @@ -278,11 +278,17 @@ tcp_lro_rx_done(struct lro_ctrl *lc) { struct lro_entry *le; + struct mbuf *m_head, *tail, *tmphead; + m_head = tail = NULL; while ((le = LIST_FIRST(&lc->lro_active)) != NULL) { tcp_lro_active_remove(le); - tcp_lro_flush(lc, le); + tmphead = tcp_lro_flush(lc, le, &tail); + if (m_head == NULL) + m_head = tmphead; } + if (m_head) + (*lc->ifp->if_input)(lc->ifp, m_head); } void @@ -290,23 +296,30 @@ { struct lro_entry *le, *le_tmp; struct timeval tv; + struct mbuf *m_head, *tail, *tmphead; if (LIST_EMPTY(&lc->lro_active)) return; getmicrotime(&tv); timevalsub(&tv, timeout); + m_head = tail = NULL; LIST_FOREACH_SAFE(le, &lc->lro_active, next, le_tmp) { if (timevalcmp(&tv, &le->mtime, >=)) { tcp_lro_active_remove(le); - tcp_lro_flush(lc, le); + tmphead = tcp_lro_flush(lc, le, &tail); + if (m_head == NULL) + m_head = tmphead; } } + if (m_head) + (*lc->ifp->if_input)(lc->ifp, m_head); } -void -tcp_lro_flush(struct lro_ctrl *lc, struct lro_entry *le) +struct mbuf * +tcp_lro_flush(struct lro_ctrl *lc, struct lro_entry *le, struct mbuf **tail) { + struct mbuf *ret_mbuf; if (le->append_cnt > 0) { struct tcphdr *th; @@ -392,10 +405,16 @@ le->m_head->m_pkthdr.lro_nsegs = le->append_cnt + 1; (*lc->ifp->if_input)(lc->ifp, le->m_head); + if (*tail) + (*tail)->m_next = le->m_head; + *tail = le->m_tail; + ret_mbuf = le->m_head; + lc->lro_queued += le->append_cnt + 1; lc->lro_flushed++; bzero(le, sizeof(*le)); LIST_INSERT_HEAD(&lc->lro_free, le, next); + return ret_mbuf; } #ifdef HAVE_INLINE_FLSLL @@ -613,6 +632,7 @@ uint16_t eh_type, tcp_data_len; struct lro_head *bucket; int force_flush = 0; + struct mbuf *m_head, *tail, *tmphead; /* We expect a contiguous header [eh, ip, tcp]. */ @@ -751,6 +771,7 @@ } /* Try to find a matching previous segment. */ + m_head = tail = NULL; LIST_FOREACH(le, bucket, hash_next) { if (le->eh_type != eh_type) continue; @@ -779,14 +800,20 @@ if (force_flush) { /* Timestamps mismatch; this is a FIN, etc */ tcp_lro_active_remove(le); - tcp_lro_flush(lc, le); + tmphead = tcp_lro_flush(lc, le, &tail); + if (m_head == NULL) + m_head = tmphead; + if (m_head) + (*lc->ifp->if_input)(lc->ifp, m_head); return (TCP_LRO_CANNOT); } /* Flush now if appending will result in overflow. */ if (le->p_len > (lc->lro_length_lim - tcp_data_len)) { tcp_lro_active_remove(le); - tcp_lro_flush(lc, le); + tmphead = tcp_lro_flush(lc, le, &tail); + if (m_head == NULL) + m_head = tmphead; break; } @@ -795,7 +822,11 @@ (tcp_data_len == 0 && le->ack_seq == th->th_ack))) { /* Out of order packet or duplicate ACK. */ tcp_lro_active_remove(le); - tcp_lro_flush(lc, le); + tmphead = tcp_lro_flush(lc, le, &tail); + if (m_head == NULL) + m_head = tmphead; + if (m_head) + (*lc->ifp->if_input)(lc->ifp, m_head); return (TCP_LRO_CANNOT); } @@ -804,8 +835,11 @@ /* Make sure timestamp values are increasing. */ /* XXX-BZ flip and use TSTMP_GEQ macro for this? */ if (__predict_false(le->tsval > tsval || - *(ts_ptr + 2) == 0)) + *(ts_ptr + 2) == 0)) { + if (m_head) + (*lc->ifp->if_input)(lc->ifp, m_head); return (TCP_LRO_CANNOT); + } le->tsval = tsval; le->tsecr = *(ts_ptr + 2); } @@ -828,8 +862,12 @@ */ if (le->append_cnt >= lc->lro_ackcnt_lim) { tcp_lro_active_remove(le); - tcp_lro_flush(lc, le); + tmphead = tcp_lro_flush(lc, le, &tail); + if (m_head == NULL) + m_head = tmphead; } + if (m_head) + (*lc->ifp->if_input)(lc->ifp, m_head); return (0); } @@ -852,10 +890,14 @@ */ if (le->p_len > (lc->lro_length_lim - lc->ifp->if_mtu)) { tcp_lro_active_remove(le); - tcp_lro_flush(lc, le); + tmphead = tcp_lro_flush(lc, le, &tail); + if (m_head == NULL) + m_head = tmphead; } else getmicrotime(&le->mtime); + if (m_head) + (*lc->ifp->if_input)(lc->ifp, m_head); return (0); } @@ -864,12 +906,17 @@ * Nothing to flush, but this segment can not be further * aggregated/delayed. */ + if (m_head) + (*lc->ifp->if_input)(lc->ifp, m_head); return (TCP_LRO_CANNOT); } /* Try to find an empty slot. */ - if (LIST_EMPTY(&lc->lro_free)) + if (LIST_EMPTY(&lc->lro_free)) { + if (m_head) + (*lc->ifp->if_input)(lc->ifp, m_head); return (TCP_LRO_NO_ENTRIES); + } /* Start a new segment chain. */ le = LIST_FIRST(&lc->lro_free); @@ -928,6 +975,8 @@ le->m_head = m; le->m_tail = m_last(m); + if (m_head) + (*lc->ifp->if_input)(lc->ifp, m_head); return (0); }