Index: sys/netinet/tcp_lro.c =================================================================== --- sys/netinet/tcp_lro.c +++ sys/netinet/tcp_lro.c @@ -80,6 +80,10 @@ size_t size; unsigned i; + /* make "lro_mbufs" power of two */ + if (lro_mbufs != 0 && powerof2(lro_mbufs) == 0) + lro_mbufs = 1U << fls(lro_mbufs); + lc->lro_bad_csum = 0; lc->lro_queued = 0; lc->lro_flushed = 0; @@ -92,7 +96,7 @@ SLIST_INIT(&lc->lro_active); /* compute size to allocate */ - size = (lro_mbufs * sizeof(struct mbuf *)) + + size = (2 * lro_mbufs * sizeof(struct mbuf *)) + (lro_entries * sizeof(*le)); lc->lro_mbuf_data = (struct mbuf **) malloc(size, M_LRO, M_NOWAIT | M_ZERO); @@ -104,7 +108,7 @@ } /* compute offset for LRO entries */ le = (struct lro_entry *) - (lc->lro_mbuf_data + lro_mbufs); + (lc->lro_mbuf_data + 2 * lro_mbufs); /* setup linked list */ for (i = 0; i != lro_entries; i++) @@ -342,6 +346,10 @@ const struct mbuf *mb = *((const struct mbuf * const *)ppb); int ret; + ret = (ma == NULL) - (mb == NULL); + if (ret != 0 || (ma == NULL)) + goto done; + ret = M_HASHTYPE_GET(ma) - M_HASHTYPE_GET(mb); if (ret != 0) goto done; @@ -355,6 +363,65 @@ return (ret); } +/* Optimised mergesort algorithm */ +static void +tcp_lro_mbuf_sort(struct lro_ctrl *lc, unsigned max) +{ + struct mbuf **ptemp = lc->lro_mbuf_data + max; + unsigned x; + unsigned y; + + for (x = 2; x <= max; x *= 2) { + for (y = 0; y != max; y += x) { + unsigned a; + unsigned b; + unsigned c; + int d; + + /* setup indexes */ + a = 0; + b = (x / 2); + c = y; + + /* + * If the two partitions are back to back, + * skip the mergesort: + */ + d = tcp_lro_mbuf_compare_header( + lc->lro_mbuf_data + c + b - 1, + lc->lro_mbuf_data + c + b); + if (d <= 0) + continue; + + /* copy data to temporary buffer */ + memcpy(ptemp, lc->lro_mbuf_data + c, x * sizeof(void *)); + + /* merge sort */ + while (1) { + if (a == (x / 2)) { + memcpy(lc->lro_mbuf_data + c, + ptemp + b, (x - b) * sizeof(void *)); + break; + } else if (b == x) { + memcpy(lc->lro_mbuf_data + c, + ptemp + a, ((x / 2) - a) * sizeof(void *)); + break; + } + + d = tcp_lro_mbuf_compare_header(ptemp + a, ptemp + b); + if (d == 0) { + lc->lro_mbuf_data[c++] = ptemp[a++]; + lc->lro_mbuf_data[c++] = ptemp[b++]; + } else if (d < 0) { + lc->lro_mbuf_data[c++] = ptemp[a++]; + } else { + lc->lro_mbuf_data[c++] = ptemp[b++]; + } + } + } + } +} + void tcp_lro_flush_all(struct lro_ctrl *lc) { @@ -362,14 +429,23 @@ uint32_t hashtype; uint32_t flowid; unsigned x; + unsigned y; /* check if no mbufs to flush */ if (__predict_false(lc->lro_mbuf_count == 0)) goto done; + y = lc->lro_mbuf_count; + if (powerof2(y) == 0) { + y = 1U << fls(y); + + /* zero rest of buffer */ + memset(lc->lro_mbuf_data + lc->lro_mbuf_count, 0, + sizeof(void *) * (y - lc->lro_mbuf_count)); + } + /* sort all mbufs according to stream */ - qsort(lc->lro_mbuf_data, lc->lro_mbuf_count, sizeof(struct mbuf *), - &tcp_lro_mbuf_compare_header); + tcp_lro_mbuf_sort(lc, y); /* input data into LRO engine, stream by stream */ flowid = 0;