Index: sys/netinet/tcp_lro.h =================================================================== --- sys/netinet/tcp_lro.h +++ sys/netinet/tcp_lro.h @@ -40,6 +40,7 @@ struct lro_entry { LIST_ENTRY(lro_entry) next; + LIST_ENTRY(lro_entry) hash_next; struct mbuf *m_head; struct mbuf *m_tail; union { @@ -95,6 +96,8 @@ unsigned short lro_ackcnt_lim; /* max # of aggregated ACKs */ unsigned lro_length_lim; /* max len of aggregated data */ + u_long lro_hashsz; + struct lro_head *lro_hash; struct lro_head lro_active; struct lro_head lro_free; }; Index: sys/netinet/tcp_lro.c =================================================================== --- sys/netinet/tcp_lro.c +++ sys/netinet/tcp_lro.c @@ -70,17 +70,20 @@ static void tcp_lro_rx_done(struct lro_ctrl *lc); static __inline void -tcp_lro_active_insert(struct lro_ctrl *lc, struct lro_entry *le) +tcp_lro_active_insert(struct lro_ctrl *lc, struct lro_head *bucket, + struct lro_entry *le) { LIST_INSERT_HEAD(&lc->lro_active, le, next); + LIST_INSERT_HEAD(bucket, le, hash_next); } static __inline void tcp_lro_active_remove(struct lro_entry *le) { - LIST_REMOVE(le, next); + LIST_REMOVE(le, next); /* active list */ + LIST_REMOVE(le, hash_next); /* hash bucket */ } int @@ -95,7 +98,7 @@ { struct lro_entry *le; size_t size; - unsigned i; + unsigned i, elements; lc->lro_bad_csum = 0; lc->lro_queued = 0; @@ -110,6 +113,18 @@ LIST_INIT(&lc->lro_free); LIST_INIT(&lc->lro_active); + /* create hash table to accelerate entry lookup */ + if (lro_entries > lro_mbufs) + elements = lro_entries; + else + elements = lro_mbufs; + lc->lro_hash = phashinit_flags(elements, M_LRO, &lc->lro_hashsz, + HASH_NOWAIT); + if (lc->lro_hash == NULL) { + memset(lc, 0, sizeof(*lc)); + return (ENOMEM); + } + /* compute size to allocate */ size = (lro_mbufs * sizeof(struct lro_mbuf_sort)) + (lro_entries * sizeof(*le)); @@ -147,6 +162,13 @@ m_freem(le->m_head); } + /* free hash table */ + if (lc->lro_hash != NULL) { + free(lc->lro_hash, M_LRO); + lc->lro_hash = NULL; + } + lc->lro_hashsz = 0; + /* free mbuf array, if any */ for (x = 0; x != lc->lro_mbuf_count; x++) m_freem(lc->lro_mbuf_data[x].mb); @@ -581,6 +603,8 @@ tcp_seq seq; int error, ip_len, l; uint16_t eh_type, tcp_data_len; + struct lro_head *bucket; + uint32_t hash; /* We expect a contiguous header [eh, ip, tcp]. */ @@ -673,8 +697,37 @@ seq = ntohl(th->th_seq); + if (M_HASHTYPE_ISHASH(m)) { + hash = m->m_pkthdr.flowid; + } else { + switch (eh_type) { +#ifdef INET + case ETHERTYPE_IP: + hash = ip4->ip_src.s_addr + ip4->ip_dst.s_addr; + break; +#endif +#ifdef INET6 + case ETHERTYPE_IPV6: + hash = ip6->ip6_src.s6_addr32[0] + + ip6->ip6_dst.s6_addr32[0]; + hash += ip6->ip6_src.s6_addr32[1] + + ip6->ip6_dst.s6_addr32[1]; + hash += ip6->ip6_src.s6_addr32[2] + + ip6->ip6_dst.s6_addr32[2]; + hash += ip6->ip6_src.s6_addr32[3] + + ip6->ip6_dst.s6_addr32[3]; + break; +#endif + default: + hash = 0; + break; + } + hash += th->th_sport + th->th_dport; + } + bucket = &lc->lro_hash[hash % lc->lro_hashsz]; + /* Try to find a matching previous segment. */ - LIST_FOREACH(le, &lc->lro_active, next) { + LIST_FOREACH(le, bucket, hash_next) { if (le->eh_type != eh_type) continue; if (le->source_port != th->th_sport || @@ -782,7 +835,7 @@ /* Start a new segment chain. */ le = LIST_FIRST(&lc->lro_free); LIST_REMOVE(le, next); - tcp_lro_active_insert(lc, le); + tcp_lro_active_insert(lc, bucket, le); getmicrotime(&le->mtime); /* Start filling in details. */