Differential D6689 Diff 18953 head/sys/netinet/tcp_lro.c

Changeset View

Standalone View

head/sys/netinet/tcp_lro.c

Show First 20 Lines • Show All 62 Lines • ▼ Show 20 Lines
static MALLOC_DEFINE(M_LRO, "LRO", "LRO control structures");		static MALLOC_DEFINE(M_LRO, "LRO", "LRO control structures");

#define TCP_LRO_UPDATE_CSUM 1		#define TCP_LRO_UPDATE_CSUM 1
#ifndef TCP_LRO_UPDATE_CSUM		#ifndef TCP_LRO_UPDATE_CSUM
#define TCP_LRO_INVALID_CSUM 0x0000		#define TCP_LRO_INVALID_CSUM 0x0000
#endif		#endif

static void tcp_lro_rx_done(struct lro_ctrl *lc);		static void tcp_lro_rx_done(struct lro_ctrl *lc);
		static int tcp_lro_rx2(struct lro_ctrl lc, struct mbuf m,
		uint32_t csum, int use_hash);

static __inline void		static __inline void
tcp_lro_active_insert(struct lro_ctrl lc, struct lro_entry le)		tcp_lro_active_insert(struct lro_ctrl lc, struct lro_head bucket,
		struct lro_entry *le)
{		{

LIST_INSERT_HEAD(&lc->lro_active, le, next);		LIST_INSERT_HEAD(&lc->lro_active, le, next);
		LIST_INSERT_HEAD(bucket, le, hash_next);
}		}

static __inline void		static __inline void
tcp_lro_active_remove(struct lro_entry *le)		tcp_lro_active_remove(struct lro_entry *le)
{		{

LIST_REMOVE(le, next);		LIST_REMOVE(le, next); /* active list */
		LIST_REMOVE(le, hash_next); /* hash bucket */
}		}

int		int
tcp_lro_init(struct lro_ctrl *lc)		tcp_lro_init(struct lro_ctrl *lc)
{		{
return (tcp_lro_init_args(lc, NULL, TCP_LRO_ENTRIES, 0));		return (tcp_lro_init_args(lc, NULL, TCP_LRO_ENTRIES, 0));
}		}

int		int
tcp_lro_init_args(struct lro_ctrl lc, struct ifnet ifp,		tcp_lro_init_args(struct lro_ctrl lc, struct ifnet ifp,
unsigned lro_entries, unsigned lro_mbufs)		unsigned lro_entries, unsigned lro_mbufs)
{		{
struct lro_entry *le;		struct lro_entry *le;
size_t size;		size_t size;
unsigned i;		unsigned i, elements;

lc->lro_bad_csum = 0;		lc->lro_bad_csum = 0;
lc->lro_queued = 0;		lc->lro_queued = 0;
lc->lro_flushed = 0;		lc->lro_flushed = 0;
lc->lro_cnt = 0;		lc->lro_cnt = 0;
lc->lro_mbuf_count = 0;		lc->lro_mbuf_count = 0;
lc->lro_mbuf_max = lro_mbufs;		lc->lro_mbuf_max = lro_mbufs;
lc->lro_cnt = lro_entries;		lc->lro_cnt = lro_entries;
lc->lro_ackcnt_lim = TCP_LRO_ACKCNT_MAX;		lc->lro_ackcnt_lim = TCP_LRO_ACKCNT_MAX;
lc->lro_length_lim = TCP_LRO_LENGTH_MAX;		lc->lro_length_lim = TCP_LRO_LENGTH_MAX;
lc->ifp = ifp;		lc->ifp = ifp;
LIST_INIT(&lc->lro_free);		LIST_INIT(&lc->lro_free);
LIST_INIT(&lc->lro_active);		LIST_INIT(&lc->lro_active);

		/* create hash table to accelerate entry lookup */
		if (lro_entries > lro_mbufs)
		elements = lro_entries;
		else
		elements = lro_mbufs;
		lc->lro_hash = phashinit_flags(elements, M_LRO, &lc->lro_hashsz,
		HASH_NOWAIT);
		if (lc->lro_hash == NULL) {
		memset(lc, 0, sizeof(*lc));
		return (ENOMEM);
		}

/* compute size to allocate */		/* compute size to allocate */
size = (lro_mbufs * sizeof(struct lro_mbuf_sort)) +		size = (lro_mbufs * sizeof(struct lro_mbuf_sort)) +
(lro_entries * sizeof(*le));		(lro_entries * sizeof(*le));
lc->lro_mbuf_data = (struct lro_mbuf_sort *)		lc->lro_mbuf_data = (struct lro_mbuf_sort *)
malloc(size, M_LRO, M_NOWAIT \| M_ZERO);		malloc(size, M_LRO, M_NOWAIT \| M_ZERO);

/* check for out of memory */		/* check for out of memory */
if (lc->lro_mbuf_data == NULL) {		if (lc->lro_mbuf_data == NULL) {
Show All 21 Lines	tcp_lro_free(struct lro_ctrl *lc)
LIST_INIT(&lc->lro_free);		LIST_INIT(&lc->lro_free);

/* free active mbufs, if any */		/* free active mbufs, if any */
while ((le = LIST_FIRST(&lc->lro_active)) != NULL) {		while ((le = LIST_FIRST(&lc->lro_active)) != NULL) {
tcp_lro_active_remove(le);		tcp_lro_active_remove(le);
m_freem(le->m_head);		m_freem(le->m_head);
}		}

		/* free hash table */
		if (lc->lro_hash != NULL) {
		free(lc->lro_hash, M_LRO);
		lc->lro_hash = NULL;
		}
		lc->lro_hashsz = 0;

/* free mbuf array, if any */		/* free mbuf array, if any */
for (x = 0; x != lc->lro_mbuf_count; x++)		for (x = 0; x != lc->lro_mbuf_count; x++)
m_freem(lc->lro_mbuf_data[x].mb);		m_freem(lc->lro_mbuf_data[x].mb);
lc->lro_mbuf_count = 0;		lc->lro_mbuf_count = 0;

/* free allocated memory, if any */		/* free allocated memory, if any */
free(lc->lro_mbuf_data, M_LRO);		free(lc->lro_mbuf_data, M_LRO);
lc->lro_mbuf_data = NULL;		lc->lro_mbuf_data = NULL;
▲ Show 20 Lines • Show All 324 Lines • ▼ Show 20 Lines	for (x = 0; x != lc->lro_mbuf_count; x++) {
if (seq != nseq) {		if (seq != nseq) {
seq = nseq;		seq = nseq;

/* flush active streams */		/* flush active streams */
tcp_lro_rx_done(lc);		tcp_lro_rx_done(lc);
}		}

/* add packet to LRO engine */		/* add packet to LRO engine */
if (tcp_lro_rx(lc, mb, 0) != 0) {		if (tcp_lro_rx2(lc, mb, 0, 0) != 0) {
/* input packet to network layer */		/* input packet to network layer */
(*lc->ifp->if_input)(lc->ifp, mb);		(*lc->ifp->if_input)(lc->ifp, mb);
lc->lro_queued++;		lc->lro_queued++;
lc->lro_flushed++;		lc->lro_flushed++;
}		}
}		}
done:		done:
/* flush active streams */		/* flush active streams */
▲ Show 20 Lines • Show All 57 Lines • ▼ Show 20 Lines	tcp_lro_rx_ipv4(struct lro_ctrl lc, struct mbuf m, struct ip *ip4,

/* Find the TCP header (we assured there are no IP options). */		/* Find the TCP header (we assured there are no IP options). */
th = (struct tcphdr )(ip4 + 1);		th = (struct tcphdr )(ip4 + 1);

return (0);		return (0);
}		}
#endif		#endif

int		static int
tcp_lro_rx(struct lro_ctrl lc, struct mbuf m, uint32_t csum)		tcp_lro_rx2(struct lro_ctrl lc, struct mbuf m, uint32_t csum, int use_hash)
{		{
struct lro_entry *le;		struct lro_entry *le;
struct ether_header *eh;		struct ether_header *eh;
#ifdef INET6		#ifdef INET6
struct ip6_hdr ip6 = NULL; / Keep compiler happy. */		struct ip6_hdr ip6 = NULL; / Keep compiler happy. */
#endif		#endif
#ifdef INET		#ifdef INET
struct ip ip4 = NULL; / Keep compiler happy. */		struct ip ip4 = NULL; / Keep compiler happy. */
#endif		#endif
struct tcphdr *th;		struct tcphdr *th;
void l3hdr = NULL; / Keep compiler happy. */		void l3hdr = NULL; / Keep compiler happy. */
uint32_t *ts_ptr;		uint32_t *ts_ptr;
tcp_seq seq;		tcp_seq seq;
int error, ip_len, l;		int error, ip_len, l;
uint16_t eh_type, tcp_data_len;		uint16_t eh_type, tcp_data_len;
		struct lro_head *bucket;

/* We expect a contiguous header [eh, ip, tcp]. */		/* We expect a contiguous header [eh, ip, tcp]. */

eh = mtod(m, struct ether_header *);		eh = mtod(m, struct ether_header *);
eh_type = ntohs(eh->ether_type);		eh_type = ntohs(eh->ether_type);
switch (eh_type) {		switch (eh_type) {
#ifdef INET6		#ifdef INET6
case ETHERTYPE_IPV6:		case ETHERTYPE_IPV6:
▲ Show 20 Lines • Show All 76 Lines • ▼ Show 20 Lines	if (l != 0 && (__predict_false(l != TCPOLEN_TSTAMP_APPA) \|\|
return (TCP_LRO_CANNOT);		return (TCP_LRO_CANNOT);

/* If the driver did not pass in the checksum, set it now. */		/* If the driver did not pass in the checksum, set it now. */
if (csum == 0x0000)		if (csum == 0x0000)
csum = th->th_sum;		csum = th->th_sum;

seq = ntohl(th->th_seq);		seq = ntohl(th->th_seq);

		if (!use_hash) {
		bucket = &lc->lro_hash[0];
		} else if (M_HASHTYPE_ISHASH(m)) {
		bucket = &lc->lro_hash[m->m_pkthdr.flowid % lc->lro_hashsz];
		} else {
		uint32_t hash;

		switch (eh_type) {
		#ifdef INET
		case ETHERTYPE_IP:
		hash = ip4->ip_src.s_addr + ip4->ip_dst.s_addr;
		break;
		#endif
		#ifdef INET6
		case ETHERTYPE_IPV6:
		hash = ip6->ip6_src.s6_addr32[0] +
		ip6->ip6_dst.s6_addr32[0];
		hash += ip6->ip6_src.s6_addr32[1] +
		ip6->ip6_dst.s6_addr32[1];
		hash += ip6->ip6_src.s6_addr32[2] +
		ip6->ip6_dst.s6_addr32[2];
		hash += ip6->ip6_src.s6_addr32[3] +
		ip6->ip6_dst.s6_addr32[3];
		break;
		#endif
		default:
		hash = 0;
		break;
		}
		hash += th->th_sport + th->th_dport;
		bucket = &lc->lro_hash[hash % lc->lro_hashsz];
		}

/* Try to find a matching previous segment. */		/* Try to find a matching previous segment. */
LIST_FOREACH(le, &lc->lro_active, next) {		LIST_FOREACH(le, bucket, hash_next) {
if (le->eh_type != eh_type)		if (le->eh_type != eh_type)
continue;		continue;
if (le->source_port != th->th_sport \|\|		if (le->source_port != th->th_sport \|\|
le->dest_port != th->th_dport)		le->dest_port != th->th_dport)
continue;		continue;
switch (eh_type) {		switch (eh_type) {
#ifdef INET6		#ifdef INET6
case ETHERTYPE_IPV6:		case ETHERTYPE_IPV6:
▲ Show 20 Lines • Show All 91 Lines • ▼ Show 20 Lines	#endif

/* Try to find an empty slot. */		/* Try to find an empty slot. */
if (LIST_EMPTY(&lc->lro_free))		if (LIST_EMPTY(&lc->lro_free))
return (TCP_LRO_NO_ENTRIES);		return (TCP_LRO_NO_ENTRIES);

/* Start a new segment chain. */		/* Start a new segment chain. */
le = LIST_FIRST(&lc->lro_free);		le = LIST_FIRST(&lc->lro_free);
LIST_REMOVE(le, next);		LIST_REMOVE(le, next);
tcp_lro_active_insert(lc, le);		tcp_lro_active_insert(lc, bucket, le);
getmicrotime(&le->mtime);		getmicrotime(&le->mtime);

/* Start filling in details. */		/* Start filling in details. */
switch (eh_type) {		switch (eh_type) {
#ifdef INET6		#ifdef INET6
case ETHERTYPE_IPV6:		case ETHERTYPE_IPV6:
le->le_ip6 = ip6;		le->le_ip6 = ip6;
le->source_ip6 = ip6->ip6_src;		le->source_ip6 = ip6->ip6_src;
Show All 39 Lines	le->ulp_csum = tcp_lro_rx_csum_fixup(le, l3hdr, th, tcp_data_len,
~csum);		~csum);
th->th_sum = csum; /* Restore checksum on first packet. */		th->th_sum = csum; /* Restore checksum on first packet. */
#endif		#endif

le->m_head = m;		le->m_head = m;
le->m_tail = m_last(m);		le->m_tail = m_last(m);

return (0);		return (0);
		}

		int
		tcp_lro_rx(struct lro_ctrl lc, struct mbuf m, uint32_t csum)
		{

		return tcp_lro_rx2(lc, m, csum, 1);
}		}

void		void
tcp_lro_queue_mbuf(struct lro_ctrl lc, struct mbuf mb)		tcp_lro_queue_mbuf(struct lro_ctrl lc, struct mbuf mb)
{		{
/* sanity checks */		/* sanity checks */
if (__predict_false(lc->ifp == NULL \|\| lc->lro_mbuf_data == NULL \|\|		if (__predict_false(lc->ifp == NULL \|\| lc->lro_mbuf_data == NULL \|\|
lc->lro_mbuf_max == 0)) {		lc->lro_mbuf_max == 0)) {
Show All 31 Lines