Changeset View
Changeset View
Standalone View
Standalone View
sys/netinet/tcp_lro.c
Show First 20 Lines • Show All 63 Lines • ▼ Show 20 Lines | |||||
#define TCP_LRO_UPDATE_CSUM 1 | #define TCP_LRO_UPDATE_CSUM 1 | ||||
#ifndef TCP_LRO_UPDATE_CSUM | #ifndef TCP_LRO_UPDATE_CSUM | ||||
#define TCP_LRO_INVALID_CSUM 0x0000 | #define TCP_LRO_INVALID_CSUM 0x0000 | ||||
#endif | #endif | ||||
static void tcp_lro_rx_done(struct lro_ctrl *lc); | static void tcp_lro_rx_done(struct lro_ctrl *lc); | ||||
static __inline void | |||||
tcp_lro_active_insert(struct lro_ctrl *lc, struct lro_entry *le) | |||||
{ | |||||
LIST_INSERT_HEAD(&lc->lro_active, le, next); | |||||
} | |||||
static __inline void | |||||
tcp_lro_active_remove(struct lro_entry *le) | |||||
{ | |||||
LIST_REMOVE(le, next); | |||||
} | |||||
int | int | ||||
tcp_lro_init(struct lro_ctrl *lc) | tcp_lro_init(struct lro_ctrl *lc) | ||||
{ | { | ||||
return (tcp_lro_init_args(lc, NULL, TCP_LRO_ENTRIES, 0)); | return (tcp_lro_init_args(lc, NULL, TCP_LRO_ENTRIES, 0)); | ||||
} | } | ||||
int | int | ||||
tcp_lro_init_args(struct lro_ctrl *lc, struct ifnet *ifp, | tcp_lro_init_args(struct lro_ctrl *lc, struct ifnet *ifp, | ||||
▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines | tcp_lro_free(struct lro_ctrl *lc) | ||||
struct lro_entry *le; | struct lro_entry *le; | ||||
unsigned x; | unsigned x; | ||||
/* reset LRO free list */ | /* reset LRO free list */ | ||||
LIST_INIT(&lc->lro_free); | LIST_INIT(&lc->lro_free); | ||||
/* free active mbufs, if any */ | /* free active mbufs, if any */ | ||||
while ((le = LIST_FIRST(&lc->lro_active)) != NULL) { | while ((le = LIST_FIRST(&lc->lro_active)) != NULL) { | ||||
LIST_REMOVE(le, next); | tcp_lro_active_remove(le); | ||||
m_freem(le->m_head); | m_freem(le->m_head); | ||||
} | } | ||||
/* free mbuf array, if any */ | /* free mbuf array, if any */ | ||||
for (x = 0; x != lc->lro_mbuf_count; x++) | for (x = 0; x != lc->lro_mbuf_count; x++) | ||||
m_freem(lc->lro_mbuf_data[x]); | m_freem(lc->lro_mbuf_data[x]); | ||||
lc->lro_mbuf_count = 0; | lc->lro_mbuf_count = 0; | ||||
▲ Show 20 Lines • Show All 88 Lines • ▼ Show 20 Lines | |||||
#endif | #endif | ||||
static void | static void | ||||
tcp_lro_rx_done(struct lro_ctrl *lc) | tcp_lro_rx_done(struct lro_ctrl *lc) | ||||
{ | { | ||||
struct lro_entry *le; | struct lro_entry *le; | ||||
while ((le = LIST_FIRST(&lc->lro_active)) != NULL) { | while ((le = LIST_FIRST(&lc->lro_active)) != NULL) { | ||||
LIST_REMOVE(le, next); | tcp_lro_active_remove(le); | ||||
tcp_lro_flush(lc, le); | tcp_lro_flush(lc, le); | ||||
} | } | ||||
} | } | ||||
void | void | ||||
tcp_lro_flush_inactive(struct lro_ctrl *lc, const struct timeval *timeout) | tcp_lro_flush_inactive(struct lro_ctrl *lc, const struct timeval *timeout) | ||||
{ | { | ||||
struct lro_entry *le, *le_tmp; | struct lro_entry *le, *le_tmp; | ||||
struct timeval tv; | struct timeval tv; | ||||
if (LIST_EMPTY(&lc->lro_active)) | if (LIST_EMPTY(&lc->lro_active)) | ||||
return; | return; | ||||
getmicrotime(&tv); | getmicrotime(&tv); | ||||
timevalsub(&tv, timeout); | timevalsub(&tv, timeout); | ||||
LIST_FOREACH_SAFE(le, &lc->lro_active, next, le_tmp) { | LIST_FOREACH_SAFE(le, &lc->lro_active, next, le_tmp) { | ||||
if (timevalcmp(&tv, &le->mtime, >=)) { | if (timevalcmp(&tv, &le->mtime, >=)) { | ||||
LIST_REMOVE(le, next); | tcp_lro_active_remove(le); | ||||
tcp_lro_flush(lc, le); | tcp_lro_flush(lc, le); | ||||
} | } | ||||
} | } | ||||
} | } | ||||
void | void | ||||
tcp_lro_flush(struct lro_ctrl *lc, struct lro_entry *le) | tcp_lro_flush(struct lro_ctrl *lc, struct lro_entry *le) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 351 Lines • ▼ Show 20 Lines | case ETHERTYPE_IP: | ||||
le->dest_ip4 != ip4->ip_dst.s_addr) | le->dest_ip4 != ip4->ip_dst.s_addr) | ||||
continue; | continue; | ||||
break; | break; | ||||
#endif | #endif | ||||
} | } | ||||
/* Flush now if appending will result in overflow. */ | /* Flush now if appending will result in overflow. */ | ||||
if (le->p_len > (lc->lro_length_lim - tcp_data_len)) { | if (le->p_len > (lc->lro_length_lim - tcp_data_len)) { | ||||
LIST_REMOVE(le, next); | tcp_lro_active_remove(le); | ||||
tcp_lro_flush(lc, le); | tcp_lro_flush(lc, le); | ||||
break; | break; | ||||
} | } | ||||
/* Try to append the new segment. */ | /* Try to append the new segment. */ | ||||
if (__predict_false(seq != le->next_seq || | if (__predict_false(seq != le->next_seq || | ||||
(tcp_data_len == 0 && le->ack_seq == th->th_ack))) { | (tcp_data_len == 0 && le->ack_seq == th->th_ack))) { | ||||
/* Out of order packet or duplicate ACK. */ | /* Out of order packet or duplicate ACK. */ | ||||
LIST_REMOVE(le, next); | tcp_lro_active_remove(le); | ||||
tcp_lro_flush(lc, le); | tcp_lro_flush(lc, le); | ||||
return (TCP_LRO_CANNOT); | return (TCP_LRO_CANNOT); | ||||
} | } | ||||
if (l != 0) { | if (l != 0) { | ||||
uint32_t tsval = ntohl(*(ts_ptr + 1)); | uint32_t tsval = ntohl(*(ts_ptr + 1)); | ||||
/* Make sure timestamp values are increasing. */ | /* Make sure timestamp values are increasing. */ | ||||
/* XXX-BZ flip and use TSTMP_GEQ macro for this? */ | /* XXX-BZ flip and use TSTMP_GEQ macro for this? */ | ||||
Show All 16 Lines | #endif | ||||
if (tcp_data_len == 0) { | if (tcp_data_len == 0) { | ||||
m_freem(m); | m_freem(m); | ||||
/* | /* | ||||
* Flush this LRO entry, if this ACK should not | * Flush this LRO entry, if this ACK should not | ||||
* be further delayed. | * be further delayed. | ||||
*/ | */ | ||||
if (le->append_cnt >= lc->lro_ackcnt_lim) { | if (le->append_cnt >= lc->lro_ackcnt_lim) { | ||||
LIST_REMOVE(le, next); | tcp_lro_active_remove(le); | ||||
tcp_lro_flush(lc, le); | tcp_lro_flush(lc, le); | ||||
} | } | ||||
return (0); | return (0); | ||||
} | } | ||||
le->p_len += tcp_data_len; | le->p_len += tcp_data_len; | ||||
/* | /* | ||||
* Adjust the mbuf so that m_data points to the first byte of | * Adjust the mbuf so that m_data points to the first byte of | ||||
* the ULP payload. Adjust the mbuf to avoid complications and | * the ULP payload. Adjust the mbuf to avoid complications and | ||||
* append new segment to existing mbuf chain. | * append new segment to existing mbuf chain. | ||||
*/ | */ | ||||
m_adj(m, m->m_pkthdr.len - tcp_data_len); | m_adj(m, m->m_pkthdr.len - tcp_data_len); | ||||
m_demote_pkthdr(m); | m_demote_pkthdr(m); | ||||
le->m_tail->m_next = m; | le->m_tail->m_next = m; | ||||
le->m_tail = m_last(m); | le->m_tail = m_last(m); | ||||
/* | /* | ||||
* If a possible next full length packet would cause an | * If a possible next full length packet would cause an | ||||
* overflow, pro-actively flush now. | * overflow, pro-actively flush now. | ||||
*/ | */ | ||||
if (le->p_len > (lc->lro_length_lim - lc->ifp->if_mtu)) { | if (le->p_len > (lc->lro_length_lim - lc->ifp->if_mtu)) { | ||||
LIST_REMOVE(le, next); | tcp_lro_active_remove(le); | ||||
tcp_lro_flush(lc, le); | tcp_lro_flush(lc, le); | ||||
} else | } else | ||||
getmicrotime(&le->mtime); | getmicrotime(&le->mtime); | ||||
return (0); | return (0); | ||||
} | } | ||||
/* Try to find an empty slot. */ | /* Try to find an empty slot. */ | ||||
if (LIST_EMPTY(&lc->lro_free)) | if (LIST_EMPTY(&lc->lro_free)) | ||||
return (TCP_LRO_NO_ENTRIES); | return (TCP_LRO_NO_ENTRIES); | ||||
/* Start a new segment chain. */ | /* Start a new segment chain. */ | ||||
le = LIST_FIRST(&lc->lro_free); | le = LIST_FIRST(&lc->lro_free); | ||||
LIST_REMOVE(le, next); | LIST_REMOVE(le, next); | ||||
LIST_INSERT_HEAD(&lc->lro_active, le, next); | tcp_lro_active_insert(lc, le); | ||||
getmicrotime(&le->mtime); | getmicrotime(&le->mtime); | ||||
/* Start filling in details. */ | /* Start filling in details. */ | ||||
switch (eh_type) { | switch (eh_type) { | ||||
#ifdef INET6 | #ifdef INET6 | ||||
case ETHERTYPE_IPV6: | case ETHERTYPE_IPV6: | ||||
le->le_ip6 = ip6; | le->le_ip6 = ip6; | ||||
le->source_ip6 = ip6->ip6_src; | le->source_ip6 = ip6->ip6_src; | ||||
▲ Show 20 Lines • Show All 83 Lines • Show Last 20 Lines |