Changeset View
Changeset View
Standalone View
Standalone View
sys/netinet/tcp_lro.c
Show First 20 Lines • Show All 105 Lines • ▼ Show 20 Lines | tcp_lro_init_args(struct lro_ctrl *lc, struct ifnet *ifp, | ||||
lc->lro_cnt = lro_entries; | lc->lro_cnt = lro_entries; | ||||
lc->lro_ackcnt_lim = TCP_LRO_ACKCNT_MAX; | lc->lro_ackcnt_lim = TCP_LRO_ACKCNT_MAX; | ||||
lc->lro_length_lim = TCP_LRO_LENGTH_MAX; | lc->lro_length_lim = TCP_LRO_LENGTH_MAX; | ||||
lc->ifp = ifp; | lc->ifp = ifp; | ||||
LIST_INIT(&lc->lro_free); | LIST_INIT(&lc->lro_free); | ||||
LIST_INIT(&lc->lro_active); | LIST_INIT(&lc->lro_active); | ||||
/* compute size to allocate */ | /* compute size to allocate */ | ||||
size = (lro_mbufs * sizeof(struct mbuf *)) + | size = (lro_mbufs * sizeof(struct lro_mbuf_sort)) + | ||||
(lro_entries * sizeof(*le)); | (lro_entries * sizeof(*le)); | ||||
lc->lro_mbuf_data = (struct mbuf **) | lc->lro_mbuf_data = (struct lro_mbuf_sort *) | ||||
malloc(size, M_LRO, M_NOWAIT | M_ZERO); | malloc(size, M_LRO, M_NOWAIT | M_ZERO); | ||||
/* check for out of memory */ | /* check for out of memory */ | ||||
if (lc->lro_mbuf_data == NULL) { | if (lc->lro_mbuf_data == NULL) { | ||||
memset(lc, 0, sizeof(*lc)); | memset(lc, 0, sizeof(*lc)); | ||||
return (ENOMEM); | return (ENOMEM); | ||||
} | } | ||||
/* compute offset for LRO entries */ | /* compute offset for LRO entries */ | ||||
Show All 19 Lines | tcp_lro_free(struct lro_ctrl *lc) | ||||
/* free active mbufs, if any */ | /* free active mbufs, if any */ | ||||
while ((le = LIST_FIRST(&lc->lro_active)) != NULL) { | while ((le = LIST_FIRST(&lc->lro_active)) != NULL) { | ||||
tcp_lro_active_remove(le); | tcp_lro_active_remove(le); | ||||
m_freem(le->m_head); | m_freem(le->m_head); | ||||
} | } | ||||
/* free mbuf array, if any */ | /* free mbuf array, if any */ | ||||
for (x = 0; x != lc->lro_mbuf_count; x++) | for (x = 0; x != lc->lro_mbuf_count; x++) | ||||
m_freem(lc->lro_mbuf_data[x]); | m_freem(lc->lro_mbuf_data[x].mb); | ||||
lc->lro_mbuf_count = 0; | lc->lro_mbuf_count = 0; | ||||
/* free allocated memory, if any */ | /* free allocated memory, if any */ | ||||
free(lc->lro_mbuf_data, M_LRO); | free(lc->lro_mbuf_data, M_LRO); | ||||
lc->lro_mbuf_data = NULL; | lc->lro_mbuf_data = NULL; | ||||
} | } | ||||
#ifdef TCP_LRO_UPDATE_CSUM | #ifdef TCP_LRO_UPDATE_CSUM | ||||
▲ Show 20 Lines • Show All 199 Lines • ▼ Show 20 Lines | #endif | ||||
(*lc->ifp->if_input)(lc->ifp, le->m_head); | (*lc->ifp->if_input)(lc->ifp, le->m_head); | ||||
lc->lro_queued += le->append_cnt + 1; | lc->lro_queued += le->append_cnt + 1; | ||||
lc->lro_flushed++; | lc->lro_flushed++; | ||||
bzero(le, sizeof(*le)); | bzero(le, sizeof(*le)); | ||||
LIST_INSERT_HEAD(&lc->lro_free, le, next); | LIST_INSERT_HEAD(&lc->lro_free, le, next); | ||||
} | } | ||||
static int | #ifdef HAVE_INLINE_FLSLL | ||||
tcp_lro_mbuf_compare_header(const void *ppa, const void *ppb) | #define tcp_lro_msb_64(x) (1ULL << (flsll(x) - 1)) | ||||
#else | |||||
static inline uint64_t | |||||
tcp_lro_msb_64(uint64_t x) | |||||
{ | { | ||||
const struct mbuf *ma = *((const struct mbuf * const *)ppa); | x |= (x >> 1); | ||||
const struct mbuf *mb = *((const struct mbuf * const *)ppb); | x |= (x >> 2); | ||||
int ret; | x |= (x >> 4); | ||||
x |= (x >> 8); | |||||
x |= (x >> 16); | |||||
x |= (x >> 32); | |||||
return (x & ~(x >> 1)); | |||||
} | |||||
#endif | |||||
ret = M_HASHTYPE_GET(ma) - M_HASHTYPE_GET(mb); | /* | ||||
kbowling: Simple typo alorithm >> algorithm | |||||
if (ret != 0) | * The tcp_lro_sort() routine is comparable to qsort(), except it has | ||||
goto done; | * a worst case complexity limit of O(MIN(N,64)*N), where N is the | ||||
* number of elements to sort and 64 is the number of sequence bits | |||||
* available. The algorithm is bit-slicing the 64-bit sequence number, | |||||
* sorting one bit at a time from the most significant bit until the | |||||
* least significant one, skipping the constant bits. | |||||
*/ | |||||
static void | |||||
tcp_lro_sort(struct lro_mbuf_sort *parray, uint32_t size) | |||||
{ | |||||
struct lro_mbuf_sort temp; | |||||
uint64_t ones; | |||||
uint64_t zeros; | |||||
uint32_t x; | |||||
uint32_t y; | |||||
if (ma->m_pkthdr.flowid > mb->m_pkthdr.flowid) | repeat: | ||||
return (1); | /* for small arrays bubble sort is faster */ | ||||
else if (ma->m_pkthdr.flowid < mb->m_pkthdr.flowid) | if (size <= 12) { | ||||
return (-1); | for (x = 0; x != size; x++) { | ||||
for (y = x + 1; y != size; y++) { | |||||
if (parray[x].seq > parray[y].seq) { | |||||
/* swap entries */ | |||||
temp = parray[x]; | |||||
parray[x] = parray[y]; | |||||
parray[y] = temp; | |||||
} | |||||
} | |||||
} | |||||
return; | |||||
} | |||||
ret = TCP_LRO_SEQUENCE(ma) - TCP_LRO_SEQUENCE(mb); | /* compute sequence bits which are constant */ | ||||
done: | ones = 0; | ||||
return (ret); | zeros = 0; | ||||
for (x = 0; x != size; x++) { | |||||
ones |= parray[x].seq; | |||||
zeros |= ~parray[x].seq; | |||||
} | } | ||||
/* compute bits which are not constant into "ones" */ | |||||
ones &= zeros; | |||||
if (ones == 0) | |||||
return; | |||||
/* pick the most significant bit which is not constant */ | |||||
ones = tcp_lro_msb_64(ones); | |||||
/* | |||||
* Move entries having cleared sequence bits to the beginning | |||||
* of the array: | |||||
*/ | |||||
for (x = y = 0; y != size; y++) { | |||||
/* skip set bits */ | |||||
if (parray[y].seq & ones) | |||||
continue; | |||||
/* swap entries */ | |||||
temp = parray[x]; | |||||
parray[x] = parray[y]; | |||||
parray[y] = temp; | |||||
x++; | |||||
} | |||||
KASSERT(x != 0 && x != size, ("Memory is corrupted\n")); | |||||
/* sort zeros */ | |||||
tcp_lro_sort(parray, x); | |||||
/* sort ones */ | |||||
parray += x; | |||||
size -= x; | |||||
goto repeat; | |||||
} | |||||
void | void | ||||
tcp_lro_flush_all(struct lro_ctrl *lc) | tcp_lro_flush_all(struct lro_ctrl *lc) | ||||
{ | { | ||||
uint32_t hashtype; | uint64_t seq; | ||||
uint32_t flowid; | uint64_t nseq; | ||||
unsigned x; | unsigned x; | ||||
/* check if no mbufs to flush */ | /* check if no mbufs to flush */ | ||||
if (lc->lro_mbuf_count == 0) | if (lc->lro_mbuf_count == 0) | ||||
goto done; | goto done; | ||||
/* sort all mbufs according to stream */ | /* sort all mbufs according to stream */ | ||||
qsort(lc->lro_mbuf_data, lc->lro_mbuf_count, sizeof(struct mbuf *), | tcp_lro_sort(lc->lro_mbuf_data, lc->lro_mbuf_count); | ||||
&tcp_lro_mbuf_compare_header); | |||||
/* input data into LRO engine, stream by stream */ | /* input data into LRO engine, stream by stream */ | ||||
flowid = 0; | seq = 0; | ||||
hashtype = M_HASHTYPE_NONE; | |||||
for (x = 0; x != lc->lro_mbuf_count; x++) { | for (x = 0; x != lc->lro_mbuf_count; x++) { | ||||
struct mbuf *mb; | struct mbuf *mb; | ||||
mb = lc->lro_mbuf_data[x]; | /* get mbuf */ | ||||
mb = lc->lro_mbuf_data[x].mb; | |||||
/* get sequence number, masking away the packet index */ | |||||
nseq = lc->lro_mbuf_data[x].seq & (-1ULL << 24); | |||||
/* check for new stream */ | /* check for new stream */ | ||||
if (mb->m_pkthdr.flowid != flowid || | if (seq != nseq) { | ||||
M_HASHTYPE_GET(mb) != hashtype) { | seq = nseq; | ||||
flowid = mb->m_pkthdr.flowid; | |||||
hashtype = M_HASHTYPE_GET(mb); | |||||
/* flush active streams */ | /* flush active streams */ | ||||
tcp_lro_rx_done(lc); | tcp_lro_rx_done(lc); | ||||
} | } | ||||
#ifdef TCP_LRO_RESET_SEQUENCE | |||||
/* reset sequence number */ | |||||
TCP_LRO_SEQUENCE(mb) = 0; | |||||
#endif | |||||
/* add packet to LRO engine */ | /* add packet to LRO engine */ | ||||
if (tcp_lro_rx(lc, mb, 0) != 0) { | if (tcp_lro_rx(lc, mb, 0) != 0) { | ||||
/* input packet to network layer */ | /* input packet to network layer */ | ||||
(*lc->ifp->if_input)(lc->ifp, mb); | (*lc->ifp->if_input)(lc->ifp, mb); | ||||
lc->lro_queued++; | lc->lro_queued++; | ||||
lc->lro_flushed++; | lc->lro_flushed++; | ||||
} | } | ||||
} | } | ||||
▲ Show 20 Lines • Show All 361 Lines • ▼ Show 20 Lines | if (__predict_false(mb->m_pkthdr.csum_flags == 0 || | ||||
(*lc->ifp->if_input) (lc->ifp, mb); | (*lc->ifp->if_input) (lc->ifp, mb); | ||||
return; | return; | ||||
} | } | ||||
/* check if array is full */ | /* check if array is full */ | ||||
if (__predict_false(lc->lro_mbuf_count == lc->lro_mbuf_max)) | if (__predict_false(lc->lro_mbuf_count == lc->lro_mbuf_max)) | ||||
tcp_lro_flush_all(lc); | tcp_lro_flush_all(lc); | ||||
/* store sequence number */ | /* create sequence number */ | ||||
TCP_LRO_SEQUENCE(mb) = lc->lro_mbuf_count; | lc->lro_mbuf_data[lc->lro_mbuf_count].seq = | ||||
(((uint64_t)M_HASHTYPE_GET(mb)) << 56) | | |||||
(((uint64_t)mb->m_pkthdr.flowid) << 24) | | |||||
((uint64_t)lc->lro_mbuf_count); | |||||
/* enter mbuf */ | /* enter mbuf */ | ||||
lc->lro_mbuf_data[lc->lro_mbuf_count++] = mb; | lc->lro_mbuf_data[lc->lro_mbuf_count++].mb = mb; | ||||
} | } | ||||
/* end */ | /* end */ |
Simple typo alorithm >> algorithm