Changeset View
Changeset View
Standalone View
Standalone View
sys/netinet/tcp_lro.c
Show First 20 Lines • Show All 359 Lines • ▼ Show 20 Lines | #endif | ||||
(*lc->ifp->if_input)(lc->ifp, le->m_head); | (*lc->ifp->if_input)(lc->ifp, le->m_head); | ||||
lc->lro_queued += le->append_cnt + 1; | lc->lro_queued += le->append_cnt + 1; | ||||
lc->lro_flushed++; | lc->lro_flushed++; | ||||
bzero(le, sizeof(*le)); | bzero(le, sizeof(*le)); | ||||
LIST_INSERT_HEAD(&lc->lro_free, le, next); | LIST_INSERT_HEAD(&lc->lro_free, le, next); | ||||
} | } | ||||
static int | static inline uint64_t | ||||
tcp_lro_mbuf_compare_header(const void *ppa, const void *ppb) | tcp_lro_msb_64(uint64_t x) | ||||
{ | { | ||||
const struct mbuf *ma = *((const struct mbuf * const *)ppa); | x |= (x >> 1); | ||||
const struct mbuf *mb = *((const struct mbuf * const *)ppb); | x |= (x >> 2); | ||||
int ret; | x |= (x >> 4); | ||||
x |= (x >> 8); | |||||
x |= (x >> 16); | |||||
x |= (x >> 32); | |||||
return (x & ~(x >> 1)); | |||||
} | |||||
ret = M_HASHTYPE_GET(ma) - M_HASHTYPE_GET(mb); | /* | ||||
if (ret != 0) | * The tcp_lro_sort() routine is comparable to qsort(), except it has | ||||
* a worst case complexity limit of O(MIN(N,64)*N), where N is the | |||||
* number of elements to sort and 64 is the number of hash bits | |||||
* available. | |||||
kbowling: Simple typo alorithm >> algorithm | |||||
*/ | |||||
static void | |||||
tcp_lro_sort(struct mbuf **parray, uint32_t size) | |||||
{ | |||||
uint64_t ones; | |||||
uint64_t zeros; | |||||
uint32_t limit; | |||||
uint32_t x; | |||||
uint32_t y; | |||||
repeat: | |||||
if (size <= 1) | |||||
return; | |||||
/* scan all hash values for set and cleared bits */ | |||||
ones = 0; | |||||
zeros = 0; | |||||
for (x = 0; x != size; x++) { | |||||
ones |= TCP_LRO_SEQUENCE_U64(parray[x]); | |||||
zeros |= ~TCP_LRO_SEQUENCE_U64(parray[x]); | |||||
} | |||||
/* check if all are same value */ | |||||
ones &= zeros; | |||||
if (ones == 0) | |||||
return; | |||||
/* compute bit-mask */ | |||||
ones = tcp_lro_msb_64(ones); | |||||
/* compute how many zeros are in our set */ | |||||
for (limit = x = 0; x != size; x++) { | |||||
if (!(TCP_LRO_SEQUENCE_U64(parray[x]) & ones)) | |||||
limit++; | |||||
} | |||||
/* sanity check */ | |||||
if (limit == 0 || limit == size) | |||||
return; | |||||
x = 0; | |||||
y = limit; | |||||
while (1) { | |||||
struct mbuf *temp; | |||||
/* find next misplaced cleared bit */ | |||||
while (TCP_LRO_SEQUENCE_U64(parray[y]) & ones) { | |||||
if (++y == size) | |||||
goto done; | goto done; | ||||
} | |||||
if (ma->m_pkthdr.flowid > mb->m_pkthdr.flowid) | /* find next misplaced set bit */ | ||||
return (1); | while (!(TCP_LRO_SEQUENCE_U64(parray[x]) & ones)) { | ||||
else if (ma->m_pkthdr.flowid < mb->m_pkthdr.flowid) | if (++x == limit) | ||||
return (-1); | goto done; | ||||
} | |||||
ret = TCP_LRO_SEQUENCE(ma) - TCP_LRO_SEQUENCE(mb); | /* swap */ | ||||
temp = parray[x]; | |||||
parray[x] = parray[y]; | |||||
parray[y] = temp; | |||||
} | |||||
done: | done: | ||||
return (ret); | /* sort zeros */ | ||||
tcp_lro_sort(parray, limit); | |||||
/* sort ones */ | |||||
parray += limit; | |||||
size -= limit; | |||||
goto repeat; | |||||
} | } | ||||
void | void | ||||
tcp_lro_flush_all(struct lro_ctrl *lc) | tcp_lro_flush_all(struct lro_ctrl *lc) | ||||
{ | { | ||||
uint32_t hashtype; | uint32_t hashtype; | ||||
uint32_t flowid; | uint32_t flowid; | ||||
unsigned x; | unsigned x; | ||||
/* check if no mbufs to flush */ | /* check if no mbufs to flush */ | ||||
if (lc->lro_mbuf_count == 0) | if (lc->lro_mbuf_count == 0) | ||||
goto done; | goto done; | ||||
/* sort all mbufs according to stream */ | /* sort all mbufs according to stream */ | ||||
qsort(lc->lro_mbuf_data, lc->lro_mbuf_count, sizeof(struct mbuf *), | tcp_lro_sort(lc->lro_mbuf_data, lc->lro_mbuf_count); | ||||
&tcp_lro_mbuf_compare_header); | |||||
/* input data into LRO engine, stream by stream */ | /* input data into LRO engine, stream by stream */ | ||||
flowid = 0; | flowid = 0; | ||||
hashtype = M_HASHTYPE_NONE; | hashtype = M_HASHTYPE_NONE; | ||||
for (x = 0; x != lc->lro_mbuf_count; x++) { | for (x = 0; x != lc->lro_mbuf_count; x++) { | ||||
struct mbuf *mb; | struct mbuf *mb; | ||||
mb = lc->lro_mbuf_data[x]; | mb = lc->lro_mbuf_data[x]; | ||||
/* check for new stream */ | /* check for new stream */ | ||||
if (mb->m_pkthdr.flowid != flowid || | if (mb->m_pkthdr.flowid != flowid || | ||||
M_HASHTYPE_GET(mb) != hashtype) { | M_HASHTYPE_GET(mb) != hashtype) { | ||||
flowid = mb->m_pkthdr.flowid; | flowid = mb->m_pkthdr.flowid; | ||||
hashtype = M_HASHTYPE_GET(mb); | hashtype = M_HASHTYPE_GET(mb); | ||||
/* flush active streams */ | /* flush active streams */ | ||||
tcp_lro_rx_done(lc); | tcp_lro_rx_done(lc); | ||||
} | } | ||||
#ifdef TCP_LRO_RESET_SEQUENCE | #ifdef TCP_LRO_RESET_SEQUENCE | ||||
/* reset sequence number */ | /* reset sequence number */ | ||||
TCP_LRO_SEQUENCE(mb) = 0; | TCP_LRO_SEQUENCE_U64(mb) = 0; | ||||
#endif | #endif | ||||
/* add packet to LRO engine */ | /* add packet to LRO engine */ | ||||
if (tcp_lro_rx(lc, mb, 0) != 0) { | if (tcp_lro_rx(lc, mb, 0) != 0) { | ||||
/* input packet to network layer */ | /* input packet to network layer */ | ||||
(*lc->ifp->if_input)(lc->ifp, mb); | (*lc->ifp->if_input)(lc->ifp, mb); | ||||
lc->lro_queued++; | lc->lro_queued++; | ||||
lc->lro_flushed++; | lc->lro_flushed++; | ||||
} | } | ||||
▲ Show 20 Lines • Show All 362 Lines • ▼ Show 20 Lines | if (__predict_false(mb->m_pkthdr.csum_flags == 0 || | ||||
(*lc->ifp->if_input) (lc->ifp, mb); | (*lc->ifp->if_input) (lc->ifp, mb); | ||||
return; | return; | ||||
} | } | ||||
/* check if array is full */ | /* check if array is full */ | ||||
if (__predict_false(lc->lro_mbuf_count == lc->lro_mbuf_max)) | if (__predict_false(lc->lro_mbuf_count == lc->lro_mbuf_max)) | ||||
tcp_lro_flush_all(lc); | tcp_lro_flush_all(lc); | ||||
/* store sequence number */ | /* create sequence number */ | ||||
TCP_LRO_SEQUENCE(mb) = lc->lro_mbuf_count; | TCP_LRO_SEQUENCE_U64(mb) = | ||||
(((uint64_t)M_HASHTYPE_GET(mb)) << 56) | | |||||
(((uint64_t)mb->m_pkthdr.flowid) << 24) | | |||||
((uint64_t)lc->lro_mbuf_count); | |||||
/* enter mbuf */ | /* enter mbuf */ | ||||
lc->lro_mbuf_data[lc->lro_mbuf_count++] = mb; | lc->lro_mbuf_data[lc->lro_mbuf_count++] = mb; | ||||
} | } | ||||
/* end */ | /* end */ |
Simple typo alorithm >> algorithm