Changeset View
Changeset View
Standalone View
Standalone View
sys/net/iflib.c
Show First 20 Lines • Show All 1,221 Lines • ▼ Show 20 Lines | |||||
#endif | #endif | ||||
#if defined(__i386__) || defined(__amd64__) | #if defined(__i386__) || defined(__amd64__) | ||||
static __inline void | static __inline void | ||||
prefetch(void *x) | prefetch(void *x) | ||||
{ | { | ||||
__asm volatile("prefetcht0 %0" :: "m" (*(unsigned long *)x)); | __asm volatile("prefetcht0 %0" :: "m" (*(unsigned long *)x)); | ||||
} | } | ||||
static __inline void | |||||
prefetch2cachelines(void *x) | |||||
{ | |||||
__asm volatile("prefetcht0 %0" :: "m" (*(unsigned long *)x)); | |||||
#if (CACHE_LINE_SIZE < 128) | |||||
__asm volatile("prefetcht0 %0" :: "m" (*(((unsigned long *)x)+CACHE_LINE_SIZE/(sizeof(unsigned long))))); | |||||
#endif | |||||
} | |||||
#else | #else | ||||
#define prefetch(x) | #define prefetch(x) | ||||
#define prefetch2cachelines(x) | |||||
#endif | #endif | ||||
static void | static void | ||||
_iflib_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int err) | _iflib_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int err) | ||||
{ | { | ||||
if (err) | if (err) | ||||
return; | return; | ||||
*(bus_addr_t *) arg = segs[0].ds_addr; | *(bus_addr_t *) arg = segs[0].ds_addr; | ||||
▲ Show 20 Lines • Show All 1,841 Lines • ▼ Show 20 Lines | if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { | ||||
max_segs = scctx->isc_tx_tso_segments_max; | max_segs = scctx->isc_tx_tso_segments_max; | ||||
} else { | } else { | ||||
desc_tag = txq->ift_desc_tag; | desc_tag = txq->ift_desc_tag; | ||||
max_segs = scctx->isc_tx_nsegments; | max_segs = scctx->isc_tx_nsegments; | ||||
} | } | ||||
m_head = *m_headp; | m_head = *m_headp; | ||||
pkt_info_zero(&pi); | pkt_info_zero(&pi); | ||||
pi.ipi_len = m_head->m_pkthdr.len; | |||||
pi.ipi_mflags = (m_head->m_flags & (M_VLANTAG|M_BCAST|M_MCAST)); | pi.ipi_mflags = (m_head->m_flags & (M_VLANTAG|M_BCAST|M_MCAST)); | ||||
pi.ipi_csum_flags = m_head->m_pkthdr.csum_flags; | |||||
pi.ipi_vtag = (m_head->m_flags & M_VLANTAG) ? m_head->m_pkthdr.ether_vtag : 0; | |||||
pi.ipi_pidx = pidx; | pi.ipi_pidx = pidx; | ||||
pi.ipi_qsidx = txq->ift_id; | pi.ipi_qsidx = txq->ift_id; | ||||
pi.ipi_len = m_head->m_pkthdr.len; | |||||
pi.ipi_csum_flags = m_head->m_pkthdr.csum_flags; | |||||
pi.ipi_vtag = (m_head->m_flags & M_VLANTAG) ? m_head->m_pkthdr.ether_vtag : 0; | |||||
/* deliberate bitwise OR to make one condition */ | /* deliberate bitwise OR to make one condition */ | ||||
if (__predict_true((pi.ipi_csum_flags | pi.ipi_vtag))) { | if (__predict_true((pi.ipi_csum_flags | pi.ipi_vtag))) { | ||||
if (__predict_false((err = iflib_parse_header(txq, &pi, m_headp)) != 0)) | if (__predict_false((err = iflib_parse_header(txq, &pi, m_headp)) != 0)) | ||||
return (err); | return (err); | ||||
m_head = *m_headp; | m_head = *m_headp; | ||||
} | } | ||||
▲ Show 20 Lines • Show All 215 Lines • ▼ Show 20 Lines | _ring_peek_one(struct ifmp_ring *r, int cidx, int offset, int remaining) | ||||
struct mbuf **items; | struct mbuf **items; | ||||
size = r->size; | size = r->size; | ||||
next = (cidx + CACHE_PTR_INCREMENT) & (size-1); | next = (cidx + CACHE_PTR_INCREMENT) & (size-1); | ||||
items = __DEVOLATILE(struct mbuf **, &r->items[0]); | items = __DEVOLATILE(struct mbuf **, &r->items[0]); | ||||
prefetch(items[(cidx + offset) & (size-1)]); | prefetch(items[(cidx + offset) & (size-1)]); | ||||
if (remaining > 1) { | if (remaining > 1) { | ||||
prefetch(&items[next]); | prefetch2cachelines(&items[next]); | ||||
prefetch(items[(cidx + offset + 1) & (size-1)]); | prefetch2cachelines(items[(cidx + offset + 1) & (size-1)]); | ||||
prefetch(items[(cidx + offset + 2) & (size-1)]); | prefetch2cachelines(items[(cidx + offset + 2) & (size-1)]); | ||||
prefetch(items[(cidx + offset + 3) & (size-1)]); | prefetch2cachelines(items[(cidx + offset + 3) & (size-1)]); | ||||
} | } | ||||
return (__DEVOLATILE(struct mbuf **, &r->items[(cidx + offset) & (size-1)])); | return (__DEVOLATILE(struct mbuf **, &r->items[(cidx + offset) & (size-1)])); | ||||
} | } | ||||
static void | static void | ||||
iflib_txq_check_drain(iflib_txq_t txq, int budget) | iflib_txq_check_drain(iflib_txq_t txq, int budget) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 2,357 Lines • Show Last 20 Lines |