Changeset View
Changeset View
Standalone View
Standalone View
sys/dev/cxgbe/t4_sge.c
Show First 20 Lines • Show All 77 Lines • ▼ Show 20 Lines | |||||
#ifdef T4_PKT_TIMESTAMP | #ifdef T4_PKT_TIMESTAMP | ||||
#define RX_COPY_THRESHOLD (MINCLSIZE - 8) | #define RX_COPY_THRESHOLD (MINCLSIZE - 8) | ||||
#else | #else | ||||
#define RX_COPY_THRESHOLD MINCLSIZE | #define RX_COPY_THRESHOLD MINCLSIZE | ||||
#endif | #endif | ||||
/* Internal mbuf flags stored in PH_loc.eight[1]. */ | /* Internal mbuf flags stored in PH_loc.eight[1]. */ | ||||
#define MC_NOMAP 0x01 | |||||
#define MC_RAW_WR 0x02 | #define MC_RAW_WR 0x02 | ||||
/* | /* | ||||
* Ethernet frames are DMA'd at this byte offset into the freelist buffer. | * Ethernet frames are DMA'd at this byte offset into the freelist buffer. | ||||
* 0-7 are valid values. | * 0-7 are valid values. | ||||
*/ | */ | ||||
static int fl_pktshift = 0; | static int fl_pktshift = 0; | ||||
SYSCTL_INT(_hw_cxgbe, OID_AUTO, fl_pktshift, CTLFLAG_RDTUN, &fl_pktshift, 0, | SYSCTL_INT(_hw_cxgbe, OID_AUTO, fl_pktshift, CTLFLAG_RDTUN, &fl_pktshift, 0, | ||||
▲ Show 20 Lines • Show All 2,335 Lines • ▼ Show 20 Lines | for (;;) { | ||||
offset = 0; | offset = 0; | ||||
MPASS(m != NULL); | MPASS(m != NULL); | ||||
} | } | ||||
*poffset = offset; | *poffset = offset; | ||||
*pm = m; | *pm = m; | ||||
return ((void *)p); | return ((void *)p); | ||||
} | } | ||||
static inline int | |||||
count_mbuf_ext_pgs(struct mbuf *m, int skip, vm_paddr_t *nextaddr) | |||||
{ | |||||
struct mbuf_ext_pgs *ext_pgs; | |||||
vm_paddr_t paddr; | |||||
int i, len, off, pglen, pgoff, seglen, segoff; | |||||
int nsegs = 0; | |||||
MBUF_EXT_PGS_ASSERT(m); | |||||
ext_pgs = m->m_ext.ext_pgs; | |||||
off = mtod(m, vm_offset_t); | |||||
len = m->m_len; | |||||
off += skip; | |||||
len -= skip; | |||||
if (ext_pgs->hdr_len != 0) { | |||||
if (off >= ext_pgs->hdr_len) { | |||||
off -= ext_pgs->hdr_len; | |||||
} else { | |||||
seglen = ext_pgs->hdr_len - off; | |||||
segoff = off; | |||||
seglen = min(seglen, len); | |||||
off = 0; | |||||
len -= seglen; | |||||
paddr = pmap_kextract( | |||||
(vm_offset_t)&ext_pgs->hdr[segoff]); | |||||
if (*nextaddr != paddr) | |||||
nsegs++; | |||||
*nextaddr = paddr + seglen; | |||||
} | |||||
} | |||||
pgoff = ext_pgs->first_pg_off; | |||||
for (i = 0; i < ext_pgs->npgs && len > 0; i++) { | |||||
pglen = mbuf_ext_pg_len(ext_pgs, i, pgoff); | |||||
if (off >= pglen) { | |||||
off -= pglen; | |||||
pgoff = 0; | |||||
continue; | |||||
} | |||||
seglen = pglen - off; | |||||
segoff = pgoff + off; | |||||
off = 0; | |||||
seglen = min(seglen, len); | |||||
len -= seglen; | |||||
paddr = ext_pgs->pa[i] + segoff; | |||||
if (*nextaddr != paddr) | |||||
nsegs++; | |||||
*nextaddr = paddr + seglen; | |||||
pgoff = 0; | |||||
}; | |||||
if (len != 0) { | |||||
seglen = min(len, ext_pgs->trail_len - off); | |||||
len -= seglen; | |||||
paddr = pmap_kextract((vm_offset_t)&ext_pgs->trail[off]); | |||||
if (*nextaddr != paddr) | |||||
nsegs++; | |||||
*nextaddr = paddr + seglen; | |||||
} | |||||
return (nsegs); | |||||
} | |||||
/* | /* | ||||
* Can deal with empty mbufs in the chain that have m_len = 0, but the chain | * Can deal with empty mbufs in the chain that have m_len = 0, but the chain | ||||
* must have at least one mbuf that's not empty. It is possible for this | * must have at least one mbuf that's not empty. It is possible for this | ||||
* routine to return 0 if skip accounts for all the contents of the mbuf chain. | * routine to return 0 if skip accounts for all the contents of the mbuf chain. | ||||
*/ | */ | ||||
static inline int | static inline int | ||||
count_mbuf_nsegs(struct mbuf *m, int skip) | count_mbuf_nsegs(struct mbuf *m, int skip, uint8_t *cflags) | ||||
{ | { | ||||
vm_paddr_t lastb, next; | vm_paddr_t nextaddr, paddr; | ||||
vm_offset_t va; | vm_offset_t va; | ||||
int len, nsegs; | int len, nsegs; | ||||
M_ASSERTPKTHDR(m); | M_ASSERTPKTHDR(m); | ||||
MPASS(m->m_pkthdr.len > 0); | MPASS(m->m_pkthdr.len > 0); | ||||
MPASS(m->m_pkthdr.len >= skip); | MPASS(m->m_pkthdr.len >= skip); | ||||
nsegs = 0; | nsegs = 0; | ||||
lastb = 0; | nextaddr = 0; | ||||
for (; m; m = m->m_next) { | for (; m; m = m->m_next) { | ||||
len = m->m_len; | len = m->m_len; | ||||
if (__predict_false(len == 0)) | if (__predict_false(len == 0)) | ||||
continue; | continue; | ||||
if (skip >= len) { | if (skip >= len) { | ||||
skip -= len; | skip -= len; | ||||
continue; | continue; | ||||
} | } | ||||
if ((m->m_flags & M_NOMAP) != 0) { | |||||
*cflags |= MC_NOMAP; | |||||
nsegs += count_mbuf_ext_pgs(m, skip, &nextaddr); | |||||
skip = 0; | |||||
continue; | |||||
} | |||||
va = mtod(m, vm_offset_t) + skip; | va = mtod(m, vm_offset_t) + skip; | ||||
len -= skip; | len -= skip; | ||||
skip = 0; | skip = 0; | ||||
next = pmap_kextract(va); | paddr = pmap_kextract(va); | ||||
nsegs += sglist_count((void *)(uintptr_t)va, len); | nsegs += sglist_count((void *)(uintptr_t)va, len); | ||||
if (lastb + 1 == next) | if (paddr == nextaddr) | ||||
nsegs--; | nsegs--; | ||||
lastb = pmap_kextract(va + len - 1); | nextaddr = pmap_kextract(va + len - 1) + 1; | ||||
} | } | ||||
return (nsegs); | return (nsegs); | ||||
} | } | ||||
/* | /* | ||||
* Analyze the mbuf to determine its tx needs. The mbuf passed in may change: | * Analyze the mbuf to determine its tx needs. The mbuf passed in may change: | ||||
* a) caller can assume it's been freed if this function returns with an error. | * a) caller can assume it's been freed if this function returns with an error. | ||||
* b) it may get defragged up if the gather list is too long for the hardware. | * b) it may get defragged up if the gather list is too long for the hardware. | ||||
*/ | */ | ||||
int | int | ||||
parse_pkt(struct adapter *sc, struct mbuf **mp) | parse_pkt(struct adapter *sc, struct mbuf **mp) | ||||
{ | { | ||||
struct mbuf *m0 = *mp, *m; | struct mbuf *m0 = *mp, *m; | ||||
int rc, nsegs, defragged = 0, offset; | int rc, nsegs, defragged = 0, offset; | ||||
struct ether_header *eh; | struct ether_header *eh; | ||||
void *l3hdr; | void *l3hdr; | ||||
#if defined(INET) || defined(INET6) | #if defined(INET) || defined(INET6) | ||||
struct tcphdr *tcp; | struct tcphdr *tcp; | ||||
#endif | #endif | ||||
uint16_t eh_type; | uint16_t eh_type; | ||||
uint8_t cflags; | |||||
cflags = 0; | |||||
M_ASSERTPKTHDR(m0); | M_ASSERTPKTHDR(m0); | ||||
if (__predict_false(m0->m_pkthdr.len < ETHER_HDR_LEN)) { | if (__predict_false(m0->m_pkthdr.len < ETHER_HDR_LEN)) { | ||||
rc = EINVAL; | rc = EINVAL; | ||||
fail: | fail: | ||||
m_freem(m0); | m_freem(m0); | ||||
*mp = NULL; | *mp = NULL; | ||||
return (rc); | return (rc); | ||||
} | } | ||||
restart: | restart: | ||||
/* | /* | ||||
* First count the number of gather list segments in the payload. | * First count the number of gather list segments in the payload. | ||||
* Defrag the mbuf if nsegs exceeds the hardware limit. | * Defrag the mbuf if nsegs exceeds the hardware limit. | ||||
*/ | */ | ||||
M_ASSERTPKTHDR(m0); | M_ASSERTPKTHDR(m0); | ||||
MPASS(m0->m_pkthdr.len > 0); | MPASS(m0->m_pkthdr.len > 0); | ||||
nsegs = count_mbuf_nsegs(m0, 0); | nsegs = count_mbuf_nsegs(m0, 0, &cflags); | ||||
if (nsegs > (needs_tso(m0) ? TX_SGL_SEGS_TSO : TX_SGL_SEGS)) { | if (nsegs > (needs_tso(m0) ? TX_SGL_SEGS_TSO : TX_SGL_SEGS)) { | ||||
if (defragged++ > 0 || (m = m_defrag(m0, M_NOWAIT)) == NULL) { | if (defragged++ > 0 || (m = m_defrag(m0, M_NOWAIT)) == NULL) { | ||||
rc = EFBIG; | rc = EFBIG; | ||||
goto fail; | goto fail; | ||||
} | } | ||||
*mp = m0 = m; /* update caller's copy after defrag */ | *mp = m0 = m; /* update caller's copy after defrag */ | ||||
goto restart; | goto restart; | ||||
} | } | ||||
if (__predict_false(nsegs > 2 && m0->m_pkthdr.len <= MHLEN)) { | if (__predict_false(nsegs > 2 && m0->m_pkthdr.len <= MHLEN && | ||||
!(cflags & MC_NOMAP))) { | |||||
m0 = m_pullup(m0, m0->m_pkthdr.len); | m0 = m_pullup(m0, m0->m_pkthdr.len); | ||||
if (m0 == NULL) { | if (m0 == NULL) { | ||||
/* Should have left well enough alone. */ | /* Should have left well enough alone. */ | ||||
rc = EFBIG; | rc = EFBIG; | ||||
goto fail; | goto fail; | ||||
} | } | ||||
*mp = m0; /* update caller's copy after pullup */ | *mp = m0; /* update caller's copy after pullup */ | ||||
goto restart; | goto restart; | ||||
} | } | ||||
set_mbuf_nsegs(m0, nsegs); | set_mbuf_nsegs(m0, nsegs); | ||||
set_mbuf_cflags(m0, 0); | set_mbuf_cflags(m0, cflags); | ||||
if (sc->flags & IS_VF) | if (sc->flags & IS_VF) | ||||
set_mbuf_len16(m0, txpkt_vm_len16(nsegs, needs_tso(m0))); | set_mbuf_len16(m0, txpkt_vm_len16(nsegs, needs_tso(m0))); | ||||
else | else | ||||
set_mbuf_len16(m0, txpkt_len16(nsegs, needs_tso(m0))); | set_mbuf_len16(m0, txpkt_len16(nsegs, needs_tso(m0))); | ||||
#ifdef RATELIMIT | #ifdef RATELIMIT | ||||
/* | /* | ||||
* Ethofld is limited to TCP and UDP for now, and only when L4 hw | * Ethofld is limited to TCP and UDP for now, and only when L4 hw | ||||
▲ Show 20 Lines • Show All 72 Lines • ▼ Show 20 Lines | #endif | ||||
} | } | ||||
#ifdef RATELIMIT | #ifdef RATELIMIT | ||||
if (needs_eo(m0)) { | if (needs_eo(m0)) { | ||||
u_int immhdrs; | u_int immhdrs; | ||||
/* EO WRs have the headers in the WR and not the GL. */ | /* EO WRs have the headers in the WR and not the GL. */ | ||||
immhdrs = m0->m_pkthdr.l2hlen + m0->m_pkthdr.l3hlen + | immhdrs = m0->m_pkthdr.l2hlen + m0->m_pkthdr.l3hlen + | ||||
m0->m_pkthdr.l4hlen; | m0->m_pkthdr.l4hlen; | ||||
nsegs = count_mbuf_nsegs(m0, immhdrs); | cflags = 0; | ||||
nsegs = count_mbuf_nsegs(m0, immhdrs, &cflags); | |||||
MPASS(cflags == mbuf_cflags(m0)); | |||||
set_mbuf_eo_nsegs(m0, nsegs); | set_mbuf_eo_nsegs(m0, nsegs); | ||||
set_mbuf_eo_len16(m0, | set_mbuf_eo_len16(m0, | ||||
txpkt_eo_len16(nsegs, immhdrs, needs_tso(m0))); | txpkt_eo_len16(nsegs, immhdrs, needs_tso(m0))); | ||||
} | } | ||||
#endif | #endif | ||||
#endif | #endif | ||||
MPASS(m0 == *mp); | MPASS(m0 == *mp); | ||||
return (0); | return (0); | ||||
▲ Show 20 Lines • Show All 2,090 Lines • ▼ Show 20 Lines | write_txpkt_wr(struct sge_txq *txq, struct fw_eth_tx_pkt_wr *wr, | ||||
MPASS(available > 0 && available < eq->sidx); | MPASS(available > 0 && available < eq->sidx); | ||||
len16 = mbuf_len16(m0); | len16 = mbuf_len16(m0); | ||||
nsegs = mbuf_nsegs(m0); | nsegs = mbuf_nsegs(m0); | ||||
pktlen = m0->m_pkthdr.len; | pktlen = m0->m_pkthdr.len; | ||||
ctrl = sizeof(struct cpl_tx_pkt_core); | ctrl = sizeof(struct cpl_tx_pkt_core); | ||||
if (needs_tso(m0)) | if (needs_tso(m0)) | ||||
ctrl += sizeof(struct cpl_tx_pkt_lso_core); | ctrl += sizeof(struct cpl_tx_pkt_lso_core); | ||||
else if (pktlen <= imm_payload(2) && available >= 2) { | else if (!(mbuf_cflags(m0) & MC_NOMAP) && pktlen <= imm_payload(2) && | ||||
available >= 2) { | |||||
/* Immediate data. Recalculate len16 and set nsegs to 0. */ | /* Immediate data. Recalculate len16 and set nsegs to 0. */ | ||||
ctrl += pktlen; | ctrl += pktlen; | ||||
len16 = howmany(sizeof(struct fw_eth_tx_pkt_wr) + | len16 = howmany(sizeof(struct fw_eth_tx_pkt_wr) + | ||||
sizeof(struct cpl_tx_pkt_core) + pktlen, 16); | sizeof(struct cpl_tx_pkt_core) + pktlen, 16); | ||||
nsegs = 0; | nsegs = 0; | ||||
} | } | ||||
ndesc = howmany(len16, EQ_ESIZE / 16); | ndesc = howmany(len16, EQ_ESIZE / 16); | ||||
MPASS(ndesc <= available); | MPASS(ndesc <= available); | ||||
▲ Show 20 Lines • Show All 1,353 Lines • Show Last 20 Lines |