Changeset View
Changeset View
Standalone View
Standalone View
sys/dev/cxgbe/t4_sge.c
Show All 26 Lines | |||||
* SUCH DAMAGE. | * SUCH DAMAGE. | ||||
*/ | */ | ||||
#include <sys/cdefs.h> | #include <sys/cdefs.h> | ||||
__FBSDID("$FreeBSD$"); | __FBSDID("$FreeBSD$"); | ||||
#include "opt_inet.h" | #include "opt_inet.h" | ||||
#include "opt_inet6.h" | #include "opt_inet6.h" | ||||
#include "opt_kern_tls.h" | |||||
#include "opt_ratelimit.h" | #include "opt_ratelimit.h" | ||||
#include <sys/types.h> | #include <sys/types.h> | ||||
#include <sys/eventhandler.h> | #include <sys/eventhandler.h> | ||||
#include <sys/mbuf.h> | #include <sys/mbuf.h> | ||||
#include <sys/socket.h> | #include <sys/socket.h> | ||||
#include <sys/kernel.h> | #include <sys/kernel.h> | ||||
#include <sys/ktls.h> | |||||
#include <sys/malloc.h> | #include <sys/malloc.h> | ||||
#include <sys/queue.h> | #include <sys/queue.h> | ||||
#include <sys/sbuf.h> | #include <sys/sbuf.h> | ||||
#include <sys/taskqueue.h> | #include <sys/taskqueue.h> | ||||
#include <sys/time.h> | #include <sys/time.h> | ||||
#include <sys/sglist.h> | #include <sys/sglist.h> | ||||
#include <sys/sysctl.h> | #include <sys/sysctl.h> | ||||
#include <sys/smp.h> | #include <sys/smp.h> | ||||
#include <sys/socketvar.h> | |||||
#include <sys/counter.h> | #include <sys/counter.h> | ||||
#include <net/bpf.h> | #include <net/bpf.h> | ||||
#include <net/ethernet.h> | #include <net/ethernet.h> | ||||
#include <net/if.h> | #include <net/if.h> | ||||
#include <net/if_vlan_var.h> | #include <net/if_vlan_var.h> | ||||
#include <netinet/in.h> | #include <netinet/in.h> | ||||
#include <netinet/ip.h> | #include <netinet/ip.h> | ||||
#include <netinet/ip6.h> | #include <netinet/ip6.h> | ||||
Show All 22 Lines | |||||
#define RX_COPY_THRESHOLD (MINCLSIZE - 8) | #define RX_COPY_THRESHOLD (MINCLSIZE - 8) | ||||
#else | #else | ||||
#define RX_COPY_THRESHOLD MINCLSIZE | #define RX_COPY_THRESHOLD MINCLSIZE | ||||
#endif | #endif | ||||
/* Internal mbuf flags stored in PH_loc.eight[1]. */ | /* Internal mbuf flags stored in PH_loc.eight[1]. */ | ||||
#define MC_NOMAP 0x01 | #define MC_NOMAP 0x01 | ||||
#define MC_RAW_WR 0x02 | #define MC_RAW_WR 0x02 | ||||
#define MC_TLS 0x04 | |||||
/* | /* | ||||
* Ethernet frames are DMA'd at this byte offset into the freelist buffer. | * Ethernet frames are DMA'd at this byte offset into the freelist buffer. | ||||
* 0-7 are valid values. | * 0-7 are valid values. | ||||
*/ | */ | ||||
static int fl_pktshift = 0; | static int fl_pktshift = 0; | ||||
SYSCTL_INT(_hw_cxgbe, OID_AUTO, fl_pktshift, CTLFLAG_RDTUN, &fl_pktshift, 0, | SYSCTL_INT(_hw_cxgbe, OID_AUTO, fl_pktshift, CTLFLAG_RDTUN, &fl_pktshift, 0, | ||||
"payload DMA offset in rx buffer (bytes)"); | "payload DMA offset in rx buffer (bytes)"); | ||||
▲ Show 20 Lines • Show All 2,139 Lines • ▼ Show 20 Lines | |||||
static inline int | static inline int | ||||
mbuf_len16(struct mbuf *m) | mbuf_len16(struct mbuf *m) | ||||
{ | { | ||||
int n; | int n; | ||||
M_ASSERTPKTHDR(m); | M_ASSERTPKTHDR(m); | ||||
n = m->m_pkthdr.PH_loc.eight[0]; | n = m->m_pkthdr.PH_loc.eight[0]; | ||||
if (!(mbuf_cflags(m) & MC_TLS)) | |||||
MPASS(n > 0 && n <= SGE_MAX_WR_LEN / 16); | MPASS(n > 0 && n <= SGE_MAX_WR_LEN / 16); | ||||
return (n); | return (n); | ||||
} | } | ||||
static inline void | static inline void | ||||
set_mbuf_len16(struct mbuf *m, uint8_t len16) | set_mbuf_len16(struct mbuf *m, uint8_t len16) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 285 Lines • ▼ Show 20 Lines | |||||
{ | { | ||||
struct mbuf *m0 = *mp, *m; | struct mbuf *m0 = *mp, *m; | ||||
int rc, nsegs, defragged = 0, offset; | int rc, nsegs, defragged = 0, offset; | ||||
struct ether_header *eh; | struct ether_header *eh; | ||||
void *l3hdr; | void *l3hdr; | ||||
#if defined(INET) || defined(INET6) | #if defined(INET) || defined(INET6) | ||||
struct tcphdr *tcp; | struct tcphdr *tcp; | ||||
#endif | #endif | ||||
#ifdef RATELIMIT | #if defined(KERN_TLS) || defined(RATELIMIT) | ||||
struct cxgbe_snd_tag *cst; | struct cxgbe_snd_tag *cst; | ||||
#endif | #endif | ||||
uint16_t eh_type; | uint16_t eh_type; | ||||
uint8_t cflags; | uint8_t cflags; | ||||
cflags = 0; | cflags = 0; | ||||
M_ASSERTPKTHDR(m0); | M_ASSERTPKTHDR(m0); | ||||
if (__predict_false(m0->m_pkthdr.len < ETHER_HDR_LEN)) { | if (__predict_false(m0->m_pkthdr.len < ETHER_HDR_LEN)) { | ||||
rc = EINVAL; | rc = EINVAL; | ||||
fail: | fail: | ||||
m_freem(m0); | m_freem(m0); | ||||
*mp = NULL; | *mp = NULL; | ||||
return (rc); | return (rc); | ||||
} | } | ||||
restart: | restart: | ||||
/* | /* | ||||
* First count the number of gather list segments in the payload. | * First count the number of gather list segments in the payload. | ||||
* Defrag the mbuf if nsegs exceeds the hardware limit. | * Defrag the mbuf if nsegs exceeds the hardware limit. | ||||
*/ | */ | ||||
M_ASSERTPKTHDR(m0); | M_ASSERTPKTHDR(m0); | ||||
MPASS(m0->m_pkthdr.len > 0); | MPASS(m0->m_pkthdr.len > 0); | ||||
nsegs = count_mbuf_nsegs(m0, 0, &cflags); | nsegs = count_mbuf_nsegs(m0, 0, &cflags); | ||||
#ifdef RATELIMIT | #if defined(KERN_TLS) || defined(RATELIMIT) | ||||
if (m0->m_pkthdr.csum_flags & CSUM_SND_TAG) | if (m0->m_pkthdr.csum_flags & CSUM_SND_TAG) | ||||
cst = mst_to_cst(m0->m_pkthdr.snd_tag); | cst = mst_to_cst(m0->m_pkthdr.snd_tag); | ||||
else | else | ||||
cst = NULL; | cst = NULL; | ||||
#endif | #endif | ||||
#ifdef KERN_TLS | |||||
if (cst != NULL && cst->type == IF_SND_TAG_TYPE_TLS) { | |||||
int len16; | |||||
cflags |= MC_TLS; | |||||
set_mbuf_cflags(m0, cflags); | |||||
rc = t6_ktls_parse_pkt(m0, &nsegs, &len16); | |||||
if (rc != 0) | |||||
goto fail; | |||||
set_mbuf_nsegs(m0, nsegs); | |||||
set_mbuf_len16(m0, len16); | |||||
return (0); | |||||
} | |||||
#endif | |||||
if (nsegs > (needs_tso(m0) ? TX_SGL_SEGS_TSO : TX_SGL_SEGS)) { | if (nsegs > (needs_tso(m0) ? TX_SGL_SEGS_TSO : TX_SGL_SEGS)) { | ||||
if (defragged++ > 0 || (m = m_defrag(m0, M_NOWAIT)) == NULL) { | if (defragged++ > 0 || (m = m_defrag(m0, M_NOWAIT)) == NULL) { | ||||
rc = EFBIG; | rc = EFBIG; | ||||
goto fail; | goto fail; | ||||
} | } | ||||
*mp = m0 = m; /* update caller's copy after defrag */ | *mp = m0 = m; /* update caller's copy after defrag */ | ||||
goto restart; | goto restart; | ||||
} | } | ||||
▲ Show 20 Lines • Show All 254 Lines • ▼ Show 20 Lines | can_resume_eth_tx(struct mp_ring *r) | ||||
return (total_available_tx_desc(eq) > eq->sidx / 8); | return (total_available_tx_desc(eq) > eq->sidx / 8); | ||||
} | } | ||||
static inline int | static inline int | ||||
cannot_use_txpkts(struct mbuf *m) | cannot_use_txpkts(struct mbuf *m) | ||||
{ | { | ||||
/* maybe put a GL limit too, to avoid silliness? */ | /* maybe put a GL limit too, to avoid silliness? */ | ||||
return (needs_tso(m) || (mbuf_cflags(m) & MC_RAW_WR) != 0); | return (needs_tso(m) || (mbuf_cflags(m) & (MC_RAW_WR | MC_TLS)) != 0); | ||||
} | } | ||||
static inline int | static inline int | ||||
discard_tx(struct sge_eq *eq) | discard_tx(struct sge_eq *eq) | ||||
{ | { | ||||
return ((eq->flags & (EQ_ENABLED | EQ_QFLUSH)) != EQ_ENABLED); | return ((eq->flags & (EQ_ENABLED | EQ_QFLUSH)) != EQ_ENABLED); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 59 Lines • ▼ Show 20 Lines | eth_tx(struct mp_ring *r, u_int cidx, u_int pidx) | ||||
dbdiff = IDXDIFF(eq->pidx, eq->dbidx, eq->sidx); | dbdiff = IDXDIFF(eq->pidx, eq->dbidx, eq->sidx); | ||||
while (remaining > 0) { | while (remaining > 0) { | ||||
m0 = r->items[cidx]; | m0 = r->items[cidx]; | ||||
M_ASSERTPKTHDR(m0); | M_ASSERTPKTHDR(m0); | ||||
MPASS(m0->m_nextpkt == NULL); | MPASS(m0->m_nextpkt == NULL); | ||||
if (available < SGE_MAX_WR_NDESC) { | if (available < howmany(mbuf_len16(m0), EQ_ESIZE / 16)) { | ||||
MPASS(howmany(mbuf_len16(m0), EQ_ESIZE / 16) <= 64); | |||||
jhb: This change is because certain TLS mbufs might span multiple work requests that in total… | |||||
available += reclaim_tx_descs(txq, 64); | available += reclaim_tx_descs(txq, 64); | ||||
if (available < howmany(mbuf_len16(m0), EQ_ESIZE / 16)) | if (available < howmany(mbuf_len16(m0), EQ_ESIZE / 16)) | ||||
break; /* out of descriptors */ | break; /* out of descriptors */ | ||||
} | } | ||||
next_cidx = cidx + 1; | next_cidx = cidx + 1; | ||||
if (__predict_false(next_cidx == r->size)) | if (__predict_false(next_cidx == r->size)) | ||||
next_cidx = 0; | next_cidx = 0; | ||||
wr = (void *)&eq->desc[eq->pidx]; | wr = (void *)&eq->desc[eq->pidx]; | ||||
if (sc->flags & IS_VF) { | if (mbuf_cflags(m0) & MC_RAW_WR) { | ||||
total++; | total++; | ||||
remaining--; | remaining--; | ||||
n = write_raw_wr(txq, (void *)wr, m0, available); | |||||
#ifdef KERN_TLS | |||||
} else if (mbuf_cflags(m0) & MC_TLS) { | |||||
total++; | |||||
remaining--; | |||||
ETHER_BPF_MTAP(ifp, m0); | ETHER_BPF_MTAP(ifp, m0); | ||||
n = t6_ktls_write_wr(txq,(void *)wr, m0, | |||||
mbuf_nsegs(m0), available); | |||||
#endif | |||||
} else if (sc->flags & IS_VF) { | |||||
total++; | |||||
remaining--; | |||||
ETHER_BPF_MTAP(ifp, m0); | |||||
n = write_txpkt_vm_wr(sc, txq, (void *)wr, m0, | n = write_txpkt_vm_wr(sc, txq, (void *)wr, m0, | ||||
available); | available); | ||||
} else if (remaining > 1 && | } else if (remaining > 1 && | ||||
try_txpkts(m0, r->items[next_cidx], &txp, available) == 0) { | try_txpkts(m0, r->items[next_cidx], &txp, available) == 0) { | ||||
/* pkts at cidx, next_cidx should both be in txp. */ | /* pkts at cidx, next_cidx should both be in txp. */ | ||||
MPASS(txp.npkt == 2); | MPASS(txp.npkt == 2); | ||||
tail = r->items[next_cidx]; | tail = r->items[next_cidx]; | ||||
Show All 14 Lines | #endif | ||||
ETHER_BPF_MTAP(ifp, tail); | ETHER_BPF_MTAP(ifp, tail); | ||||
if (__predict_false(++next_cidx == r->size)) | if (__predict_false(++next_cidx == r->size)) | ||||
next_cidx = 0; | next_cidx = 0; | ||||
} | } | ||||
n = write_txpkts_wr(txq, wr, m0, &txp, available); | n = write_txpkts_wr(txq, wr, m0, &txp, available); | ||||
total += txp.npkt; | total += txp.npkt; | ||||
remaining -= txp.npkt; | remaining -= txp.npkt; | ||||
} else if (mbuf_cflags(m0) & MC_RAW_WR) { | |||||
total++; | |||||
remaining--; | |||||
n = write_raw_wr(txq, (void *)wr, m0, available); | |||||
} else { | } else { | ||||
total++; | total++; | ||||
remaining--; | remaining--; | ||||
ETHER_BPF_MTAP(ifp, m0); | ETHER_BPF_MTAP(ifp, m0); | ||||
n = write_txpkt_wr(txq, (void *)wr, m0, available); | n = write_txpkt_wr(txq, (void *)wr, m0, available); | ||||
} | } | ||||
MPASS(n >= 1 && n <= available && n <= SGE_MAX_WR_NDESC); | MPASS(n >= 1 && n <= available); | ||||
if (!(mbuf_cflags(m0) & MC_TLS)) | |||||
MPASS(n <= SGE_MAX_WR_NDESC); | |||||
available -= n; | available -= n; | ||||
dbdiff += n; | dbdiff += n; | ||||
IDXINCR(eq->pidx, n, eq->sidx); | IDXINCR(eq->pidx, n, eq->sidx); | ||||
if (wr_can_update_eq(wr)) { | if (wr_can_update_eq(wr)) { | ||||
if (total_available_tx_desc(eq) < eq->sidx / 4 && | if (total_available_tx_desc(eq) < eq->sidx / 4 && | ||||
atomic_cmpset_int(&eq->equiq, 0, 1)) { | atomic_cmpset_int(&eq->equiq, 0, 1)) { | ||||
▲ Show 20 Lines • Show All 1,195 Lines • ▼ Show 20 Lines | alloc_txq(struct vi_info *vi, struct sge_txq *txq, int idx, | ||||
SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "txpkts0_pkts", | SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "txpkts0_pkts", | ||||
CTLFLAG_RD, &txq->txpkts0_pkts, | CTLFLAG_RD, &txq->txpkts0_pkts, | ||||
"# of frames tx'd using type0 txpkts work requests"); | "# of frames tx'd using type0 txpkts work requests"); | ||||
SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "txpkts1_pkts", | SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "txpkts1_pkts", | ||||
CTLFLAG_RD, &txq->txpkts1_pkts, | CTLFLAG_RD, &txq->txpkts1_pkts, | ||||
"# of frames tx'd using type1 txpkts work requests"); | "# of frames tx'd using type1 txpkts work requests"); | ||||
SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "raw_wrs", CTLFLAG_RD, | SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "raw_wrs", CTLFLAG_RD, | ||||
&txq->raw_wrs, "# of raw work requests (non-packets)"); | &txq->raw_wrs, "# of raw work requests (non-packets)"); | ||||
SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "tls_wrs", CTLFLAG_RD, | |||||
&txq->tls_wrs, "# of TLS work requests (TLS records)"); | |||||
#ifdef KERN_TLS | |||||
if (sc->flags & KERN_TLS_OK) { | |||||
SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, | |||||
"kern_tls_records", CTLFLAG_RD, &txq->kern_tls_records, | |||||
"# of NIC TLS records transmitted"); | |||||
SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, | |||||
"kern_tls_short", CTLFLAG_RD, &txq->kern_tls_short, | |||||
"# of short NIC TLS records transmitted"); | |||||
SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, | |||||
"kern_tls_partial", CTLFLAG_RD, &txq->kern_tls_partial, | |||||
"# of partial NIC TLS records transmitted"); | |||||
SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, | |||||
"kern_tls_full", CTLFLAG_RD, &txq->kern_tls_full, | |||||
"# of full NIC TLS records transmitted"); | |||||
SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, | |||||
"kern_tls_octets", CTLFLAG_RD, &txq->kern_tls_octets, | |||||
"# of payload octets in transmitted NIC TLS records"); | |||||
SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, | |||||
"kern_tls_waste", CTLFLAG_RD, &txq->kern_tls_waste, | |||||
"# of octets DMAd but not transmitted in NIC TLS records"); | |||||
SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, | |||||
"kern_tls_options", CTLFLAG_RD, &txq->kern_tls_options, | |||||
"# of NIC TLS options-only packets transmitted"); | |||||
SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, | |||||
"kern_tls_header", CTLFLAG_RD, &txq->kern_tls_header, | |||||
"# of NIC TLS header-only packets transmitted"); | |||||
SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, | |||||
"kern_tls_fin", CTLFLAG_RD, &txq->kern_tls_fin, | |||||
"# of NIC TLS FIN-only packets transmitted"); | |||||
SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, | |||||
"kern_tls_fin_short", CTLFLAG_RD, &txq->kern_tls_fin_short, | |||||
"# of NIC TLS padded FIN packets on short TLS records"); | |||||
SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, | |||||
"kern_tls_cbc", CTLFLAG_RD, &txq->kern_tls_cbc, | |||||
"# of NIC TLS sessions using AES-CBC"); | |||||
SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, | |||||
"kern_tls_gcm", CTLFLAG_RD, &txq->kern_tls_gcm, | |||||
"# of NIC TLS sessions using AES-GCM"); | |||||
} | |||||
#endif | |||||
SYSCTL_ADD_COUNTER_U64(&vi->ctx, children, OID_AUTO, "r_enqueues", | SYSCTL_ADD_COUNTER_U64(&vi->ctx, children, OID_AUTO, "r_enqueues", | ||||
CTLFLAG_RD, &txq->r->enqueues, | CTLFLAG_RD, &txq->r->enqueues, | ||||
"# of enqueues to the mp_ring for this queue"); | "# of enqueues to the mp_ring for this queue"); | ||||
SYSCTL_ADD_COUNTER_U64(&vi->ctx, children, OID_AUTO, "r_drops", | SYSCTL_ADD_COUNTER_U64(&vi->ctx, children, OID_AUTO, "r_drops", | ||||
CTLFLAG_RD, &txq->r->drops, | CTLFLAG_RD, &txq->r->drops, | ||||
"# of drops in the mp_ring for this queue"); | "# of drops in the mp_ring for this queue"); | ||||
SYSCTL_ADD_COUNTER_U64(&vi->ctx, children, OID_AUTO, "r_starts", | SYSCTL_ADD_COUNTER_U64(&vi->ctx, children, OID_AUTO, "r_starts", | ||||
▲ Show 20 Lines • Show All 1,963 Lines • Show Last 20 Lines |
This change is because certain TLS mbufs might span multiple work requests that in total require more than SGE_MAX_WR_NDESC descriptors even though each individual WR is smaller than that limit. This ensures we have enough descriptors for the next mbuf.