Page MenuHomeFreeBSD
Differential D54356 Diff 169266 sys/net/iflib.c

Changeset View

Standalone View

View Options

sys/net/iflib.c

Show First 20 Lines • Show All 196 Lines • ▼ Show 20 Linesstruct iflib_ctx {
uint16_t ifc_sysctl_tx_abdicate; uint16_t ifc_sysctl_tx_abdicate;
uint16_t ifc_sysctl_core_offset; uint16_t ifc_sysctl_core_offset;
#define CORE_OFFSET_UNSPECIFIED 0xffff #define CORE_OFFSET_UNSPECIFIED 0xffff
uint8_t ifc_sysctl_separate_txrx; uint8_t ifc_sysctl_separate_txrx;
uint8_t ifc_sysctl_use_logical_cores; uint8_t ifc_sysctl_use_logical_cores;
uint16_t ifc_sysctl_extra_msix_vectors; uint16_t ifc_sysctl_extra_msix_vectors;
bool ifc_cpus_are_physical_cores; bool ifc_cpus_are_physical_cores;
bool ifc_sysctl_simple_tx; bool ifc_sysctl_simple_tx;
bool ifc_sysctl_tx_defer_mfree;
uint16_t ifc_sysctl_tx_reclaim_thresh; uint16_t ifc_sysctl_tx_reclaim_thresh;
uint16_t ifc_sysctl_tx_reclaim_ticks; uint16_t ifc_sysctl_tx_reclaim_ticks;
qidx_t ifc_sysctl_ntxds[8]; qidx_t ifc_sysctl_ntxds[8];
qidx_t ifc_sysctl_nrxds[8]; qidx_t ifc_sysctl_nrxds[8];
struct if_txrx ifc_txrx; struct if_txrx ifc_txrx;
#define isc_txd_encap ifc_txrx.ift_txd_encap #define isc_txd_encap ifc_txrx.ift_txd_encap
#define isc_txd_flush ifc_txrx.ift_txd_flush #define isc_txd_flush ifc_txrx.ift_txd_flush
▲ Show 20 Lines • Show All 80 Lines • ▼ Show 20 Linestypedef struct iflib_sw_rx_desc_array {
caddr_t *ifsd_cl; /* direct cluster pointer for rx */ caddr_t *ifsd_cl; /* direct cluster pointer for rx */
bus_addr_t *ifsd_ba; /* bus addr of cluster for rx */ bus_addr_t *ifsd_ba; /* bus addr of cluster for rx */
} iflib_rxsd_array_t; } iflib_rxsd_array_t;
typedef struct iflib_sw_tx_desc_array { typedef struct iflib_sw_tx_desc_array {
bus_dmamap_t *ifsd_map; /* bus_dma maps for packet */ bus_dmamap_t *ifsd_map; /* bus_dma maps for packet */
bus_dmamap_t *ifsd_tso_map; /* bus_dma maps for TSO packet */ bus_dmamap_t *ifsd_tso_map; /* bus_dma maps for TSO packet */
struct mbuf **ifsd_m; /* pkthdr mbufs */ struct mbuf **ifsd_m; /* pkthdr mbufs */
struct mbuf **ifsd_m_defer; /* deferred mbuf ptr */
struct mbuf **ifsd_m_deferb;/* deferred mbuf backing ptr */
} if_txsd_vec_t; } if_txsd_vec_t;
/* magic number that should be high enough for any hardware */ /* magic number that should be high enough for any hardware */
#define IFLIB_MAX_TX_SEGS 128 #define IFLIB_MAX_TX_SEGS 128
#define IFLIB_RX_COPY_THRESH 128 #define IFLIB_RX_COPY_THRESH 128
#define IFLIB_MAX_RX_REFRESH 32 #define IFLIB_MAX_RX_REFRESH 32
/* The minimum descriptors per second before we start coalescing */ /* The minimum descriptors per second before we start coalescing */
#define IFLIB_MIN_DESC_SEC 16384 #define IFLIB_MIN_DESC_SEC 16384
#define IFLIB_DEFAULT_TX_UPDATE_FREQ 16 #define IFLIB_DEFAULT_TX_UPDATE_FREQ 16
#define IFLIB_QUEUE_IDLE 0 #define IFLIB_QUEUE_IDLE 0
#define IFLIB_QUEUE_HUNG 1 #define IFLIB_QUEUE_HUNG 1
#define IFLIB_QUEUE_WORKING 2 #define IFLIB_QUEUE_WORKING 2
/* maximum number of txqs that can share an rx interrupt */ /* maximum number of txqs that can share an rx interrupt */
#define IFLIB_MAX_TX_SHARED_INTR 4 #define IFLIB_MAX_TX_SHARED_INTR 4
/* this should really scale with ring size - this is a fairly arbitrary value */ /* this should really scale with ring size - this is a fairly arbitrary value */
#define TX_BATCH_SIZE 32 #define TX_BATCH_SIZE 32
#define IFLIB_RESTART_BUDGET 8 #define IFLIB_RESTART_BUDGET 8
/*
* Encode TSO or !TSO in the low bits of the tx ifsd_m pointer so as
* to avoid defref'ing the mbuf to determine the correct busdma resources
* to release
*/
#define IFLIB_TSO (1ULL << 0)
#define IFLIB_NO_TSO (2ULL << 0)
#define IFLIB_FLAGS_MASK (0x3ULL)
#define IFLIB_SAVE_MBUF(mbuf, flags) ((void *)(((uintptr_t)mbuf) | flags))
#define IFLIB_GET_FLAGS(a) ((uintptr_t)a & IFLIB_FLAGS_MASK)
#define IFLIB_GET_MBUF(a) ((struct mbuf *)((uintptr_t)a & ~IFLIB_FLAGS_MASK))
#define IFC_LEGACY 0x001 #define IFC_LEGACY 0x001
#define IFC_QFLUSH 0x002 #define IFC_QFLUSH 0x002
#define IFC_MULTISEG 0x004 #define IFC_MULTISEG 0x004
#define IFC_SPARE1 0x008 #define IFC_SPARE1 0x008
#define IFC_SC_ALLOCATED 0x010 #define IFC_SC_ALLOCATED 0x010
#define IFC_INIT_DONE 0x020 #define IFC_INIT_DONE 0x020
#define IFC_PREFETCH 0x040 #define IFC_PREFETCH 0x040
#define IFC_DO_RESET 0x080 #define IFC_DO_RESET 0x080
Show All 9 Lines#define CSUM_OFFLOAD (CSUM_IP_TSO | CSUM_IP6_TSO | CSUM_IP | \
CSUM_IP6_UDP | CSUM_IP6_TCP | CSUM_IP6_SCTP) CSUM_IP6_UDP | CSUM_IP6_TCP | CSUM_IP6_SCTP)
struct iflib_txq { struct iflib_txq {
qidx_t ift_in_use; qidx_t ift_in_use;
qidx_t ift_cidx; qidx_t ift_cidx;
qidx_t ift_cidx_processed; qidx_t ift_cidx_processed;
qidx_t ift_pidx; qidx_t ift_pidx;
uint8_t ift_gen; uint8_t ift_gen;
uint8_t ift_br_offset; uint8_t ift_br_offset:1,
ift_defer_mfree:1,
ift_spare_bits0:6;
uint16_t ift_npending; uint16_t ift_npending;
uint16_t ift_db_pending; uint16_t ift_db_pending;
uint16_t ift_rs_pending; uint16_t ift_rs_pending;
uint32_t ift_last_reclaim; uint32_t ift_last_reclaim;
uint16_t ift_reclaim_thresh; uint16_t ift_reclaim_thresh;
uint16_t ift_reclaim_ticks; uint16_t ift_reclaim_ticks;
uint8_t ift_txd_size[8]; uint8_t ift_txd_size[8];
uint64_t ift_processed; uint64_t ift_processed;
▲ Show 20 Lines • Show All 375 Lines • ▼ Show 20 Lines
static void iflib_ifmp_purge(iflib_txq_t txq); static void iflib_ifmp_purge(iflib_txq_t txq);
static void _iflib_pre_assert(if_softc_ctx_t scctx); static void _iflib_pre_assert(if_softc_ctx_t scctx);
static void iflib_stop(if_ctx_t ctx); static void iflib_stop(if_ctx_t ctx);
static void iflib_if_init_locked(if_ctx_t ctx); static void iflib_if_init_locked(if_ctx_t ctx);
static void iflib_free_intr_mem(if_ctx_t ctx); static void iflib_free_intr_mem(if_ctx_t ctx);
#ifndef __NO_STRICT_ALIGNMENT #ifndef __NO_STRICT_ALIGNMENT
static struct mbuf *iflib_fixup_rx(struct mbuf *m); static struct mbuf *iflib_fixup_rx(struct mbuf *m);
#endif #endif
static __inline int iflib_completed_tx_reclaim(iflib_txq_t txq); static __inline int iflib_completed_tx_reclaim(iflib_txq_t txq,
struct mbuf **m_defer);
static SLIST_HEAD(cpu_offset_list, cpu_offset) cpu_offsets = static SLIST_HEAD(cpu_offset_list, cpu_offset) cpu_offsets =
SLIST_HEAD_INITIALIZER(cpu_offsets); SLIST_HEAD_INITIALIZER(cpu_offsets);
struct cpu_offset { struct cpu_offset {
SLIST_ENTRY(cpu_offset) entries; SLIST_ENTRY(cpu_offset) entries;
cpuset_t set; cpuset_t set;
unsigned int refcount; unsigned int refcount;
uint16_t next_cpuid; uint16_t next_cpuid;
▲ Show 20 Lines • Show All 1,034 Lines • ▼ Show 20 Linesiflib_txsd_alloc(iflib_txq_t txq)
/* Allocate memory for the TX mbuf map. */ /* Allocate memory for the TX mbuf map. */
if (!(txq->ift_sds.ifsd_m = if (!(txq->ift_sds.ifsd_m =
(struct mbuf **) malloc(sizeof(struct mbuf *) * (struct mbuf **) malloc(sizeof(struct mbuf *) *
scctx->isc_ntxd[txq->ift_br_offset], M_IFLIB, M_NOWAIT | M_ZERO))) { scctx->isc_ntxd[txq->ift_br_offset], M_IFLIB, M_NOWAIT | M_ZERO))) {
device_printf(dev, "Unable to allocate TX mbuf map memory\n"); device_printf(dev, "Unable to allocate TX mbuf map memory\n");
err = ENOMEM; err = ENOMEM;
goto fail; goto fail;
} }
if (ctx->ifc_sysctl_simple_tx) {
if (!(txq->ift_sds.ifsd_m_defer =
(struct mbuf **) malloc(sizeof(struct mbuf *) *
scctx->isc_ntxd[txq->ift_br_offset], M_IFLIB, M_NOWAIT | M_ZERO))) {
device_printf(dev, "Unable to allocate TX mbuf map memory\n");
err = ENOMEM;
goto fail;
}
}
txq->ift_sds.ifsd_m_deferb = txq->ift_sds.ifsd_m_defer;
/* /*
* Create the DMA maps for TX buffers. * Create the DMA maps for TX buffers.
*/ */
if ((txq->ift_sds.ifsd_map = (bus_dmamap_t *)malloc( if ((txq->ift_sds.ifsd_map = (bus_dmamap_t *)malloc(
sizeof(bus_dmamap_t) * scctx->isc_ntxd[txq->ift_br_offset], sizeof(bus_dmamap_t) * scctx->isc_ntxd[txq->ift_br_offset],
M_IFLIB, M_NOWAIT | M_ZERO)) == NULL) { M_IFLIB, M_NOWAIT | M_ZERO)) == NULL) {
device_printf(dev, device_printf(dev,
"Unable to allocate TX buffer DMA map memory\n"); "Unable to allocate TX buffer DMA map memory\n");
▲ Show 20 Lines • Show All 76 Lines • ▼ Show 20 Linesiflib_txq_destroy(iflib_txq_t txq)
if (txq->ift_sds.ifsd_tso_map != NULL) { if (txq->ift_sds.ifsd_tso_map != NULL) {
free(txq->ift_sds.ifsd_tso_map, M_IFLIB); free(txq->ift_sds.ifsd_tso_map, M_IFLIB);
txq->ift_sds.ifsd_tso_map = NULL; txq->ift_sds.ifsd_tso_map = NULL;
} }
if (txq->ift_sds.ifsd_m != NULL) { if (txq->ift_sds.ifsd_m != NULL) {
free(txq->ift_sds.ifsd_m, M_IFLIB); free(txq->ift_sds.ifsd_m, M_IFLIB);
txq->ift_sds.ifsd_m = NULL; txq->ift_sds.ifsd_m = NULL;
} }
if (txq->ift_sds.ifsd_m_defer != NULL) {
free(txq->ift_sds.ifsd_m_defer, M_IFLIB);
txq->ift_sds.ifsd_m_defer = NULL;
}
if (txq->ift_buf_tag != NULL) { if (txq->ift_buf_tag != NULL) {
bus_dma_tag_destroy(txq->ift_buf_tag); bus_dma_tag_destroy(txq->ift_buf_tag);
txq->ift_buf_tag = NULL; txq->ift_buf_tag = NULL;
} }
if (txq->ift_tso_buf_tag != NULL) { if (txq->ift_tso_buf_tag != NULL) {
bus_dma_tag_destroy(txq->ift_tso_buf_tag); bus_dma_tag_destroy(txq->ift_tso_buf_tag);
txq->ift_tso_buf_tag = NULL; txq->ift_tso_buf_tag = NULL;
} }
if (txq->ift_ifdi != NULL) { if (txq->ift_ifdi != NULL) {
free(txq->ift_ifdi, M_IFLIB); free(txq->ift_ifdi, M_IFLIB);
} }
} }
static void static void
iflib_txsd_free(if_ctx_t ctx, iflib_txq_t txq, int i) iflib_txsd_free(if_ctx_t ctx, iflib_txq_t txq, int i)
{ {
struct mbuf **mp; struct mbuf *m;
mp = &txq->ift_sds.ifsd_m[i]; m = IFLIB_GET_MBUF(txq->ift_sds.ifsd_m[i]);
if (*mp == NULL) if (m == NULL)
return; return;
if (txq->ift_sds.ifsd_map != NULL) { if (txq->ift_sds.ifsd_map != NULL) {
bus_dmamap_sync(txq->ift_buf_tag, bus_dmamap_sync(txq->ift_buf_tag,
txq->ift_sds.ifsd_map[i], BUS_DMASYNC_POSTWRITE); txq->ift_sds.ifsd_map[i], BUS_DMASYNC_POSTWRITE);
bus_dmamap_unload(txq->ift_buf_tag, txq->ift_sds.ifsd_map[i]); bus_dmamap_unload(txq->ift_buf_tag, txq->ift_sds.ifsd_map[i]);
} }
if (txq->ift_sds.ifsd_tso_map != NULL) { if (txq->ift_sds.ifsd_tso_map != NULL) {
bus_dmamap_sync(txq->ift_tso_buf_tag, bus_dmamap_sync(txq->ift_tso_buf_tag,
txq->ift_sds.ifsd_tso_map[i], BUS_DMASYNC_POSTWRITE); txq->ift_sds.ifsd_tso_map[i], BUS_DMASYNC_POSTWRITE);
bus_dmamap_unload(txq->ift_tso_buf_tag, bus_dmamap_unload(txq->ift_tso_buf_tag,
txq->ift_sds.ifsd_tso_map[i]); txq->ift_sds.ifsd_tso_map[i]);
} }
m_freem(*mp); m_freem(m);
DBG_COUNTER_INC(tx_frees); DBG_COUNTER_INC(tx_frees);
*mp = NULL;
} }
static int static int
iflib_txq_setup(iflib_txq_t txq) iflib_txq_setup(iflib_txq_t txq)
{ {
if_ctx_t ctx = txq->ift_ctx; if_ctx_t ctx = txq->ift_ctx;
if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
if_shared_ctx_t sctx = ctx->ifc_sctx; if_shared_ctx_t sctx = ctx->ifc_sctx;
▲ Show 20 Lines • Show All 1,509 Lines • ▼ Show 20 Lines
{ {
int ntxd, pidx; int ntxd, pidx;
struct mbuf *m, **ifsd_m; struct mbuf *m, **ifsd_m;
ifsd_m = txq->ift_sds.ifsd_m; ifsd_m = txq->ift_sds.ifsd_m;
ntxd = txq->ift_size; ntxd = txq->ift_size;
pidx = txq->ift_pidx & (ntxd - 1); pidx = txq->ift_pidx & (ntxd - 1);
ifsd_m = txq->ift_sds.ifsd_m; ifsd_m = txq->ift_sds.ifsd_m;
m = ifsd_m[pidx]; m = IFLIB_GET_MBUF(ifsd_m[pidx]);
ifsd_m[pidx] = NULL; ifsd_m[pidx] = NULL;
bus_dmamap_unload(txq->ift_buf_tag, txq->ift_sds.ifsd_map[pidx]); bus_dmamap_unload(txq->ift_buf_tag, txq->ift_sds.ifsd_map[pidx]);
if (txq->ift_sds.ifsd_tso_map != NULL) if (txq->ift_sds.ifsd_tso_map != NULL)
bus_dmamap_unload(txq->ift_tso_buf_tag, bus_dmamap_unload(txq->ift_tso_buf_tag,
txq->ift_sds.ifsd_tso_map[pidx]); txq->ift_sds.ifsd_tso_map[pidx]);
#if MEMORY_LOGGING #if MEMORY_LOGGING
txq->ift_dequeued++; txq->ift_dequeued++;
#endif #endif
▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Linesiflib_encap(iflib_txq_t txq, struct mbuf **m_headp)
if_ctx_t ctx; if_ctx_t ctx;
if_shared_ctx_t sctx; if_shared_ctx_t sctx;
if_softc_ctx_t scctx; if_softc_ctx_t scctx;
bus_dma_tag_t buf_tag; bus_dma_tag_t buf_tag;
bus_dma_segment_t *segs; bus_dma_segment_t *segs;
struct mbuf *m_head, **ifsd_m; struct mbuf *m_head, **ifsd_m;
bus_dmamap_t map; bus_dmamap_t map;
struct if_pkt_info pi; struct if_pkt_info pi;
uintptr_t flags;
int remap = 0; int remap = 0;
int err, nsegs, ndesc, max_segs, pidx; int err, nsegs, ndesc, max_segs, pidx;
ctx = txq->ift_ctx; ctx = txq->ift_ctx;
sctx = ctx->ifc_sctx; sctx = ctx->ifc_sctx;
scctx = &ctx->ifc_softc_ctx; scctx = &ctx->ifc_softc_ctx;
segs = txq->ift_segs; segs = txq->ift_segs;
m_head = *m_headp; m_head = *m_headp;
map = NULL; map = NULL;
/* /*
* If we're doing TSO the next descriptor to clean may be quite far ahead * If we're doing TSO the next descriptor to clean may be quite far ahead
*/ */
pidx = txq->ift_pidx; pidx = txq->ift_pidx;
map = txq->ift_sds.ifsd_map[pidx]; map = txq->ift_sds.ifsd_map[pidx];
ifsd_m = txq->ift_sds.ifsd_m; ifsd_m = txq->ift_sds.ifsd_m;
if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
buf_tag = txq->ift_tso_buf_tag; buf_tag = txq->ift_tso_buf_tag;
max_segs = scctx->isc_tx_tso_segments_max; max_segs = scctx->isc_tx_tso_segments_max;
map = txq->ift_sds.ifsd_tso_map[pidx]; map = txq->ift_sds.ifsd_tso_map[pidx];
MPASS(buf_tag != NULL); MPASS(buf_tag != NULL);
MPASS(max_segs > 0); MPASS(max_segs > 0);
flags = IFLIB_TSO;
} else { } else {
buf_tag = txq->ift_buf_tag; buf_tag = txq->ift_buf_tag;
max_segs = scctx->isc_tx_nsegments; max_segs = scctx->isc_tx_nsegments;
map = txq->ift_sds.ifsd_map[pidx]; map = txq->ift_sds.ifsd_map[pidx];
flags = IFLIB_NO_TSO;
} }
if ((sctx->isc_flags & IFLIB_NEED_ETHER_PAD) && if ((sctx->isc_flags & IFLIB_NEED_ETHER_PAD) &&
__predict_false(m_head->m_pkthdr.len < scctx->isc_min_frame_size)) { __predict_false(m_head->m_pkthdr.len < scctx->isc_min_frame_size)) {
err = iflib_ether_pad(ctx->ifc_dev, m_headp, scctx->isc_min_frame_size); err = iflib_ether_pad(ctx->ifc_dev, m_headp, scctx->isc_min_frame_size);
if (err) { if (err) {
DBG_COUNTER_INC(encap_txd_encap_fail); DBG_COUNTER_INC(encap_txd_encap_fail);
return (err); return (err);
} }
▲ Show 20 Lines • Show All 56 Lines • ▼ Show 20 Lines default:
*m_headp = NULL; *m_headp = NULL;
break; break;
} }
txq->ift_map_failed++; txq->ift_map_failed++;
DBG_COUNTER_INC(encap_load_mbuf_fail); DBG_COUNTER_INC(encap_load_mbuf_fail);
DBG_COUNTER_INC(encap_txd_encap_fail); DBG_COUNTER_INC(encap_txd_encap_fail);
return (err); return (err);
} }
ifsd_m[pidx] = m_head; ifsd_m[pidx] = IFLIB_SAVE_MBUF(m_head, flags);
if (m_head->m_pkthdr.csum_flags & CSUM_SND_TAG) if (m_head->m_pkthdr.csum_flags & CSUM_SND_TAG)
pi.ipi_mbuf = m_head; pi.ipi_mbuf = m_head;
else else
pi.ipi_mbuf = NULL; pi.ipi_mbuf = NULL;
/* /*
* XXX assumes a 1 to 1 relationship between segments and * XXX assumes a 1 to 1 relationship between segments and
* descriptors - this does not hold true on all drivers, e.g. * descriptors - this does not hold true on all drivers, e.g.
* cxgb * cxgb
*/ */
if (__predict_false(nsegs > TXQ_AVAIL(txq))) { if (__predict_false(nsegs > TXQ_AVAIL(txq))) {
(void)iflib_completed_tx_reclaim(txq); (void)iflib_completed_tx_reclaim(txq, NULL);
if (__predict_false(nsegs > TXQ_AVAIL(txq))) { if (__predict_false(nsegs > TXQ_AVAIL(txq))) {
txq->ift_no_desc_avail++; txq->ift_no_desc_avail++;
bus_dmamap_unload(buf_tag, map); bus_dmamap_unload(buf_tag, map);
DBG_COUNTER_INC(encap_txq_avail_fail); DBG_COUNTER_INC(encap_txq_avail_fail);
DBG_COUNTER_INC(encap_txd_encap_fail); DBG_COUNTER_INC(encap_txd_encap_fail);
if (ctx->ifc_sysctl_simple_tx) { if (ctx->ifc_sysctl_simple_tx) {
*m_headp = m_head = iflib_remove_mbuf(txq); *m_headp = m_head = iflib_remove_mbuf(txq);
m_freem(*m_headp); m_freem(*m_headp);
▲ Show 20 Lines • Show All 73 Lines • ▼ Show 20 Linesdefrag_failed:
m_freem(*m_headp); m_freem(*m_headp);
DBG_COUNTER_INC(tx_frees); DBG_COUNTER_INC(tx_frees);
*m_headp = NULL; *m_headp = NULL;
DBG_COUNTER_INC(encap_txd_encap_fail); DBG_COUNTER_INC(encap_txd_encap_fail);
return (ENOMEM); return (ENOMEM);
} }
static void static void
iflib_tx_desc_free(iflib_txq_t txq, int n) iflib_tx_desc_free(iflib_txq_t txq, int n, struct mbuf **m_defer)
{ {
uint32_t qsize, cidx, gen; uint32_t qsize, cidx, gen;
struct mbuf *m, **ifsd_m; struct mbuf *m, **ifsd_m;
uintptr_t flags;
cidx = txq->ift_cidx; cidx = txq->ift_cidx;
gen = txq->ift_gen; gen = txq->ift_gen;
qsize = txq->ift_size; qsize = txq->ift_size;
ifsd_m = txq->ift_sds.ifsd_m; ifsd_m =txq->ift_sds.ifsd_m;
while (n-- > 0) { while (n-- > 0) {
if ((m = ifsd_m[cidx]) != NULL) { if ((m = IFLIB_GET_MBUF(ifsd_m[cidx])) != NULL) {
if (m->m_pkthdr.csum_flags & CSUM_TSO) { flags = IFLIB_GET_FLAGS(ifsd_m[cidx]);
MPASS(flags != 0);
if (flags & IFLIB_TSO) {
bus_dmamap_sync(txq->ift_tso_buf_tag, bus_dmamap_sync(txq->ift_tso_buf_tag,
txq->ift_sds.ifsd_tso_map[cidx], txq->ift_sds.ifsd_tso_map[cidx],
BUS_DMASYNC_POSTWRITE); BUS_DMASYNC_POSTWRITE);
bus_dmamap_unload(txq->ift_tso_buf_tag, bus_dmamap_unload(txq->ift_tso_buf_tag,
txq->ift_sds.ifsd_tso_map[cidx]); txq->ift_sds.ifsd_tso_map[cidx]);
} else { } else {
bus_dmamap_sync(txq->ift_buf_tag, bus_dmamap_sync(txq->ift_buf_tag,
txq->ift_sds.ifsd_map[cidx], txq->ift_sds.ifsd_map[cidx],
BUS_DMASYNC_POSTWRITE); BUS_DMASYNC_POSTWRITE);
bus_dmamap_unload(txq->ift_buf_tag, bus_dmamap_unload(txq->ift_buf_tag,
txq->ift_sds.ifsd_map[cidx]); txq->ift_sds.ifsd_map[cidx]);
} }
/* XXX we don't support any drivers that batch packets yet */ /* XXX we don't support any drivers that batch packets yet */
MPASS(m->m_nextpkt == NULL); MPASS(m->m_nextpkt == NULL);
if (m_defer == NULL) {
m_freem(m); m_freem(m);
} else if (m != NULL) {
*m_defer = m;
m_defer++;
}
ifsd_m[cidx] = NULL; ifsd_m[cidx] = NULL;
#if MEMORY_LOGGING #if MEMORY_LOGGING
txq->ift_dequeued++; txq->ift_dequeued++;
#endif #endif
DBG_COUNTER_INC(tx_frees); DBG_COUNTER_INC(tx_frees);
} }
if (__predict_false(++cidx == qsize)) { if (__predict_false(++cidx == qsize)) {
cidx = 0; cidx = 0;
gen = 0; gen = 0;
} }
} }
txq->ift_cidx = cidx; txq->ift_cidx = cidx;
txq->ift_gen = gen; txq->ift_gen = gen;
} }
static __inline int static __inline int
iflib_completed_tx_reclaim(iflib_txq_t txq) iflib_txq_can_reclaim(iflib_txq_t txq)
{ {
int reclaim, thresh; int reclaim, thresh;
uint32_t now;
if_ctx_t ctx = txq->ift_ctx;
thresh = txq->ift_reclaim_thresh; thresh = txq->ift_reclaim_thresh;
KASSERT(thresh >= 0, ("invalid threshold to reclaim")); KASSERT(thresh >= 0, ("invalid threshold to reclaim"));
MPASS(thresh /*+ MAX_TX_DESC(txq->ift_ctx) */ < txq->ift_size); MPASS(thresh /*+ MAX_TX_DESC(txq->ift_ctx) */ < txq->ift_size);
now = ticks; if (ticks <= (txq->ift_last_reclaim + txq->ift_reclaim_ticks) &&
if (now <= (txq->ift_last_reclaim + txq->ift_reclaim_ticks) &&
txq->ift_in_use < thresh) txq->ift_in_use < thresh)
return (0); return (false);
txq->ift_last_reclaim = now; iflib_tx_credits_update(txq->ift_ctx, txq);
/*
* Need a rate-limiting check so that this isn't called every time
*/
iflib_tx_credits_update(ctx, txq);
reclaim = DESC_RECLAIMABLE(txq); reclaim = DESC_RECLAIMABLE(txq);
if (reclaim <= thresh) {
if (reclaim <= thresh /* + MAX_TX_DESC(txq->ift_ctx) */) {
#ifdef INVARIANTS #ifdef INVARIANTS
if (iflib_verbose_debug) { if (iflib_verbose_debug) {
printf("%s processed=%ju cleaned=%ju tx_nsegments=%d reclaim=%d thresh=%d\n", __func__, printf("%s processed=%ju cleaned=%ju tx_nsegments=%d reclaim=%d thresh=%d\n", __func__,
txq->ift_processed, txq->ift_cleaned, txq->ift_ctx->ifc_softc_ctx.isc_tx_nsegments, txq->ift_processed, txq->ift_cleaned, txq->ift_ctx->ifc_softc_ctx.isc_tx_nsegments,
reclaim, thresh); reclaim, thresh);
} }
#endif #endif
return (0); return (0);
} }
iflib_tx_desc_free(txq, reclaim); return (reclaim);
}
static __inline void
_iflib_completed_tx_reclaim(iflib_txq_t txq, struct mbuf **m_defer, int reclaim)
{
txq->ift_last_reclaim = ticks;
iflib_tx_desc_free(txq, reclaim, m_defer);
txq->ift_cleaned += reclaim; txq->ift_cleaned += reclaim;
txq->ift_in_use -= reclaim; txq->ift_in_use -= reclaim;
}
static __inline int
iflib_completed_tx_reclaim(iflib_txq_t txq, struct mbuf **m_defer)
{
int reclaim;
reclaim = iflib_txq_can_reclaim(txq);
if (reclaim == 0)
return (0);
_iflib_completed_tx_reclaim(txq, m_defer, reclaim);
return (reclaim); return (reclaim);
} }
static struct mbuf ** static struct mbuf **
_ring_peek_one(struct ifmp_ring *r, int cidx, int offset, int remaining) _ring_peek_one(struct ifmp_ring *r, int cidx, int offset, int remaining)
{ {
int next, size; int next, size;
struct mbuf **items; struct mbuf **items;
▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Linesiflib_txq_drain(struct ifmp_ring *r, uint32_t cidx, uint32_t pidx)
int mcast_sent, pkt_sent, reclaimed; int mcast_sent, pkt_sent, reclaimed;
bool do_prefetch, rang, ring; bool do_prefetch, rang, ring;
if (__predict_false(!(if_getdrvflags(ifp) & IFF_DRV_RUNNING) || if (__predict_false(!(if_getdrvflags(ifp) & IFF_DRV_RUNNING) ||
!LINK_ACTIVE(ctx))) { !LINK_ACTIVE(ctx))) {
DBG_COUNTER_INC(txq_drain_notready); DBG_COUNTER_INC(txq_drain_notready);
return (0); return (0);
} }
reclaimed = iflib_completed_tx_reclaim(txq); reclaimed = iflib_completed_tx_reclaim(txq, NULL);
rang = iflib_txd_db_check(txq, reclaimed && txq->ift_db_pending); rang = iflib_txd_db_check(txq, reclaimed && txq->ift_db_pending);
avail = IDXDIFF(pidx, cidx, r->size); avail = IDXDIFF(pidx, cidx, r->size);
if (__predict_false(ctx->ifc_flags & IFC_QFLUSH)) { if (__predict_false(ctx->ifc_flags & IFC_QFLUSH)) {
/* /*
* The driver is unloading so we need to free all pending packets. * The driver is unloading so we need to free all pending packets.
*/ */
DBG_COUNTER_INC(txq_drain_flushing); DBG_COUNTER_INC(txq_drain_flushing);
▲ Show 20 Lines • Show All 142 Lines • ▼ Show 20 Lines if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING))
return; return;
#ifdef DEV_NETMAP #ifdef DEV_NETMAP
if ((if_getcapenable(ifp) & IFCAP_NETMAP) && if ((if_getcapenable(ifp) & IFCAP_NETMAP) &&
netmap_tx_irq(ifp, txq->ift_id)) netmap_tx_irq(ifp, txq->ift_id))
goto skip_ifmp; goto skip_ifmp;
#endif #endif
if (ctx->ifc_sysctl_simple_tx) { if (ctx->ifc_sysctl_simple_tx) {
mtx_lock(&txq->ift_mtx); mtx_lock(&txq->ift_mtx);
(void)iflib_completed_tx_reclaim(txq); (void)iflib_completed_tx_reclaim(txq, NULL);
mtx_unlock(&txq->ift_mtx); mtx_unlock(&txq->ift_mtx);
goto skip_ifmp; goto skip_ifmp;
} }
#ifdef ALTQ #ifdef ALTQ
if (if_altq_is_enabled(ifp)) if (if_altq_is_enabled(ifp))
iflib_altq_if_start(ifp); iflib_altq_if_start(ifp);
#endif #endif
if (txq->ift_db_pending) if (txq->ift_db_pending)
▲ Show 20 Lines • Show All 2,795 Lines • ▼ Show 20 Linesiflib_handle_tx_reclaim_ticks(SYSCTL_HANDLER_ARGS)
txq = &ctx->ifc_txqs[0]; txq = &ctx->ifc_txqs[0];
for (i = 0; i < NTXQSETS(ctx); i++, txq++) { for (i = 0; i < NTXQSETS(ctx); i++, txq++) {
txq->ift_reclaim_ticks = ticks; txq->ift_reclaim_ticks = ticks;
} }
return (err); return (err);
} }
static int
iflib_handle_tx_defer_mfree(SYSCTL_HANDLER_ARGS)
{
if_ctx_t ctx = (void *)arg1;
iflib_txq_t txq;
int i, err;
int defer;
defer = ctx->ifc_sysctl_tx_defer_mfree;
err = sysctl_handle_int(oidp, &defer, arg2, req);
if (err != 0) {
return err;
}
if (defer == ctx->ifc_sysctl_tx_defer_mfree)
return 0;
ctx->ifc_sysctl_tx_defer_mfree = defer;
if (ctx->ifc_txqs == NULL)
return (err);
txq = &ctx->ifc_txqs[0];
for (i = 0; i < NTXQSETS(ctx); i++, txq++) {
txq->ift_defer_mfree = defer;
}
return (err);
}
#define NAME_BUFLEN 32 #define NAME_BUFLEN 32
static void static void
iflib_add_device_sysctl_pre(if_ctx_t ctx) iflib_add_device_sysctl_pre(if_ctx_t ctx)
{ {
device_t dev = iflib_get_dev(ctx); device_t dev = iflib_get_dev(ctx);
struct sysctl_oid_list *child, *oid_list; struct sysctl_oid_list *child, *oid_list;
struct sysctl_ctx_list *ctx_list; struct sysctl_ctx_list *ctx_list;
struct sysctl_oid *node; struct sysctl_oid *node;
▲ Show 20 Lines • Show All 82 Lines • ▼ Show 20 Lines child = SYSCTL_CHILDREN(node);
0, iflib_handle_tx_reclaim_thresh, "I", 0, iflib_handle_tx_reclaim_thresh, "I",
"Number of TX descs outstanding before reclaim is called"); "Number of TX descs outstanding before reclaim is called");
SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "tx_reclaim_ticks", SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "tx_reclaim_ticks",
CTLTYPE_INT | CTLFLAG_RWTUN, ctx, CTLTYPE_INT | CTLFLAG_RWTUN, ctx,
0, iflib_handle_tx_reclaim_ticks, "I", 0, iflib_handle_tx_reclaim_ticks, "I",
"Number of ticks before a TX reclaim is forced"); "Number of ticks before a TX reclaim is forced");
SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "tx_defer_mfree",
CTLTYPE_INT | CTLFLAG_RWTUN, ctx,
0, iflib_handle_tx_defer_mfree, "I",
"Free completed transmits outside of TX ring lock");
if (scctx->isc_ntxqsets > 100) if (scctx->isc_ntxqsets > 100)
qfmt = "txq%03d"; qfmt = "txq%03d";
else if (scctx->isc_ntxqsets > 10) else if (scctx->isc_ntxqsets > 10)
qfmt = "txq%02d"; qfmt = "txq%02d";
else else
qfmt = "txq%d"; qfmt = "txq%d";
for (i = 0, txq = ctx->ifc_txqs; i < scctx->isc_ntxqsets; i++, txq++) { for (i = 0, txq = ctx->ifc_txqs; i < scctx->isc_ntxqsets; i++, txq++) {
snprintf(namebuf, NAME_BUFLEN, qfmt, i); snprintf(namebuf, NAME_BUFLEN, qfmt, i);
▲ Show 20 Lines • Show All 231 Lines • ▼ Show 20 Linesiflib_debugnet_poll(if_t ifp, int count)
ctx = if_getsoftc(ifp); ctx = if_getsoftc(ifp);
scctx = &ctx->ifc_softc_ctx; scctx = &ctx->ifc_softc_ctx;
if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
IFF_DRV_RUNNING) IFF_DRV_RUNNING)
return (EBUSY); return (EBUSY);
txq = &ctx->ifc_txqs[0]; txq = &ctx->ifc_txqs[0];
(void)iflib_completed_tx_reclaim(txq); (void)iflib_completed_tx_reclaim(txq, NULL);
NET_EPOCH_ENTER(et); NET_EPOCH_ENTER(et);
for (i = 0; i < scctx->isc_nrxqsets; i++) for (i = 0; i < scctx->isc_nrxqsets; i++)
(void)iflib_rxeof(&ctx->ifc_rxqs[i], 16 /* XXX */); (void)iflib_rxeof(&ctx->ifc_rxqs[i], 16 /* XXX */);
NET_EPOCH_EXIT(et); NET_EPOCH_EXIT(et);
return (0); return (0);
} }
#endif /* DEBUGNET */ #endif /* DEBUGNET */
Show All 11 Linesiflib_simple_select_queue(if_ctx_t ctx, struct mbuf *m)
return (&ctx->ifc_txqs[qidx]); return (&ctx->ifc_txqs[qidx]);
} }
static int static int
iflib_simple_transmit(if_t ifp, struct mbuf *m) iflib_simple_transmit(if_t ifp, struct mbuf *m)
{ {
if_ctx_t ctx; if_ctx_t ctx;
iflib_txq_t txq; iflib_txq_t txq;
int error; struct mbuf **m_defer;
int error, i, reclaimable;
int bytes_sent = 0, pkt_sent = 0, mcast_sent = 0; int bytes_sent = 0, pkt_sent = 0, mcast_sent = 0;
ctx = if_getsoftc(ifp); ctx = if_getsoftc(ifp);
if (__predict_false((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0 if (__predict_false((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0
|| !LINK_ACTIVE(ctx))) { || !LINK_ACTIVE(ctx))) {
DBG_COUNTER_INC(tx_frees); DBG_COUNTER_INC(tx_frees);
m_freem(m); m_freem(m);
Show All 9 Lines if (error == 0) {
mcast_sent += !!(m->m_flags & M_MCAST); mcast_sent += !!(m->m_flags & M_MCAST);
(void)iflib_txd_db_check(txq, true); (void)iflib_txd_db_check(txq, true);
} else { } else {
if (error == ENOBUFS) if (error == ENOBUFS)
if_inc_counter(ifp, IFCOUNTER_OQDROPS, 1); if_inc_counter(ifp, IFCOUNTER_OQDROPS, 1);
else else
if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
} }
(void)iflib_completed_tx_reclaim(txq); m_defer = NULL;
reclaimable = iflib_txq_can_reclaim(txq);
if (reclaimable != 0) {
/*
* Try to set m_defer to the deferred mbuf reclaim array. If
* we can, the frees will happen outside the tx lock. If we
* can't, it means another thread is still proccessing frees.
*/
if (txq->ift_defer_mfree &&
atomic_cmpset_acq_ptr((uintptr_t *)&txq->ift_sds.ifsd_m_defer,
(uintptr_t )txq->ift_sds.ifsd_m_deferb, 0)) {
m_defer = txq->ift_sds.ifsd_m_deferb;
}
_iflib_completed_tx_reclaim(txq, m_defer, reclaimable);
}
mtx_unlock(&txq->ift_mtx); mtx_unlock(&txq->ift_mtx);
/*
* Process mbuf frees outside the tx lock
*/
if (m_defer != NULL) {
for (i = 0; m_defer[i] != NULL; i++) {
m_freem(m_defer[i]);
m_defer[i] = NULL;
}
atomic_store_rel_ptr((uintptr_t *)&txq->ift_sds.ifsd_m_defer,
(uintptr_t)m_defer);
}
if_inc_counter(ifp, IFCOUNTER_OBYTES, bytes_sent); if_inc_counter(ifp, IFCOUNTER_OBYTES, bytes_sent);
if_inc_counter(ifp, IFCOUNTER_OPACKETS, pkt_sent); if_inc_counter(ifp, IFCOUNTER_OPACKETS, pkt_sent);
if (mcast_sent) if (mcast_sent)
if_inc_counter(ifp, IFCOUNTER_OMCASTS, mcast_sent); if_inc_counter(ifp, IFCOUNTER_OMCASTS, mcast_sent);
return (error); return (error);
} }
#endif #endif