Page MenuHomeFreeBSD
Files F142448689

D54356.id168560.diff
No OneTemporary
Actions

D54356.id168560.diff
View Options

diff --git a/FreeBSD/sys/net/iflib.c b/FreeBSD/sys/net/iflib.c
--- a/FreeBSD/sys/net/iflib.c
+++ b/FreeBSD/sys/net/iflib.c
@@ -203,6 +203,7 @@
uint16_t ifc_sysctl_extra_msix_vectors;
bool ifc_cpus_are_physical_cores;
bool ifc_sysctl_simple_tx;
+ bool ifc_sysctl_tx_defer_mfree;
uint16_t ifc_sysctl_tx_reclaim_thresh;
uint16_t ifc_sysctl_tx_reclaim_ticks;
@@ -299,6 +300,8 @@
bus_dmamap_t *ifsd_map; /* bus_dma maps for packet */
bus_dmamap_t *ifsd_tso_map; /* bus_dma maps for TSO packet */
struct mbuf **ifsd_m; /* pkthdr mbufs */
+ struct mbuf **ifsd_m_defer; /* deferred mbuf ptr */
+ struct mbuf **ifsd_m_deferb;/* deferred mbuf backing ptr */
} if_txsd_vec_t;
/* magic number that should be high enough for any hardware */
@@ -319,6 +322,20 @@
#define IFLIB_RESTART_BUDGET 8
+
+/*
+ * Encode TSO or !TSO in the low bits of the tx ifsd_m pointer so as
+ * to avoid defref'ing the mbuf to determine the correct busdma resources
+ * to release
+ */
+#define IFLIB_TSO (1ULL << 0)
+#define IFLIB_NO_TSO (2ULL << 0)
+#define IFLIB_FLAGS_MASK (0x3ULL)
+#define IFLIB_SAVE_MBUF(mbuf, flags) ((void *)(((uintptr_t)mbuf) | flags))
+#define IFLIB_GET_FLAGS(a) ((uintptr_t)a & IFLIB_FLAGS_MASK)
+#define IFLIB_GET_MBUF(a) ((struct mbuf *)((uintptr_t)a & ~IFLIB_FLAGS_MASK))
+
+
#define IFC_LEGACY 0x001
#define IFC_QFLUSH 0x002
#define IFC_MULTISEG 0x004
@@ -344,7 +361,9 @@
qidx_t ift_cidx_processed;
qidx_t ift_pidx;
uint8_t ift_gen;
- uint8_t ift_br_offset;
+ uint8_t ift_br_offset:1,
+ ift_defer_mfree:1,
+ ift_spare_bits0:6;
uint16_t ift_npending;
uint16_t ift_db_pending;
uint16_t ift_rs_pending;
@@ -736,7 +755,8 @@
#ifndef __NO_STRICT_ALIGNMENT
static struct mbuf *iflib_fixup_rx(struct mbuf *m);
#endif
-static __inline int iflib_completed_tx_reclaim(iflib_txq_t txq);
+static __inline int iflib_completed_tx_reclaim(iflib_txq_t txq,
+ struct mbuf **m_defer);
static SLIST_HEAD(cpu_offset_list, cpu_offset) cpu_offsets =
SLIST_HEAD_INITIALIZER(cpu_offsets);
@@ -1787,7 +1807,16 @@
err = ENOMEM;
goto fail;
}
-
+ if (ctx->ifc_sysctl_simple_tx) {
+ if (!(txq->ift_sds.ifsd_m_defer =
+ (struct mbuf **) malloc(sizeof(struct mbuf *) *
+ scctx->isc_ntxd[txq->ift_br_offset], M_IFLIB, M_NOWAIT | M_ZERO))) {
+ device_printf(dev, "Unable to allocate TX mbuf map memory\n");
+ err = ENOMEM;
+ goto fail;
+ }
+ }
+ txq->ift_sds.ifsd_m_deferb = txq->ift_sds.ifsd_m_defer;
/*
* Create the DMA maps for TX buffers.
*/
@@ -1880,6 +1909,10 @@
free(txq->ift_sds.ifsd_m, M_IFLIB);
txq->ift_sds.ifsd_m = NULL;
}
+ if (txq->ift_sds.ifsd_m_defer != NULL) {
+ free(txq->ift_sds.ifsd_m_defer, M_IFLIB);
+ txq->ift_sds.ifsd_m_defer = NULL;
+ }
if (txq->ift_buf_tag != NULL) {
bus_dma_tag_destroy(txq->ift_buf_tag);
txq->ift_buf_tag = NULL;
@@ -1896,10 +1929,10 @@
static void
iflib_txsd_free(if_ctx_t ctx, iflib_txq_t txq, int i)
{
- struct mbuf **mp;
+ struct mbuf *m;
- mp = &txq->ift_sds.ifsd_m[i];
- if (*mp == NULL)
+ m = IFLIB_GET_MBUF(txq->ift_sds.ifsd_m[i]);
+ if (m == NULL)
return;
if (txq->ift_sds.ifsd_map != NULL) {
@@ -1913,9 +1946,8 @@
bus_dmamap_unload(txq->ift_tso_buf_tag,
txq->ift_sds.ifsd_tso_map[i]);
}
- m_freem(*mp);
+ m_freem(m);
DBG_COUNTER_INC(tx_frees);
- *mp = NULL;
}
static int
@@ -3442,7 +3474,7 @@
ntxd = txq->ift_size;
pidx = txq->ift_pidx & (ntxd - 1);
ifsd_m = txq->ift_sds.ifsd_m;
- m = ifsd_m[pidx];
+ m = IFLIB_GET_MBUF(ifsd_m[pidx]);
ifsd_m[pidx] = NULL;
bus_dmamap_unload(txq->ift_buf_tag, txq->ift_sds.ifsd_map[pidx]);
if (txq->ift_sds.ifsd_tso_map != NULL)
@@ -3509,6 +3541,7 @@
struct mbuf *m_head, **ifsd_m;
bus_dmamap_t map;
struct if_pkt_info pi;
+ uintptr_t flags;
int remap = 0;
int err, nsegs, ndesc, max_segs, pidx;
@@ -3532,10 +3565,12 @@
map = txq->ift_sds.ifsd_tso_map[pidx];
MPASS(buf_tag != NULL);
MPASS(max_segs > 0);
+ flags = IFLIB_TSO;
} else {
buf_tag = txq->ift_buf_tag;
max_segs = scctx->isc_tx_nsegments;
map = txq->ift_sds.ifsd_map[pidx];
+ flags = IFLIB_NO_TSO;
}
if ((sctx->isc_flags & IFLIB_NEED_ETHER_PAD) &&
__predict_false(m_head->m_pkthdr.len < scctx->isc_min_frame_size)) {
@@ -3608,7 +3643,7 @@
DBG_COUNTER_INC(encap_txd_encap_fail);
return (err);
}
- ifsd_m[pidx] = m_head;
+ ifsd_m[pidx] = IFLIB_SAVE_MBUF(m_head, flags);
if (m_head->m_pkthdr.csum_flags & CSUM_SND_TAG)
pi.ipi_mbuf = m_head;
else
@@ -3619,7 +3654,7 @@
* cxgb
*/
if (__predict_false(nsegs > TXQ_AVAIL(txq))) {
- (void)iflib_completed_tx_reclaim(txq);
+ (void)iflib_completed_tx_reclaim(txq, NULL);
if (__predict_false(nsegs > TXQ_AVAIL(txq))) {
txq->ift_no_desc_avail++;
bus_dmamap_unload(buf_tag, map);
@@ -3709,19 +3744,22 @@
}
static void
-iflib_tx_desc_free(iflib_txq_t txq, int n)
+iflib_tx_desc_free(iflib_txq_t txq, int n, struct mbuf **m_defer)
{
uint32_t qsize, cidx, gen;
struct mbuf *m, **ifsd_m;
+ uintptr_t flags;
cidx = txq->ift_cidx;
gen = txq->ift_gen;
qsize = txq->ift_size;
- ifsd_m = txq->ift_sds.ifsd_m;
+ ifsd_m =txq->ift_sds.ifsd_m;
while (n-- > 0) {
- if ((m = ifsd_m[cidx]) != NULL) {
- if (m->m_pkthdr.csum_flags & CSUM_TSO) {
+ if ((m = IFLIB_GET_MBUF(ifsd_m[cidx])) != NULL) {
+ flags = IFLIB_GET_FLAGS(ifsd_m[cidx]);
+ MPASS(flags != 0);
+ if (flags & IFLIB_TSO) {
bus_dmamap_sync(txq->ift_tso_buf_tag,
txq->ift_sds.ifsd_tso_map[cidx],
BUS_DMASYNC_POSTWRITE);
@@ -3736,7 +3774,12 @@
}
/* XXX we don't support any drivers that batch packets yet */
MPASS(m->m_nextpkt == NULL);
- m_freem(m);
+ if (m_defer == NULL) {
+ m_freem(m);
+ } else if (m != NULL) {
+ *m_defer = m;
+ m_defer++;
+ }
ifsd_m[cidx] = NULL;
#if MEMORY_LOGGING
txq->ift_dequeued++;
@@ -3753,28 +3796,20 @@
}
static __inline int
-iflib_completed_tx_reclaim(iflib_txq_t txq)
+iflib_txq_can_reclaim(iflib_txq_t txq)
{
int reclaim, thresh;
- uint32_t now;
- if_ctx_t ctx = txq->ift_ctx;
thresh = txq->ift_reclaim_thresh;
KASSERT(thresh >= 0, ("invalid threshold to reclaim"));
MPASS(thresh /*+ MAX_TX_DESC(txq->ift_ctx) */ < txq->ift_size);
- now = ticks;
- if (now <= (txq->ift_last_reclaim + txq->ift_reclaim_ticks) &&
+ if (ticks <= (txq->ift_last_reclaim + txq->ift_reclaim_ticks) &&
txq->ift_in_use < thresh)
- return (0);
- txq->ift_last_reclaim = now;
- /*
- * Need a rate-limiting check so that this isn't called every time
- */
- iflib_tx_credits_update(ctx, txq);
+ return (false);
+ iflib_tx_credits_update(txq->ift_ctx, txq);
reclaim = DESC_RECLAIMABLE(txq);
-
- if (reclaim <= thresh /* + MAX_TX_DESC(txq->ift_ctx) */) {
+ if (reclaim <= thresh) {
#ifdef INVARIANTS
if (iflib_verbose_debug) {
printf("%s processed=%ju cleaned=%ju tx_nsegments=%d reclaim=%d thresh=%d\n", __func__,
@@ -3784,10 +3819,27 @@
#endif
return (0);
}
- iflib_tx_desc_free(txq, reclaim);
+ return (reclaim);
+}
+
+static __inline void
+_iflib_completed_tx_reclaim(iflib_txq_t txq, struct mbuf **m_defer, int reclaim)
+{
+ txq->ift_last_reclaim = ticks;
+ iflib_tx_desc_free(txq, reclaim, m_defer);
txq->ift_cleaned += reclaim;
txq->ift_in_use -= reclaim;
+}
+
+static __inline int
+iflib_completed_tx_reclaim(iflib_txq_t txq, struct mbuf **m_defer)
+{
+ int reclaim;
+ reclaim = iflib_txq_can_reclaim(txq);
+ if (reclaim == 0)
+ return (0);
+ _iflib_completed_tx_reclaim(txq, m_defer, reclaim);
return (reclaim);
}
@@ -3848,7 +3900,7 @@
DBG_COUNTER_INC(txq_drain_notready);
return (0);
}
- reclaimed = iflib_completed_tx_reclaim(txq);
+ reclaimed = iflib_completed_tx_reclaim(txq, NULL);
rang = iflib_txd_db_check(txq, reclaimed && txq->ift_db_pending);
avail = IDXDIFF(pidx, cidx, r->size);
@@ -4007,7 +4059,7 @@
#endif
if (ctx->ifc_sysctl_simple_tx) {
mtx_lock(&txq->ift_mtx);
- (void)iflib_completed_tx_reclaim(txq);
+ (void)iflib_completed_tx_reclaim(txq, NULL);
mtx_unlock(&txq->ift_mtx);
goto skip_ifmp;
}
@@ -6820,6 +6872,34 @@
return (err);
}
+static int
+iflib_handle_tx_defer_mfree(SYSCTL_HANDLER_ARGS)
+{
+ if_ctx_t ctx = (void *)arg1;
+ iflib_txq_t txq;
+ int i, err;
+ int defer;
+
+ defer = ctx->ifc_sysctl_tx_defer_mfree;
+ err = sysctl_handle_int(oidp, &defer, arg2, req);
+ if (err != 0) {
+ return err;
+ }
+
+ if (defer == ctx->ifc_sysctl_tx_defer_mfree)
+ return 0;
+
+ ctx->ifc_sysctl_tx_defer_mfree = defer;
+ if (ctx->ifc_txqs == NULL)
+ return (err);
+
+ txq = &ctx->ifc_txqs[0];
+ for (i = 0; i < NTXQSETS(ctx); i++, txq++) {
+ txq->ift_defer_mfree = defer;
+ }
+ return (err);
+}
+
#define NAME_BUFLEN 32
static void
iflib_add_device_sysctl_pre(if_ctx_t ctx)
@@ -6921,6 +7001,11 @@
0, iflib_handle_tx_reclaim_ticks, "I",
"Number of ticks before a TX reclaim is forced");
+ SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "tx_defer_mfree",
+ CTLTYPE_INT | CTLFLAG_RWTUN, ctx,
+ 0, iflib_handle_tx_defer_mfree, "I",
+ "Free completed transmits outside of TX ring lock");
+
if (scctx->isc_ntxqsets > 100)
qfmt = "txq%03d";
else if (scctx->isc_ntxqsets > 10)
@@ -7196,7 +7281,8 @@
{
if_ctx_t ctx;
iflib_txq_t txq;
- int error;
+ struct mbuf **m_defer;
+ int error, i, reclaimable;
int bytes_sent = 0, pkt_sent = 0, mcast_sent = 0;
@@ -7222,8 +7308,34 @@
else
if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
}
- (void)iflib_completed_tx_reclaim(txq);
+ m_defer = NULL;
+ reclaimable = iflib_txq_can_reclaim(txq);
+ if (reclaimable != 0) {
+ /*
+ * Try to set m_defer to the deferred mbuf reclaim array. If
+ * we can, the frees will happen outside the tx lock. If we
+ * can't, it means another thread is still proccessing frees.
+ */
+ if (txq->ift_defer_mfree &&
+ atomic_cmpset_acq_ptr((uintptr_t *)&txq->ift_sds.ifsd_m_defer,
+ (uintptr_t )txq->ift_sds.ifsd_m_deferb, 0)) {
+ m_defer = txq->ift_sds.ifsd_m_deferb;
+ }
+ _iflib_completed_tx_reclaim(txq, m_defer, reclaimable);
+ }
mtx_unlock(&txq->ift_mtx);
+
+ /*
+ * Process mbuf frees outside the tx lock
+ */
+ if (m_defer != NULL) {
+ for (i = 0; m_defer[i] != NULL; i++) {
+ m_freem(m_defer[i]);
+ m_defer[i] = NULL;
+ }
+ atomic_set_rel_ptr((uintptr_t *)&txq->ift_sds.ifsd_m_defer,
+ (uintptr_t)m_defer);
+ }
if_inc_counter(ifp, IFCOUNTER_OBYTES, bytes_sent);
if_inc_counter(ifp, IFCOUNTER_OPACKETS, pkt_sent);
if (mcast_sent)

File Metadata

Mime Type
text/plain
Expires
Wed, Jan 21, 1:55 AM (4 h, 46 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
27233097
Default Alt Text
D54356.id168560.diff (10 KB)

Event Timeline