Page MenuHomeFreeBSD

D60416.1794918821.diff
No OneTemporary

Authored By
Unknown
Size
12 KB
Referenced Files
None
Subscribers
None

D60416.1794918821.diff

diff --git a/share/man/man4/iflib.4 b/share/man/man4/iflib.4
--- a/share/man/man4/iflib.4
+++ b/share/man/man4/iflib.4
@@ -1,4 +1,4 @@
-.Dd September 16, 2026
+.Dd October 20, 2026
.Dt IFLIB 4
.Os
.Sh NAME
@@ -186,6 +186,24 @@
This setting applies only when
.Va simple_tx
is enabled.
+.It Va net.iflib.tx_db_batched
+When set, a receive task that passes a batch of frames to the network stack
+notifies the hardware of the frames it hands to transmit queues during that
+batch, for instance when forwarding, once per transmit queue at the end of
+the batch instead of once per frame.
+It saves register writes, at the cost of the time it takes to pass the rest
+of the batch up.
+A transmit queue is held only while it is at most an eighth full, where
+.Nm
+would otherwise notify the hardware for every frame; above that it is
+notified as before.
+It has no effect when
+.Va net.iflib.min_tx_latency
+is set.
+The default is zero.
+This is a read-only tunable that must be set before
+.Nm
+is loaded.
.It Va net.iflib.tx_watchdog_periods
Number of consecutive
.Va net.iflib.timer_default
diff --git a/sys/net/iflib.c b/sys/net/iflib.c
--- a/sys/net/iflib.c
+++ b/sys/net/iflib.c
@@ -196,7 +196,7 @@
static int iflib_simple_transmit_txq_select(if_t ifp, struct mbuf *m);
static int iflib_simple_transmit_txq_select_v2(if_t ifp, struct mbuf *m);
static void iflib_simple_if_start(if_t ifp);
-static void iflib_simple_txq_drain(iflib_txq_t txq);
+static void iflib_simple_txq_drain(iflib_txq_t txq, u_int quota);
typedef struct iflib_filter_info {
driver_filter_t *ifi_filter;
@@ -462,6 +462,10 @@
struct mtx ift_mtx;
struct mtx ift_db_mtx;
+ /* Receive batch holding the doorbell: compared, never dereferenced. */
+ struct iflib_txdb_batch *ift_db_batch;
+ SLIST_ENTRY(iflib_txq) ift_db_link;
+
/* constant values */
if_ctx_t ift_ctx;
struct ifmp_ring *ift_br;
@@ -680,6 +684,10 @@
SYSCTL_INT(_net_iflib, OID_AUTO, no_tx_batch, CTLFLAG_RW,
&iflib_no_tx_batch, 0,
"minimize transmit latency at the possible expense of throughput");
+static int iflib_tx_db_batched = 0;
+SYSCTL_INT(_net_iflib, OID_AUTO, tx_db_batched, CTLFLAG_RDTUN,
+ &iflib_tx_db_batched, 0,
+ "ring transmit doorbells once per receive batch");
static int iflib_timer_default = 1000;
SYSCTL_INT(_net_iflib, OID_AUTO, timer_default, CTLFLAG_RW,
&iflib_timer_default, 0, "number of ticks between iflib_timer calls");
@@ -855,6 +863,8 @@
static int iflib_legacy_setup(if_ctx_t ctx, driver_filter_t filter, void *filterarg, int *rid, const char *str);
static void iflib_txq_check_drain(iflib_txq_t txq, int budget);
static uint32_t iflib_txq_can_drain(struct ifmp_ring *);
+static bool iflib_txd_db_check(iflib_txq_t txq, int ring);
+static bool iflib_txd_db_flush(iflib_txq_t txq);
#ifdef ALTQ
static void iflib_altq_if_start(if_t ifp);
static int iflib_altq_if_transmit(if_t ifp, struct mbuf *m);
@@ -1999,6 +2009,8 @@
{
if_ctx_t ctx = txq->ift_ctx;
+ KASSERT(txq->ift_db_batch == NULL,
+ ("%s: txq %p still held by a receive batch", __func__, txq));
for (int i = 0; i < txq->ift_size; i++)
iflib_txsd_destroy(ctx, txq, i);
@@ -3262,6 +3274,98 @@
GROUPTASK_ENQUEUE(&rxq->ifr_task);
}
+/*
+ * Transmit doorbells batched per receive pass.
+ *
+ * On a ring at most an eighth full txq_max_db_deferred() rings for every
+ * packet. With direct dispatch the receive task transmits from within its
+ * pass (if_input(), the LRO flush, and tcp_lro_queue_mbuf() inside the
+ * loop), so for the length of the pass the queues it sends on hold their
+ * doorbells, and it rings each of them once at the end. That includes
+ * frames other threads queued that the task sends for them as the mp_ring
+ * consumer or the holder of ift_mtx. Above an eighth a queue rings as it
+ * would without the batch.
+ *
+ * The batch lives on the receive task's stack and is found through its
+ * thread (td_netbatch). A queue is held by one batch at a time: the first
+ * hold claims it and links it into the batch, and other passes that send on
+ * it ring as before. The transmit task rings a held queue when it drains
+ * on an idle ring, so a completion during a pass costs at most one extra
+ * doorbell and a stalled pass is caught by the laggard timer. Nested
+ * passes (the debugnet poll after a panic) stack.
+ *
+ * The hold is the lowest band of the doorbell policy: iflib_txd_db_check()
+ * leaves a queue to the pass where it would otherwise ring for every packet.
+ * iflib_txd_db_flush() rings whatever is pending, held or not, for the
+ * backstops. A pass decides once whether to batch and always ends what it
+ * began.
+ */
+struct iflib_txdb_batch {
+ void *itb_prev; /* batch this one is nested in */
+ SLIST_HEAD(, iflib_txq) itb_txqs; /* the queues it holds */
+};
+
+static void
+iflib_txdb_batch_begin(struct iflib_txdb_batch *b)
+{
+ b->itb_prev = curthread->td_netbatch;
+ SLIST_INIT(&b->itb_txqs);
+ curthread->td_netbatch = b;
+}
+
+/*
+ * Whether txq's doorbell may be left to the end of the receive batch the
+ * current thread is passing up, if any. The first hold claims the queue.
+ */
+static bool
+iflib_txdb_hold(iflib_txq_t txq)
+{
+ struct iflib_txdb_batch *b, *owner;
+
+ if (iflib_tx_db_batched == 0)
+ return (false);
+ b = curthread->td_netbatch;
+ if (b == NULL)
+ return (false);
+ /* Unordered: a stale reader at worst kicks the ring once more. */
+ owner = atomic_load_ptr(&txq->ift_db_batch);
+ if (owner == b)
+ return (true);
+ if (owner != NULL)
+ return (false);
+ if (!atomic_cmpset_ptr((uintptr_t *)&txq->ift_db_batch, (uintptr_t)NULL,
+ (uintptr_t)b))
+ return (false);
+ SLIST_INSERT_HEAD(&b->itb_txqs, txq, ift_db_link);
+ return (true);
+}
+
+/* The pass has ended: ring what the queues it held back have pending. */
+static void
+iflib_txdb_batch_end(struct iflib_txdb_batch *b)
+{
+ iflib_txq_t txq;
+
+ KASSERT(curthread->td_netbatch == b,
+ ("%s: not the current batch", __func__));
+ curthread->td_netbatch = b->itb_prev;
+
+ while ((txq = SLIST_FIRST(&b->itb_txqs)) != NULL) {
+ SLIST_REMOVE_HEAD(&b->itb_txqs, ift_db_link);
+ atomic_store_rel_ptr((uintptr_t *)&txq->ift_db_batch,
+ (uintptr_t)NULL);
+ if (atomic_load_16(&txq->ift_db_pending) == 0)
+ continue;
+ if (txq->ift_ctx->ifc_sysctl_simple_tx)
+ iflib_simple_txq_drain(txq,
+ iflib_simple_drain_quota_thread);
+ else
+ /* Only the consumer may ring: have the next drain. */
+ ifmp_ring_enqueue(txq->ift_br, (void **)&txq, 1,
+ TX_BATCH_SIZE, false);
+ }
+}
+
static uint8_t
iflib_rxeof(iflib_rxq_t rxq, qidx_t budget)
{
@@ -3274,7 +3378,8 @@
struct if_rxd_info ri;
int err, budget_left, rx_bytes, rx_pkts;
iflib_fl_t fl;
- bool rx_err;
+ struct iflib_txdb_batch txdb;
+ bool rx_err, txdb_batched;
#if defined(INET6) || defined(INET)
int lro_enabled;
#endif
@@ -3307,6 +3412,9 @@
#if defined(INET6) || defined(INET)
lro_enabled = (if_getcapenable(ifp) & IFCAP_LRO);
#endif
+ txdb_batched = iflib_tx_db_batched && !iflib_min_tx_latency;
+ if (txdb_batched)
+ iflib_txdb_batch_begin(&txdb);
/* pfil needs the vnet to be set */
CURVNET_SET_QUIET(if_getvnet(ifp));
@@ -3393,6 +3501,8 @@
#if defined(INET6) || defined(INET)
tcp_lro_flush_all(&rxq->ifr_lc);
#endif
+ if (txdb_batched)
+ iflib_txdb_batch_end(&txdb);
if (__predict_false(rx_err)) {
STATE_LOCK(ctx);
ctx->ifc_flags |= IFC_DO_RESET;
@@ -3448,11 +3558,34 @@
#define QIDX(ctx, m) ((((m)->m_pkthdr.flowid & ctx->ifc_softc_ctx.isc_rss_table_mask) % NTXQSETS(ctx)) + FIRST_QSET(ctx))
#define DESC_RECLAIMABLE(q) ((int)((q)->ift_processed - (q)->ift_cleaned - (q)->ift_ctx->ifc_softc_ctx.isc_tx_nsegments))
+static inline void
+iflib_txd_db_ring(iflib_txq_t txq)
+{
+ if_ctx_t ctx = txq->ift_ctx;
+ qidx_t dbval;
+
+ /*
+ * 'npending' is used if the card's doorbell is in terms of the number
+ * of descriptors pending flush (BRCM). 'pidx' is used in cases where
+ * the card's doorbell uses the producer index explicitly (INTC).
+ */
+ dbval = txq->ift_npending ? txq->ift_npending : txq->ift_pidx;
+ bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,
+ BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
+ ctx->isc_txd_flush(ctx->ifc_softc, txq->ift_id, dbval);
+
+ /*
+ * Absent bugs there are zero packets pending so reset pending counts
+ * to zero.
+ */
+ txq->ift_db_pending = txq->ift_npending = 0;
+}
+
static inline bool
iflib_txd_db_check(iflib_txq_t txq, int ring)
{
if_ctx_t ctx = txq->ift_ctx;
- qidx_t dbval, max;
+ qidx_t max;
/*
* Nothing has been queued since the last doorbell. The threshold is
@@ -3463,26 +3596,29 @@
max = TXQ_MAX_DB_DEFERRED(txq, txq->ift_in_use);
+ /*
+ * Where the policy would ring for every packet, a receive pass on
+ * this thread may hold the queue to its end instead.
+ */
+ if (max == 0 && iflib_txdb_hold(txq))
+ return (false);
+
/* Defer unless forced, at the threshold or at the edge of the ring. */
if (!ring && txq->ift_db_pending < max &&
TXQ_AVAIL(txq) > MAX_TX_DESC(ctx))
return (false);
- /*
- * 'npending' is used if the card's doorbell is in terms of the number
- * of descriptors pending flush (BRCM). 'pidx' is used in cases where
- * the card's doorbell uses the producer index explicitly (INTC).
- */
- dbval = txq->ift_npending ? txq->ift_npending : txq->ift_pidx;
- bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,
- BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
- ctx->isc_txd_flush(ctx->ifc_softc, txq->ift_id, dbval);
+ iflib_txd_db_ring(txq);
+ return (true);
+}
- /*
- * Absent bugs there are zero packets pending so reset pending counts
- * to zero.
- */
- txq->ift_db_pending = txq->ift_npending = 0;
+/* Ring txq's doorbell for what is pending, whether or not a pass holds it. */
+static bool
+iflib_txd_db_flush(iflib_txq_t txq)
+{
+ if (txq->ift_db_pending == 0)
+ return (false);
+ iflib_txd_db_ring(txq);
return (true);
}
@@ -4419,7 +4555,7 @@
goto skip_ifmp;
#endif
if (ctx->ifc_sysctl_simple_tx) {
- iflib_simple_txq_drain(txq);
+ iflib_simple_txq_drain(txq, iflib_simple_drain_quota);
goto skip_ifmp;
}
#ifdef ALTQ
@@ -4499,7 +4635,7 @@
iflib_txq_t txq;
int i;
bool do_reset, do_reset_if_up, do_watchdog;
- bool in_detach;
+ bool in_detach, kick;
STATE_LOCK(ctx);
do_reset = (ctx->ifc_flags & IFC_DO_RESET);
@@ -4530,7 +4666,16 @@
for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) {
CALLOUT_LOCK(txq);
callout_stop(&txq->ift_timer);
+ /*
+ * callout_stop() made us an ift_mtx holder, so a batch's
+ * release may have lost its trylock to us. The timer that
+ * would ring for it is stopped here and re-armed for a full
+ * period below.
+ */
+ kick = ctx->ifc_sysctl_simple_tx && txq->ift_db_pending != 0;
CALLOUT_UNLOCK(txq);
+ if (kick)
+ GROUPTASK_ENQUEUE(&txq->ift_task);
}
if (ctx->ifc_sctx->isc_flags & IFLIB_HAS_ADMINCQ)
IFDI_ADMIN_COMPLETION_HANDLE(ctx);
@@ -8013,7 +8158,7 @@
txq = &ctx->ifc_txqs[0];
error = iflib_encap(txq, &m, &bytes_sent, &pkt_sent);
if (error == 0)
- (void)iflib_txd_db_check(txq, true);
+ (void)iflib_txd_db_flush(txq);
return (error);
}
@@ -8229,9 +8374,15 @@
* the tx lock was held. Called from tx completion and from the timer.
* A thread already holding the lock is draining, and will re-arm us if it
* cannot finish, so never wait for it here.
+
+ *
+ * The end of a receive pass calls this for a queue it held, with the
+ * thread quota. Whoever holds ift_mtx drains the deferral ring and rings
+ * before dropping it, or has the transmit task do it (iflib_timer(),
+ * _task_fn_admin()), so a busy lock is not ours to wait for.
*/
static void
-iflib_simple_txq_drain(iflib_txq_t txq)
+iflib_simple_txq_drain(iflib_txq_t txq, u_int quota)
{
if_t ifp;
int bytes_sent = 0, pkt_sent = 0, mcast_sent = 0;
@@ -8249,10 +8400,9 @@
(void)iflib_completed_tx_reclaim(txq, NULL);
if (!drbr_empty(ifp, txq->ift_drbr))
- iflib_simple_drbr_drain(txq, iflib_simple_drain_quota,
- &bytes_sent, &pkt_sent, &mcast_sent);
- if (txq->ift_db_pending != 0)
- (void)iflib_txd_db_check(txq, true);
+ iflib_simple_drbr_drain(txq, quota, &bytes_sent, &pkt_sent,
+ &mcast_sent);
+ (void)iflib_txd_db_flush(txq);
mtx_unlock(&txq->ift_mtx);
if_inc_counter(ifp, IFCOUNTER_OBYTES, bytes_sent);
@@ -8479,8 +8629,7 @@
(void)iflib_completed_tx_reclaim(txq, NULL);
iflib_simple_drbr_drain(txq, UINT_MAX, &bytes_sent, &pkt_sent,
&mcast_sent);
- if (txq->ift_db_pending != 0)
- (void)iflib_txd_db_check(txq, true);
+ (void)iflib_txd_db_flush(txq);
retry = !drbr_empty(ifp, txq->ift_drbr);
mtx_unlock(&txq->ift_mtx);

File Metadata

Mime Type
text/plain
Expires
Tue, Nov 17, 12:33 PM (2 h, 19 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
41799757
Default Alt Text
D60416.1794918821.diff (12 KB)

Event Timeline