Page Menu
Home
FreeBSD
Search
Configure Global Search
Log In
Files
F177639123
D60416.1794918821.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Flag For Later
Award Token
Authored By
Unknown
Size
12 KB
Referenced Files
None
Subscribers
None
D60416.1794918821.diff
View Options
diff --git a/share/man/man4/iflib.4 b/share/man/man4/iflib.4
--- a/share/man/man4/iflib.4
+++ b/share/man/man4/iflib.4
@@ -1,4 +1,4 @@
-.Dd September 16, 2026
+.Dd October 20, 2026
.Dt IFLIB 4
.Os
.Sh NAME
@@ -186,6 +186,24 @@
This setting applies only when
.Va simple_tx
is enabled.
+.It Va net.iflib.tx_db_batched
+When set, a receive task that passes a batch of frames to the network stack
+notifies the hardware of the frames it hands to transmit queues during that
+batch, for instance when forwarding, once per transmit queue at the end of
+the batch instead of once per frame.
+It saves register writes, at the cost of the time it takes to pass the rest
+of the batch up.
+A transmit queue is held only while it is at most an eighth full, where
+.Nm
+would otherwise notify the hardware for every frame; above that it is
+notified as before.
+It has no effect when
+.Va net.iflib.min_tx_latency
+is set.
+The default is zero.
+This is a read-only tunable that must be set before
+.Nm
+is loaded.
.It Va net.iflib.tx_watchdog_periods
Number of consecutive
.Va net.iflib.timer_default
diff --git a/sys/net/iflib.c b/sys/net/iflib.c
--- a/sys/net/iflib.c
+++ b/sys/net/iflib.c
@@ -196,7 +196,7 @@
static int iflib_simple_transmit_txq_select(if_t ifp, struct mbuf *m);
static int iflib_simple_transmit_txq_select_v2(if_t ifp, struct mbuf *m);
static void iflib_simple_if_start(if_t ifp);
-static void iflib_simple_txq_drain(iflib_txq_t txq);
+static void iflib_simple_txq_drain(iflib_txq_t txq, u_int quota);
typedef struct iflib_filter_info {
driver_filter_t *ifi_filter;
@@ -462,6 +462,10 @@
struct mtx ift_mtx;
struct mtx ift_db_mtx;
+ /* Receive batch holding the doorbell: compared, never dereferenced. */
+ struct iflib_txdb_batch *ift_db_batch;
+ SLIST_ENTRY(iflib_txq) ift_db_link;
+
/* constant values */
if_ctx_t ift_ctx;
struct ifmp_ring *ift_br;
@@ -680,6 +684,10 @@
SYSCTL_INT(_net_iflib, OID_AUTO, no_tx_batch, CTLFLAG_RW,
&iflib_no_tx_batch, 0,
"minimize transmit latency at the possible expense of throughput");
+static int iflib_tx_db_batched = 0;
+SYSCTL_INT(_net_iflib, OID_AUTO, tx_db_batched, CTLFLAG_RDTUN,
+ &iflib_tx_db_batched, 0,
+ "ring transmit doorbells once per receive batch");
static int iflib_timer_default = 1000;
SYSCTL_INT(_net_iflib, OID_AUTO, timer_default, CTLFLAG_RW,
&iflib_timer_default, 0, "number of ticks between iflib_timer calls");
@@ -855,6 +863,8 @@
static int iflib_legacy_setup(if_ctx_t ctx, driver_filter_t filter, void *filterarg, int *rid, const char *str);
static void iflib_txq_check_drain(iflib_txq_t txq, int budget);
static uint32_t iflib_txq_can_drain(struct ifmp_ring *);
+static bool iflib_txd_db_check(iflib_txq_t txq, int ring);
+static bool iflib_txd_db_flush(iflib_txq_t txq);
#ifdef ALTQ
static void iflib_altq_if_start(if_t ifp);
static int iflib_altq_if_transmit(if_t ifp, struct mbuf *m);
@@ -1999,6 +2009,8 @@
{
if_ctx_t ctx = txq->ift_ctx;
+ KASSERT(txq->ift_db_batch == NULL,
+ ("%s: txq %p still held by a receive batch", __func__, txq));
for (int i = 0; i < txq->ift_size; i++)
iflib_txsd_destroy(ctx, txq, i);
@@ -3262,6 +3274,98 @@
GROUPTASK_ENQUEUE(&rxq->ifr_task);
}
+/*
+ * Transmit doorbells batched per receive pass.
+ *
+ * On a ring at most an eighth full txq_max_db_deferred() rings for every
+ * packet. With direct dispatch the receive task transmits from within its
+ * pass (if_input(), the LRO flush, and tcp_lro_queue_mbuf() inside the
+ * loop), so for the length of the pass the queues it sends on hold their
+ * doorbells, and it rings each of them once at the end. That includes
+ * frames other threads queued that the task sends for them as the mp_ring
+ * consumer or the holder of ift_mtx. Above an eighth a queue rings as it
+ * would without the batch.
+ *
+ * The batch lives on the receive task's stack and is found through its
+ * thread (td_netbatch). A queue is held by one batch at a time: the first
+ * hold claims it and links it into the batch, and other passes that send on
+ * it ring as before. The transmit task rings a held queue when it drains
+ * on an idle ring, so a completion during a pass costs at most one extra
+ * doorbell and a stalled pass is caught by the laggard timer. Nested
+ * passes (the debugnet poll after a panic) stack.
+ *
+ * The hold is the lowest band of the doorbell policy: iflib_txd_db_check()
+ * leaves a queue to the pass where it would otherwise ring for every packet.
+ * iflib_txd_db_flush() rings whatever is pending, held or not, for the
+ * backstops. A pass decides once whether to batch and always ends what it
+ * began.
+ */
+struct iflib_txdb_batch {
+ void *itb_prev; /* batch this one is nested in */
+ SLIST_HEAD(, iflib_txq) itb_txqs; /* the queues it holds */
+};
+
+static void
+iflib_txdb_batch_begin(struct iflib_txdb_batch *b)
+{
+ b->itb_prev = curthread->td_netbatch;
+ SLIST_INIT(&b->itb_txqs);
+ curthread->td_netbatch = b;
+}
+
+/*
+ * Whether txq's doorbell may be left to the end of the receive batch the
+ * current thread is passing up, if any. The first hold claims the queue.
+ */
+static bool
+iflib_txdb_hold(iflib_txq_t txq)
+{
+ struct iflib_txdb_batch *b, *owner;
+
+ if (iflib_tx_db_batched == 0)
+ return (false);
+ b = curthread->td_netbatch;
+ if (b == NULL)
+ return (false);
+ /* Unordered: a stale reader at worst kicks the ring once more. */
+ owner = atomic_load_ptr(&txq->ift_db_batch);
+ if (owner == b)
+ return (true);
+ if (owner != NULL)
+ return (false);
+ if (!atomic_cmpset_ptr((uintptr_t *)&txq->ift_db_batch, (uintptr_t)NULL,
+ (uintptr_t)b))
+ return (false);
+ SLIST_INSERT_HEAD(&b->itb_txqs, txq, ift_db_link);
+ return (true);
+}
+
+/* The pass has ended: ring what the queues it held back have pending. */
+static void
+iflib_txdb_batch_end(struct iflib_txdb_batch *b)
+{
+ iflib_txq_t txq;
+
+ KASSERT(curthread->td_netbatch == b,
+ ("%s: not the current batch", __func__));
+ curthread->td_netbatch = b->itb_prev;
+
+ while ((txq = SLIST_FIRST(&b->itb_txqs)) != NULL) {
+ SLIST_REMOVE_HEAD(&b->itb_txqs, ift_db_link);
+ atomic_store_rel_ptr((uintptr_t *)&txq->ift_db_batch,
+ (uintptr_t)NULL);
+ if (atomic_load_16(&txq->ift_db_pending) == 0)
+ continue;
+ if (txq->ift_ctx->ifc_sysctl_simple_tx)
+ iflib_simple_txq_drain(txq,
+ iflib_simple_drain_quota_thread);
+ else
+ /* Only the consumer may ring: have the next drain. */
+ ifmp_ring_enqueue(txq->ift_br, (void **)&txq, 1,
+ TX_BATCH_SIZE, false);
+ }
+}
+
static uint8_t
iflib_rxeof(iflib_rxq_t rxq, qidx_t budget)
{
@@ -3274,7 +3378,8 @@
struct if_rxd_info ri;
int err, budget_left, rx_bytes, rx_pkts;
iflib_fl_t fl;
- bool rx_err;
+ struct iflib_txdb_batch txdb;
+ bool rx_err, txdb_batched;
#if defined(INET6) || defined(INET)
int lro_enabled;
#endif
@@ -3307,6 +3412,9 @@
#if defined(INET6) || defined(INET)
lro_enabled = (if_getcapenable(ifp) & IFCAP_LRO);
#endif
+ txdb_batched = iflib_tx_db_batched && !iflib_min_tx_latency;
+ if (txdb_batched)
+ iflib_txdb_batch_begin(&txdb);
/* pfil needs the vnet to be set */
CURVNET_SET_QUIET(if_getvnet(ifp));
@@ -3393,6 +3501,8 @@
#if defined(INET6) || defined(INET)
tcp_lro_flush_all(&rxq->ifr_lc);
#endif
+ if (txdb_batched)
+ iflib_txdb_batch_end(&txdb);
if (__predict_false(rx_err)) {
STATE_LOCK(ctx);
ctx->ifc_flags |= IFC_DO_RESET;
@@ -3448,11 +3558,34 @@
#define QIDX(ctx, m) ((((m)->m_pkthdr.flowid & ctx->ifc_softc_ctx.isc_rss_table_mask) % NTXQSETS(ctx)) + FIRST_QSET(ctx))
#define DESC_RECLAIMABLE(q) ((int)((q)->ift_processed - (q)->ift_cleaned - (q)->ift_ctx->ifc_softc_ctx.isc_tx_nsegments))
+static inline void
+iflib_txd_db_ring(iflib_txq_t txq)
+{
+ if_ctx_t ctx = txq->ift_ctx;
+ qidx_t dbval;
+
+ /*
+ * 'npending' is used if the card's doorbell is in terms of the number
+ * of descriptors pending flush (BRCM). 'pidx' is used in cases where
+ * the card's doorbell uses the producer index explicitly (INTC).
+ */
+ dbval = txq->ift_npending ? txq->ift_npending : txq->ift_pidx;
+ bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,
+ BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
+ ctx->isc_txd_flush(ctx->ifc_softc, txq->ift_id, dbval);
+
+ /*
+ * Absent bugs there are zero packets pending so reset pending counts
+ * to zero.
+ */
+ txq->ift_db_pending = txq->ift_npending = 0;
+}
+
static inline bool
iflib_txd_db_check(iflib_txq_t txq, int ring)
{
if_ctx_t ctx = txq->ift_ctx;
- qidx_t dbval, max;
+ qidx_t max;
/*
* Nothing has been queued since the last doorbell. The threshold is
@@ -3463,26 +3596,29 @@
max = TXQ_MAX_DB_DEFERRED(txq, txq->ift_in_use);
+ /*
+ * Where the policy would ring for every packet, a receive pass on
+ * this thread may hold the queue to its end instead.
+ */
+ if (max == 0 && iflib_txdb_hold(txq))
+ return (false);
+
/* Defer unless forced, at the threshold or at the edge of the ring. */
if (!ring && txq->ift_db_pending < max &&
TXQ_AVAIL(txq) > MAX_TX_DESC(ctx))
return (false);
- /*
- * 'npending' is used if the card's doorbell is in terms of the number
- * of descriptors pending flush (BRCM). 'pidx' is used in cases where
- * the card's doorbell uses the producer index explicitly (INTC).
- */
- dbval = txq->ift_npending ? txq->ift_npending : txq->ift_pidx;
- bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,
- BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
- ctx->isc_txd_flush(ctx->ifc_softc, txq->ift_id, dbval);
+ iflib_txd_db_ring(txq);
+ return (true);
+}
- /*
- * Absent bugs there are zero packets pending so reset pending counts
- * to zero.
- */
- txq->ift_db_pending = txq->ift_npending = 0;
+/* Ring txq's doorbell for what is pending, whether or not a pass holds it. */
+static bool
+iflib_txd_db_flush(iflib_txq_t txq)
+{
+ if (txq->ift_db_pending == 0)
+ return (false);
+ iflib_txd_db_ring(txq);
return (true);
}
@@ -4419,7 +4555,7 @@
goto skip_ifmp;
#endif
if (ctx->ifc_sysctl_simple_tx) {
- iflib_simple_txq_drain(txq);
+ iflib_simple_txq_drain(txq, iflib_simple_drain_quota);
goto skip_ifmp;
}
#ifdef ALTQ
@@ -4499,7 +4635,7 @@
iflib_txq_t txq;
int i;
bool do_reset, do_reset_if_up, do_watchdog;
- bool in_detach;
+ bool in_detach, kick;
STATE_LOCK(ctx);
do_reset = (ctx->ifc_flags & IFC_DO_RESET);
@@ -4530,7 +4666,16 @@
for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) {
CALLOUT_LOCK(txq);
callout_stop(&txq->ift_timer);
+ /*
+ * callout_stop() made us an ift_mtx holder, so a batch's
+ * release may have lost its trylock to us. The timer that
+ * would ring for it is stopped here and re-armed for a full
+ * period below.
+ */
+ kick = ctx->ifc_sysctl_simple_tx && txq->ift_db_pending != 0;
CALLOUT_UNLOCK(txq);
+ if (kick)
+ GROUPTASK_ENQUEUE(&txq->ift_task);
}
if (ctx->ifc_sctx->isc_flags & IFLIB_HAS_ADMINCQ)
IFDI_ADMIN_COMPLETION_HANDLE(ctx);
@@ -8013,7 +8158,7 @@
txq = &ctx->ifc_txqs[0];
error = iflib_encap(txq, &m, &bytes_sent, &pkt_sent);
if (error == 0)
- (void)iflib_txd_db_check(txq, true);
+ (void)iflib_txd_db_flush(txq);
return (error);
}
@@ -8229,9 +8374,15 @@
* the tx lock was held. Called from tx completion and from the timer.
* A thread already holding the lock is draining, and will re-arm us if it
* cannot finish, so never wait for it here.
+
+ *
+ * The end of a receive pass calls this for a queue it held, with the
+ * thread quota. Whoever holds ift_mtx drains the deferral ring and rings
+ * before dropping it, or has the transmit task do it (iflib_timer(),
+ * _task_fn_admin()), so a busy lock is not ours to wait for.
*/
static void
-iflib_simple_txq_drain(iflib_txq_t txq)
+iflib_simple_txq_drain(iflib_txq_t txq, u_int quota)
{
if_t ifp;
int bytes_sent = 0, pkt_sent = 0, mcast_sent = 0;
@@ -8249,10 +8400,9 @@
(void)iflib_completed_tx_reclaim(txq, NULL);
if (!drbr_empty(ifp, txq->ift_drbr))
- iflib_simple_drbr_drain(txq, iflib_simple_drain_quota,
- &bytes_sent, &pkt_sent, &mcast_sent);
- if (txq->ift_db_pending != 0)
- (void)iflib_txd_db_check(txq, true);
+ iflib_simple_drbr_drain(txq, quota, &bytes_sent, &pkt_sent,
+ &mcast_sent);
+ (void)iflib_txd_db_flush(txq);
mtx_unlock(&txq->ift_mtx);
if_inc_counter(ifp, IFCOUNTER_OBYTES, bytes_sent);
@@ -8479,8 +8629,7 @@
(void)iflib_completed_tx_reclaim(txq, NULL);
iflib_simple_drbr_drain(txq, UINT_MAX, &bytes_sent, &pkt_sent,
&mcast_sent);
- if (txq->ift_db_pending != 0)
- (void)iflib_txd_db_check(txq, true);
+ (void)iflib_txd_db_flush(txq);
retry = !drbr_empty(ifp, txq->ift_drbr);
mtx_unlock(&txq->ift_mtx);
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Tue, Nov 17, 12:33 PM (2 h, 19 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
41799757
Default Alt Text
D60416.1794918821.diff (12 KB)
Attached To
Mode
D60416: iflib: Ring transmit doorbells once per receive batch
Attached
Detach File
Event Timeline