Index: head/sys/net/iflib.c =================================================================== --- head/sys/net/iflib.c +++ head/sys/net/iflib.c @@ -197,6 +197,7 @@ uint16_t ifc_sysctl_nrxqs; uint16_t ifc_sysctl_qs_eq_override; uint16_t ifc_sysctl_rx_budget; + uint16_t ifc_sysctl_tx_abdicate; qidx_t ifc_sysctl_ntxds[8]; qidx_t ifc_sysctl_nrxds[8]; @@ -3756,6 +3757,7 @@ iflib_txq_t txq = context; if_ctx_t ctx = txq->ift_ctx; struct ifnet *ifp = ctx->ifc_ifp; + int abdicate = ctx->ifc_sysctl_tx_abdicate; #ifdef IFLIB_DIAGNOSTICS txq->ift_cpu_exec_count[curcpu]++; @@ -3769,7 +3771,14 @@ return; } if (txq->ift_db_pending) - ifmp_ring_enqueue(txq->ift_br, (void **)&txq, 1, TX_BATCH_SIZE); + ifmp_ring_enqueue(txq->ift_br, (void **)&txq, 1, TX_BATCH_SIZE, abdicate); + else if (!abdicate) + ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE); + /* + * When abdicating, we always need to check drainage, not just when we don't enqueue + */ + if (abdicate) + ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE); ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE); if (ctx->ifc_flags & IFC_LEGACY) IFDI_INTR_ENABLE(ctx); @@ -3940,6 +3949,7 @@ iflib_txq_t txq; int err, qidx; + int abdicate = ctx->ifc_sysctl_tx_abdicate; if (__predict_false((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || !LINK_ACTIVE(ctx))) { DBG_COUNTER_INC(tx_frees); @@ -3991,10 +4001,13 @@ } #endif DBG_COUNTER_INC(tx_seen); - err = ifmp_ring_enqueue(txq->ift_br, (void **)&m, 1, TX_BATCH_SIZE); + err = ifmp_ring_enqueue(txq->ift_br, (void **)&m, 1, TX_BATCH_SIZE, abdicate); - GROUPTASK_ENQUEUE(&txq->ift_task); - if (err) { + if (abdicate) + GROUPTASK_ENQUEUE(&txq->ift_task); + if (err) { + if (!abdicate) + GROUPTASK_ENQUEUE(&txq->ift_task); /* support forthcoming later */ #ifdef DRIVER_BACKPRESSURE txq->ift_closed = TRUE; @@ -6200,6 +6213,9 @@ SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "rx_budget", CTLFLAG_RWTUN, &ctx->ifc_sysctl_rx_budget, 0, "set the rx budget"); + SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "tx_abdicate", + CTLFLAG_RWTUN, &ctx->ifc_sysctl_tx_abdicate, 0, + "cause tx to abdicate instead of running to completion"); /* XXX change for per-queue sizes */ SYSCTL_ADD_PROC(ctx_list, oid_list, OID_AUTO, "override_ntxds", Index: head/sys/net/mp_ring.h =================================================================== --- head/sys/net/mp_ring.h +++ head/sys/net/mp_ring.h @@ -63,7 +63,7 @@ int ifmp_ring_alloc(struct ifmp_ring **, int, void *, mp_ring_drain_t, mp_ring_can_drain_t, struct malloc_type *, int); void ifmp_ring_free(struct ifmp_ring *); -int ifmp_ring_enqueue(struct ifmp_ring *, void **, int, int); +int ifmp_ring_enqueue(struct ifmp_ring *, void **, int, int, int); void ifmp_ring_check_drainage(struct ifmp_ring *, int); void ifmp_ring_reset_stats(struct ifmp_ring *); int ifmp_ring_is_idle(struct ifmp_ring *); Index: head/sys/net/mp_ring.c =================================================================== --- head/sys/net/mp_ring.c +++ head/sys/net/mp_ring.c @@ -327,7 +327,7 @@ */ #ifdef NO_64BIT_ATOMICS int -ifmp_ring_enqueue(struct ifmp_ring *r, void **items, int n, int budget) +ifmp_ring_enqueue(struct ifmp_ring *r, void **items, int n, int budget, int abdicate) { union ring_state os, ns; uint16_t pidx_start, pidx_stop; @@ -380,16 +380,24 @@ */ os.state = ns.state = r->state; ns.pidx_tail = pidx_stop; - ns.flags = BUSY; + if (abdicate) { + if (os.flags == IDLE) + ns.flags = ABDICATED; + } + else { + ns.flags = BUSY; + } r->state = ns.state; counter_u64_add(r->enqueues, n); - /* - * Turn into a consumer if some other thread isn't active as a consumer - * already. - */ - if (os.flags != BUSY) - drain_ring_locked(r, ns, os.flags, budget); + if (!abdicate) { + /* + * Turn into a consumer if some other thread isn't active as a consumer + * already. + */ + if (os.flags != BUSY) + drain_ring_locked(r, ns, os.flags, budget); + } mtx_unlock(&r->lock); return (0); @@ -397,7 +405,7 @@ #else int -ifmp_ring_enqueue(struct ifmp_ring *r, void **items, int n, int budget) +ifmp_ring_enqueue(struct ifmp_ring *r, void **items, int n, int budget, int abdicate) { union ring_state os, ns; uint16_t pidx_start, pidx_stop; @@ -455,11 +463,25 @@ do { os.state = ns.state = r->state; ns.pidx_tail = pidx_stop; - if (os.flags == IDLE) - ns.flags = ABDICATED; + if (abdicate) { + if (os.flags == IDLE) + ns.flags = ABDICATED; + } + else { + ns.flags = BUSY; + } } while (atomic_cmpset_rel_64(&r->state, os.state, ns.state) == 0); critical_exit(); counter_u64_add(r->enqueues, n); + + if (!abdicate) { + /* + * Turn into a consumer if some other thread isn't active as a consumer + * already. + */ + if (os.flags != BUSY) + drain_ring_lockless(r, ns, os.flags, budget); + } return (0); }