Index: sys/net/iflib.c =================================================================== --- sys/net/iflib.c +++ sys/net/iflib.c @@ -129,6 +129,9 @@ */ MALLOC_DEFINE(M_IFLIB, "iflib", "ifnet library"); +#define IFLIB_RXEOF_MORE (1U << 0) +#define IFLIB_RXEOF_EMPTY (2U << 0) + struct iflib_txq; typedef struct iflib_txq *iflib_txq_t; struct iflib_rxq; @@ -434,6 +437,7 @@ uint8_t ifr_fl_offset; struct lro_ctrl ifr_lc; struct grouptask ifr_task; + struct callout ifr_watchdog; struct iflib_filter_info ifr_filter_info; iflib_dma_info_t ifr_ifdi; @@ -2071,13 +2075,14 @@ fl->ifl_fragidx = frag_idx; } -static __inline void +static __inline uint8_t __iflib_fl_refill_lt(if_ctx_t ctx, iflib_fl_t fl, int max) { /* we avoid allowing pidx to catch up with cidx as it confuses ixl */ int32_t reclaimable = fl->ifl_size - fl->ifl_credits - 1; + int32_t inuse = get_inuse(fl->ifl_size, fl->ifl_cidx, fl->ifl_pidx, fl->ifl_gen); #ifdef INVARIANTS - int32_t delta = fl->ifl_size - get_inuse(fl->ifl_size, fl->ifl_cidx, fl->ifl_pidx, fl->ifl_gen) - 1; + int32_t delta = fl->ifl_size - inuse - 1; #endif MPASS(fl->ifl_credits <= fl->ifl_size); @@ -2085,6 +2090,8 @@ if (reclaimable > 0) _iflib_fl_refill(ctx, fl, min(max, reclaimable)); + + return ((inuse == 0) ? IFLIB_RXEOF_EMPTY : 0); } uint8_t @@ -2172,7 +2179,7 @@ /* avoid pre-allocating zillions of clusters to an idle card * potentially speeding up attach */ - _iflib_fl_refill(ctx, fl, min(128, fl->ifl_size)); + (void) _iflib_fl_refill(ctx, fl, min(128, fl->ifl_size)); MPASS(min(128, fl->ifl_size) == fl->ifl_credits); if (min(128, fl->ifl_size) != fl->ifl_credits) return (ENOBUFS); @@ -2738,7 +2745,15 @@ } #endif -static bool +static void +_task_fn_rx_watchdog(void *context) +{ + iflib_rxq_t rxq = context; + + GROUPTASK_ENQUEUE(&rxq->ifr_task); +} + +static uint8_t iflib_rxeof(iflib_rxq_t rxq, qidx_t budget) { if_t ifp; @@ -2752,6 +2767,7 @@ iflib_fl_t fl; int lro_enabled; bool v4_forwarding, v6_forwarding, lro_possible; + uint8_t retval = 0; /* * XXX early demux data packets so that if_input processing only handles @@ -2770,9 +2786,9 @@ cidxp = &rxq->ifr_fl[0].ifl_cidx; if ((avail = iflib_rxd_avail(ctx, rxq, *cidxp, budget)) == 0) { for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++) - __iflib_fl_refill_lt(ctx, fl, budget + 8); + retval |= __iflib_fl_refill_lt(ctx, fl, budget + 8); DBG_COUNTER_INC(rx_unavail); - return (false); + return (retval); } /* pfil needs the vnet to be set */ @@ -2830,7 +2846,7 @@ CURVNET_RESTORE(); /* make sure that we can refill faster than drain */ for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++) - __iflib_fl_refill_lt(ctx, fl, budget + 8); + retval |= __iflib_fl_refill_lt(ctx, fl, budget + 8); lro_enabled = (if_getcapenable(ifp) & IFCAP_LRO); if (lro_enabled) @@ -2889,15 +2905,15 @@ #if defined(INET6) || defined(INET) tcp_lro_flush_all(&rxq->ifr_lc); #endif - if (avail) - return true; - return (iflib_rxd_avail(ctx, rxq, *cidxp, 1)); + if (avail != 0 || iflib_rxd_avail(ctx, rxq, *cidxp, 1) != 0) + retval |= IFLIB_RXEOF_MORE; + return (retval); err: STATE_LOCK(ctx); ctx->ifc_flags |= IFC_DO_RESET; iflib_admin_intr_deferred(ctx); STATE_UNLOCK(ctx); - return (false); + return (0); } #define TXD_NOTIFY_COUNT(txq) (((txq)->ift_size / (txq)->ift_update_freq)-1) @@ -3781,7 +3797,7 @@ { iflib_rxq_t rxq = context; if_ctx_t ctx = rxq->ifr_ctx; - bool more; + uint8_t more; uint16_t budget; #ifdef IFLIB_DIAGNOSTICS @@ -3790,19 +3806,23 @@ DBG_COUNTER_INC(task_fn_rxs); if (__predict_false(!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING))) return; - more = true; #ifdef DEV_NETMAP if (if_getcapenable(ctx->ifc_ifp) & IFCAP_NETMAP) { u_int work = 0; if (netmap_rx_irq(ctx->ifc_ifp, rxq->ifr_id, &work)) { - more = false; + more = 0; + goto skip_rxeof; } } #endif budget = ctx->ifc_sysctl_rx_budget; if (budget == 0) budget = 16; /* XXX */ - if (more == false || (more = iflib_rxeof(rxq, budget)) == false) { + more = iflib_rxeof(rxq, budget); +#ifdef DEV_NETMAP +skip_rxeof: +#endif + if ((more & IFLIB_RXEOF_MORE) == 0) { if (ctx->ifc_flags & IFC_LEGACY) IFDI_INTR_ENABLE(ctx); else @@ -3811,8 +3831,11 @@ } if (__predict_false(!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING))) return; - if (more) + + if (more & IFLIB_RXEOF_MORE) GROUPTASK_ENQUEUE(&rxq->ifr_task); + else if (more & IFLIB_RXEOF_EMPTY) + callout_reset_curcpu(&rxq->ifr_watchdog, 1, &_task_fn_rx_watchdog, rxq); } static void @@ -5029,6 +5052,7 @@ taskqgroup_detach(tqg, &txq->ift_task); } for (i = 0, rxq = ctx->ifc_rxqs; i < NRXQSETS(ctx); i++, rxq++) { + callout_drain(&rxq->ifr_watchdog); if (rxq->ifr_task.gt_uniq != NULL) taskqgroup_detach(tqg, &rxq->ifr_task); @@ -5531,6 +5555,7 @@ for (rxconf = i = 0; i < nrxqsets; i++, rxconf++, rxq++) { /* Set up some basics */ + callout_init(&rxq->ifr_watchdog, 1); if ((ifdip = malloc(sizeof(struct iflib_dma_info) * nrxqs, M_IFLIB, M_NOWAIT | M_ZERO)) == NULL) {