Index: sys/arm/allwinner/if_awg.c =================================================================== --- sys/arm/allwinner/if_awg.c +++ sys/arm/allwinner/if_awg.c @@ -1013,6 +1013,7 @@ static void awg_intr(void *arg) { + struct epoch_tracker et; struct awg_softc *sc; uint32_t val; @@ -1022,8 +1023,11 @@ val = RD4(sc, EMAC_INT_STA); WR4(sc, EMAC_INT_STA, val); - if (val & RX_INT) + if (val & RX_INT) { + NET_EPOCH_ENTER(et); awg_rxintr(sc); + NET_EPOCH_EXIT(et); + } if (val & TX_INT) awg_txeof(sc); @@ -1040,6 +1044,7 @@ static int awg_poll(if_t ifp, enum poll_cmd cmd, int count) { + struct epoch_tracker et; struct awg_softc *sc; uint32_t val; int rx_npkts; @@ -1054,7 +1059,9 @@ return (0); } + NET_EPOCH_ENTER(et); rx_npkts = awg_rxintr(sc); + NET_EPOCH_EXIT(et); awg_txeof(sc); if (!if_sendq_empty(ifp)) awg_start_locked(sc); Index: sys/arm/allwinner/if_emac.c =================================================================== --- sys/arm/allwinner/if_emac.c +++ sys/arm/allwinner/if_emac.c @@ -704,6 +704,7 @@ static void emac_intr(void *arg) { + struct epoch_tracker et; struct emac_softc *sc; struct ifnet *ifp; uint32_t reg_val; @@ -719,8 +720,11 @@ EMAC_WRITE_REG(sc, EMAC_INT_STA, reg_val); /* Received incoming packet */ - if (reg_val & EMAC_INT_STA_RX) + if (reg_val & EMAC_INT_STA_RX) { + NET_EPOCH_ENTER(et); emac_rxeof(sc, sc->emac_rx_process_limit); + NET_EPOCH_EXIT(et); + } /* Transmit Interrupt check */ if (reg_val & EMAC_INT_STA_TX) { Index: sys/arm/ralink/if_fv.c =================================================================== --- sys/arm/ralink/if_fv.c +++ sys/arm/ralink/if_fv.c @@ -1720,6 +1720,7 @@ static void fv_intr(void *arg) { + struct epoch_tracker et; struct fv_softc *sc = arg; uint32_t status; struct ifnet *ifp = sc->fv_ifp; @@ -1736,7 +1737,9 @@ device_printf(sc->fv_dev, "Transmit Underflow\n"); } if (status & sc->sc_rxint_mask) { + NET_EPOCH_ENTER(et); fv_rx(sc); + NET_EPOCH_EXIT(et); } if (status & sc->sc_txint_mask) { fv_tx(sc); Index: sys/arm/ti/cpsw/if_cpsw.c =================================================================== --- sys/arm/ti/cpsw/if_cpsw.c +++ sys/arm/ti/cpsw/if_cpsw.c @@ -1563,6 +1563,7 @@ static void cpsw_intr_rx(void *arg) { + struct epoch_tracker et; struct cpsw_softc *sc; struct ifnet *ifp; struct mbuf *received, *next; @@ -1579,6 +1580,7 @@ cpsw_write_4(sc, CPSW_CPDMA_CPDMA_EOI_VECTOR, 1); CPSW_RX_UNLOCK(sc); + NET_EPOCH_ENTER(et); while (received != NULL) { next = received->m_nextpkt; received->m_nextpkt = NULL; @@ -1587,6 +1589,7 @@ if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); received = next; } + NET_EPOCH_EXIT(et); } static struct mbuf * @@ -2056,6 +2059,7 @@ static void cpsw_intr_rx_thresh(void *arg) { + struct epoch_tracker et; struct cpsw_softc *sc; struct ifnet *ifp; struct mbuf *received, *next; @@ -2067,6 +2071,7 @@ cpsw_write_4(sc, CPSW_CPDMA_CPDMA_EOI_VECTOR, 0); CPSW_RX_UNLOCK(sc); + NET_EPOCH_ENTER(et); while (received != NULL) { next = received->m_nextpkt; received->m_nextpkt = NULL; @@ -2075,6 +2080,7 @@ if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); received = next; } + NET_EPOCH_EXIT(et); } static void Index: sys/dev/ae/if_ae.c =================================================================== --- sys/dev/ae/if_ae.c +++ sys/dev/ae/if_ae.c @@ -1754,6 +1754,7 @@ static void ae_int_task(void *arg, int pending) { + struct epoch_tracker et; ae_softc_t *sc; struct ifnet *ifp; uint32_t val; @@ -1789,8 +1790,11 @@ } if ((val & AE_ISR_TX_EVENT) != 0) ae_tx_intr(sc); - if ((val & AE_ISR_RX_EVENT) != 0) + if ((val & AE_ISR_RX_EVENT) != 0) { + NET_EPOCH_ENTER(et); ae_rx_intr(sc); + NET_EPOCH_EXIT(et); + } /* * Re-enable interrupts. */ Index: sys/dev/age/if_age.c =================================================================== --- sys/dev/age/if_age.c +++ sys/dev/age/if_age.c @@ -2136,6 +2136,7 @@ static void age_int_task(void *arg, int pending) { + struct epoch_tracker et; struct age_softc *sc; struct ifnet *ifp; struct cmb *cmb; @@ -2167,9 +2168,12 @@ ifp = sc->age_ifp; if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0) { - if ((status & INTR_CMB_RX) != 0) + if ((status & INTR_CMB_RX) != 0) { + NET_EPOCH_ENTER(et); sc->age_morework = age_rxintr(sc, sc->age_rr_prod, sc->age_process_limit); + NET_EPOCH_EXIT(et); + } if ((status & INTR_CMB_TX) != 0) age_txintr(sc, sc->age_tpd_cons); if ((status & (INTR_DMA_RD_TO_RST | INTR_DMA_WR_TO_RST)) != 0) { Index: sys/dev/al_eth/al_eth.c =================================================================== --- sys/dev/al_eth/al_eth.c +++ sys/dev/al_eth/al_eth.c @@ -1585,6 +1585,7 @@ static void al_eth_rx_recv_work(void *arg, int pending) { + struct epoch_tracker et; struct al_eth_ring *rx_ring = arg; struct mbuf *mbuf; struct lro_entry *queued; @@ -1595,6 +1596,8 @@ uint32_t refill_actual; uint32_t do_if_input; + NET_EPOCH_ENTER(et); + if (napi != 0) { rx_ring->enqueue_is_running = 1; al_data_memory_barrier(); @@ -1691,6 +1694,8 @@ } /* unmask irq */ al_eth_irq_config(rx_ring->unmask_reg_offset, rx_ring->unmask_val); + + NET_EPOCH_EXIT(et); } static void Index: sys/dev/alc/if_alc.c =================================================================== --- sys/dev/alc/if_alc.c +++ sys/dev/alc/if_alc.c @@ -3352,6 +3352,7 @@ static void alc_int_task(void *arg, int pending) { + struct epoch_tracker et; struct alc_softc *sc; struct ifnet *ifp; uint32_t status; @@ -3375,7 +3376,9 @@ more = 0; if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0) { if ((status & INTR_RX_PKT) != 0) { + NET_EPOCH_ENTER(et); more = alc_rxintr(sc, sc->alc_process_limit); + NET_EPOCH_EXIT(et); if (more == EAGAIN) sc->alc_morework = 1; else if (more == EIO) { @@ -4700,6 +4703,8 @@ alc_debugnet_poll(struct ifnet *ifp, int count) { struct alc_softc *sc; + struct epoch_tracker et; + int err; sc = if_getsoftc(ifp); if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != @@ -4707,6 +4712,10 @@ return (EBUSY); alc_txeof(sc); - return (alc_rxintr(sc, count)); + + NET_EPOCH_ENTER(et); + err = alc_rxintr(sc, count); + NET_EPOCH_EXIT(et); + return (err); } #endif /* DEBUGNET */ Index: sys/dev/ale/if_ale.c =================================================================== --- sys/dev/ale/if_ale.c +++ sys/dev/ale/if_ale.c @@ -2246,6 +2246,7 @@ static void ale_int_task(void *arg, int pending) { + struct epoch_tracker et; struct ale_softc *sc; struct ifnet *ifp; uint32_t status; @@ -2266,7 +2267,9 @@ ifp = sc->ale_ifp; more = 0; if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0) { + NET_EPOCH_ENTER(et); more = ale_rxeof(sc, sc->ale_process_limit); + NET_EPOCH_EXIT(et); if (more == EAGAIN) sc->ale_morework = 1; else if (more == EIO) { Index: sys/dev/an/if_an.c =================================================================== --- sys/dev/an/if_an.c +++ sys/dev/an/if_an.c @@ -1187,6 +1187,7 @@ void an_intr(void *xsc) { + struct epoch_tracker et; struct an_softc *sc; struct ifnet *ifp; u_int16_t status; @@ -1222,7 +1223,9 @@ } if (status & AN_EV_RX) { + NET_EPOCH_ENTER(et); an_rxeof(sc); + NET_EPOCH_EXIT(et); CSR_WRITE_2(sc, AN_EVENT_ACK(sc->mpi350), AN_EV_RX); } Index: sys/dev/axgbe/xgbe-drv.c =================================================================== --- sys/dev/axgbe/xgbe-drv.c +++ sys/dev/axgbe/xgbe-drv.c @@ -1029,6 +1029,7 @@ static int xgbe_one_poll(struct xgbe_channel *channel, int budget) { + struct epoch_tracker et; int processed = 0; DBGPR("-->xgbe_one_poll: budget=%d\n", budget); @@ -1037,7 +1038,9 @@ xgbe_tx_poll(channel); /* Process Rx ring next */ + NET_EPOCH_ENTER(et); processed = xgbe_rx_poll(channel, budget); + NET_EPOCH_EXIT(et); DBGPR("<--xgbe_one_poll: received = %d\n", processed); @@ -1046,6 +1049,7 @@ static int xgbe_all_poll(struct xgbe_prv_data *pdata, int budget) { + struct epoch_tracker et; struct xgbe_channel *channel; int ring_budget; int processed, last_processed; @@ -1053,6 +1057,7 @@ DBGPR("-->xgbe_all_poll: budget=%d\n", budget); + NET_EPOCH_ENTER(et); processed = 0; ring_budget = budget / pdata->rx_ring_count; do { @@ -1070,6 +1075,8 @@ } } while ((processed < budget) && (processed != last_processed)); + NET_EPOCH_EXIT(et); + DBGPR("<--xgbe_all_poll: received = %d\n", processed); return processed; Index: sys/dev/bce/if_bce.c =================================================================== --- sys/dev/bce/if_bce.c +++ sys/dev/bce/if_bce.c @@ -7926,6 +7926,7 @@ static void bce_intr(void *xsc) { + struct epoch_tracker et; struct bce_softc *sc; struct ifnet *ifp; u32 status_attn_bits; @@ -8013,8 +8014,11 @@ } /* Check for any completed RX frames. */ - if (hw_rx_cons != sc->hw_rx_cons) + if (hw_rx_cons != sc->hw_rx_cons) { + NET_EPOCH_ENTER(et); bce_rx_intr(sc); + NET_EPOCH_EXIT(et); + } /* Check for any completed TX frames. */ if (hw_tx_cons != sc->hw_tx_cons) Index: sys/dev/bfe/if_bfe.c =================================================================== --- sys/dev/bfe/if_bfe.c +++ sys/dev/bfe/if_bfe.c @@ -1447,6 +1447,7 @@ static void bfe_intr(void *xsc) { + struct epoch_tracker et; struct bfe_softc *sc = xsc; struct ifnet *ifp; u_int32_t istat; @@ -1473,8 +1474,11 @@ } /* A packet was received */ - if (istat & BFE_ISTAT_RX) + if (istat & BFE_ISTAT_RX) { + NET_EPOCH_ENTER(et); bfe_rxeof(sc); + NET_EPOCH_EXIT(et); + } /* A packet was sent */ if (istat & BFE_ISTAT_TX) Index: sys/dev/bge/if_bge.c =================================================================== --- sys/dev/bge/if_bge.c +++ sys/dev/bge/if_bge.c @@ -4578,6 +4578,7 @@ static int bge_poll(if_t ifp, enum poll_cmd cmd, int count) { + struct epoch_tracker et; struct bge_softc *sc = if_getsoftc(ifp); uint16_t rx_prod, tx_cons; uint32_t statusword; @@ -4615,7 +4616,9 @@ bge_link_upd(sc); sc->rxcycles = count; + NET_EPOCH_ENTER(et); rx_npkts = bge_rxeof(sc, rx_prod, 1); + NET_EPOCH_EXIT(et); if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) { BGE_UNLOCK(sc); return (rx_npkts); @@ -4706,6 +4709,7 @@ static void bge_intr(void *xsc) { + struct epoch_tracker et; struct bge_softc *sc; if_t ifp; uint32_t statusword; @@ -4768,8 +4772,10 @@ bge_link_upd(sc); if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { + NET_EPOCH_ENTER(et); /* Check RX return ring producer/consumer. */ bge_rxeof(sc, rx_prod, 1); + NET_EPOCH_EXIT(et); } if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { @@ -6885,6 +6891,7 @@ static int bge_debugnet_poll(if_t ifp, int count) { + struct epoch_tracker et; struct bge_softc *sc; uint32_t rx_prod, tx_cons; @@ -6904,7 +6911,9 @@ sc->bge_cdata.bge_status_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); + NET_EPOCH_ENTER(et); (void)bge_rxeof(sc, rx_prod, 0); + NET_EPOCH_EXIT(et); bge_txeof(sc, tx_cons); return (0); } Index: sys/dev/bwi/if_bwi.c =================================================================== --- sys/dev/bwi/if_bwi.c +++ sys/dev/bwi/if_bwi.c @@ -1506,6 +1506,7 @@ void bwi_intr(void *xsc) { + struct epoch_tracker et; struct bwi_softc *sc = xsc; struct bwi_mac *mac; uint32_t intr_status; @@ -1625,7 +1626,10 @@ device_printf(sc->sc_dev, "intr noise\n"); if (txrx_intr_status[0] & BWI_TXRX_INTR_RX) { + NET_EPOCH_ENTER(et); rx_data = sc->sc_rxeof(sc); + NET_EPOCH_EXIT(et); + if (sc->sc_flags & BWI_F_STOP) { BWI_UNLOCK(sc); return; Index: sys/dev/bwn/if_bwn.c =================================================================== --- sys/dev/bwn/if_bwn.c +++ sys/dev/bwn/if_bwn.c @@ -5362,6 +5362,7 @@ static int bwn_pio_rx(struct bwn_pio_rxqueue *prq) { + struct epoch_tracker et; struct bwn_mac *mac = prq->prq_mac; struct bwn_softc *sc = mac->mac_sc; unsigned int i; @@ -5371,10 +5372,13 @@ if (mac->mac_status < BWN_MAC_STATUS_STARTED) return (0); + NET_EPOCH_ENTER(et); for (i = 0; i < 5000; i++) { if (bwn_pio_rxeof(prq) == 0) break; } + NET_EPOCH_EXIT(et); + if (i >= 5000) device_printf(sc->sc_dev, "too many RX frames in PIO mode\n"); return ((i > 0) ? 1 : 0); @@ -5383,8 +5387,11 @@ static void bwn_dma_rx(struct bwn_dma_ring *dr) { + struct epoch_tracker et; int slot, curslot; + NET_EPOCH_ENTER(et); + KASSERT(!dr->dr_tx, ("%s:%d: fail", __func__, __LINE__)); curslot = dr->get_curslot(dr); KASSERT(curslot >= 0 && curslot < dr->dr_numslots, @@ -5399,6 +5406,8 @@ dr->set_curslot(dr, slot); dr->dr_curslot = slot; + + NET_EPOCH_EXIT(et); } static void Index: sys/dev/bxe/bxe.c =================================================================== --- sys/dev/bxe/bxe.c +++ sys/dev/bxe/bxe.c @@ -8734,6 +8734,7 @@ bxe_handle_fp_tq(void *context, int pending) { + struct epoch_tracker et; struct bxe_fastpath *fp = (struct bxe_fastpath *)context; struct bxe_softc *sc = fp->sc; uint8_t more_tx = FALSE; @@ -8765,7 +8766,9 @@ } if (bxe_has_rx_work(fp)) { + NET_EPOCH_ENTER(et); more_rx = bxe_rxeof(sc, fp); + NET_EPOCH_EXIT(et); } if (more_rx /*|| more_tx*/) { @@ -8781,6 +8784,7 @@ static void bxe_task_fp(struct bxe_fastpath *fp) { + struct epoch_tracker et; struct bxe_softc *sc = fp->sc; uint8_t more_tx = FALSE; uint8_t more_rx = FALSE; @@ -8799,7 +8803,9 @@ } if (bxe_has_rx_work(fp)) { + NET_EPOCH_ENTER(et); more_rx = bxe_rxeof(sc, fp); + NET_EPOCH_EXIT(et); } if (more_rx /*|| more_tx*/) { @@ -19561,6 +19567,7 @@ static int bxe_debugnet_poll(struct ifnet *ifp, int count) { + struct epoch_tracker et; struct bxe_softc *sc; int i; @@ -19569,8 +19576,11 @@ !sc->link_vars.link_up) return (ENOENT); - for (i = 0; i < sc->num_queues; i++) + for (i = 0; i < sc->num_queues; i++) { + NET_EPOCH_ENTER(et); (void)bxe_rxeof(sc, &sc->fp[i]); + NET_EPOCH_EXIT(et); + } (void)bxe_txeof(sc, &sc->fp[0]); return (0); } Index: sys/dev/cadence/if_cgem.c =================================================================== --- sys/dev/cadence/if_cgem.c +++ sys/dev/cadence/if_cgem.c @@ -926,6 +926,7 @@ static void cgem_intr(void *arg) { + struct epoch_tracker et; struct cgem_softc *sc = (struct cgem_softc *)arg; if_t ifp = sc->ifp; uint32_t istatus; @@ -942,8 +943,11 @@ WR4(sc, CGEM_INTR_STAT, istatus); /* Packets received. */ - if ((istatus & CGEM_INTR_RX_COMPLETE) != 0) + if ((istatus & CGEM_INTR_RX_COMPLETE) != 0) { + NET_EPOCH_ENTER(et); cgem_recv(sc); + NET_EPOCH_EXIT(et); + } /* Free up any completed transmit buffers. */ cgem_clean_tx(sc); Index: sys/dev/cas/if_cas.c =================================================================== --- sys/dev/cas/if_cas.c +++ sys/dev/cas/if_cas.c @@ -1613,10 +1613,13 @@ cas_rint_timeout(void *arg) { struct cas_softc *sc = arg; + struct epoch_tracker et; CAS_LOCK_ASSERT(sc, MA_OWNED); + NET_EPOCH_ENTER(et); cas_rint(sc); + NET_EPOCH_EXIT(et); } static void @@ -1964,6 +1967,7 @@ static void cas_intr_task(void *arg, int pending __unused) { + struct epoch_tracker et; struct cas_softc *sc = arg; struct ifnet *ifp = sc->sc_ifp; uint32_t status, status2; @@ -2043,7 +2047,9 @@ if ((status & (CAS_INTR_RX_DONE | CAS_INTR_RX_BUF_NA | CAS_INTR_RX_COMP_FULL | CAS_INTR_RX_BUF_AEMPTY | CAS_INTR_RX_COMP_AFULL)) != 0) { + NET_EPOCH_ENTER(et); cas_rint(sc); + NET_EPOCH_EXIT(et); #ifdef CAS_DEBUG if (__predict_false((status & (CAS_INTR_RX_BUF_NA | CAS_INTR_RX_COMP_FULL | Index: sys/dev/ce/if_ce.c =================================================================== --- sys/dev/ce/if_ce.c +++ sys/dev/ce/if_ce.c @@ -376,6 +376,9 @@ static void ce_intr (void *arg) { +#ifndef NETGRAPH + struct epoch_tracker et; +#endif bdrv_t *bd = arg; ce_board_t *b = bd->board; int s; @@ -420,7 +423,9 @@ IF_DRAIN (&d->rqueue); } #else - sppp_input (d->ifp, m); + NET_EPOCH_ENTER(et); + sppp_input (d->ifp, m); + NET_EPOCH_EXIT(et); #endif } } Index: sys/dev/cp/if_cp.c =================================================================== --- sys/dev/cp/if_cp.c +++ sys/dev/cp/if_cp.c @@ -275,6 +275,7 @@ bdrv_t *bd = arg; cp_board_t *b = bd->board; #ifndef NETGRAPH + struct epoch_tracker et; int i; #endif int s = splimp (); @@ -301,6 +302,7 @@ splx (s); #ifndef NETGRAPH + NET_EPOCH_ENTER(et); /* Pass packets in a lock-free state */ for (i = 0; i < NCHAN && b->chan[i].type; i++) { drv_t *d = b->chan[i].sys; @@ -314,6 +316,7 @@ sppp_input (d->ifp, m); } } + NET_EPOCH_EXIT(et); #endif } Index: sys/dev/ctau/if_ct.c =================================================================== --- sys/dev/ctau/if_ct.c +++ sys/dev/ctau/if_ct.c @@ -248,6 +248,7 @@ bdrv_t *bd = arg; ct_board_t *b = bd->board; #ifndef NETGRAPH + struct epoch_tracker et; int i; #endif int s = splimp (); @@ -264,6 +265,7 @@ splx (s); #ifndef NETGRAPH + NET_EPOCH_ENTER(et); /* Pass packets in a lock-free state */ for (i = 0; i < NCHAN && b->chan[i].type; i++) { drv_t *d = b->chan[i].sys; @@ -277,6 +279,7 @@ sppp_input (d->ifp, m); } } + NET_EPOCH_EXIT(et); #endif } Index: sys/dev/cx/if_cx.c =================================================================== --- sys/dev/cx/if_cx.c +++ sys/dev/cx/if_cx.c @@ -336,6 +336,7 @@ bdrv_t *bd = arg; cx_board_t *b = bd->board; #ifndef NETGRAPH + struct epoch_tracker et; int i; #endif int s = splhigh (); @@ -352,6 +353,7 @@ splx (s); #ifndef NETGRAPH + NET_EPOCH_ENTER(et); /* Pass packets in a lock-free state */ for (i = 0; i < NCHAN && b->chan[i].type; i++) { drv_t *d = b->chan[i].sys; @@ -365,6 +367,7 @@ sppp_input (d->ifp, m); } } + NET_EPOCH_EXIT(et); #endif } Index: sys/dev/dc/if_dc.c =================================================================== --- sys/dev/dc/if_dc.c +++ sys/dev/dc/if_dc.c @@ -3243,6 +3243,7 @@ static int dc_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) { + struct epoch_tracker et; struct dc_softc *sc = ifp->if_softc; int rx_npkts = 0; @@ -3254,7 +3255,9 @@ } sc->rxcycles = count; + NET_EPOCH_ENTER(et); rx_npkts = dc_rxeof(sc); + NET_EPOCH_EXIT(et); dc_txeof(sc); if (!IFQ_IS_EMPTY(&ifp->if_snd) && !(ifp->if_drv_flags & IFF_DRV_OACTIVE)) @@ -3278,8 +3281,11 @@ uint32_t r = CSR_READ_4(sc, DC_FRAMESDISCARDED); if_inc_counter(ifp, IFCOUNTER_IERRORS, (r & 0xffff) + ((r >> 17) & 0x7ff)); - if (dc_rx_resync(sc)) + if (dc_rx_resync(sc)) { + NET_EPOCH_ENTER(et); dc_rxeof(sc); + NET_EPOCH_EXIT(et); + } } /* restart transmit unit if necessary */ if (status & DC_ISR_TX_IDLE && sc->dc_cdata.dc_tx_cnt) @@ -3302,6 +3308,7 @@ static void dc_intr(void *arg) { + struct epoch_tracker et; struct dc_softc *sc; struct ifnet *ifp; uint32_t r, status; @@ -3335,10 +3342,12 @@ CSR_WRITE_4(sc, DC_ISR, status); if (status & DC_ISR_RX_OK) { + NET_EPOCH_ENTER(et); if (dc_rxeof(sc) == 0) { while (dc_rx_resync(sc)) dc_rxeof(sc); } + NET_EPOCH_EXIT(et); } if (status & (DC_ISR_TX_OK | DC_ISR_TX_NOBUF)) @@ -3359,10 +3368,12 @@ || (status & DC_ISR_RX_NOBUF)) { r = CSR_READ_4(sc, DC_FRAMESDISCARDED); if_inc_counter(ifp, IFCOUNTER_IERRORS, (r & 0xffff) + ((r >> 17) & 0x7ff)); + NET_EPOCH_ENTER(et); if (dc_rxeof(sc) == 0) { while (dc_rx_resync(sc)) dc_rxeof(sc); } + NET_EPOCH_EXIT(et); } if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) Index: sys/dev/dme/if_dme.c =================================================================== --- sys/dev/dme/if_dme.c +++ sys/dev/dme/if_dme.c @@ -558,6 +558,7 @@ static void dme_intr(void *arg) { + struct epoch_tracker et; struct dme_softc *sc; uint32_t intr_status; @@ -609,9 +610,11 @@ } if (intr_status & ISR_PR) { + NET_EPOCH_ENTER(et); /* Read the packets off the device */ while (dme_rxeof(sc) == 0) continue; + NET_EPOCH_EXIT(et); } DME_UNLOCK(sc); } Index: sys/dev/dwc/if_dwc.c =================================================================== --- sys/dev/dwc/if_dwc.c +++ sys/dev/dwc/if_dwc.c @@ -808,6 +808,7 @@ static void dwc_intr(void *arg) { + struct epoch_tracker et; struct dwc_softc *sc; uint32_t reg; @@ -821,8 +822,11 @@ reg = READ4(sc, DMA_STATUS); if (reg & DMA_STATUS_NIS) { - if (reg & DMA_STATUS_RI) + if (reg & DMA_STATUS_RI) { + NET_EPOCH_ENTER(et); dwc_rxfinish_locked(sc); + NET_EPOCH_EXIT(et); + } if (reg & DMA_STATUS_TI) { dwc_txfinish_locked(sc); Index: sys/dev/ena/ena_datapath.c =================================================================== --- sys/dev/ena/ena_datapath.c +++ sys/dev/ena/ena_datapath.c @@ -63,6 +63,7 @@ void ena_cleanup(void *arg, int pending) { + struct epoch_tracker et; struct ena_que *que = arg; struct ena_adapter *adapter = que->adapter; if_t ifp = adapter->ifp; @@ -78,6 +79,8 @@ ena_trace(ENA_DBG, "MSI-X TX/RX routine\n"); + NET_EPOCH_ENTER(et); + tx_ring = que->tx_ring; rx_ring = que->rx_ring; qid = que->id; @@ -92,7 +95,7 @@ txc = ena_tx_cleanup(tx_ring); if (unlikely((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)) - return; + goto done; if ((txc != TX_BUDGET) && (rxc != RX_BUDGET)) break; @@ -104,6 +107,8 @@ TX_IRQ_INTERVAL, true); ena_com_unmask_intr(io_cq, &intr_reg); +done: + NET_EPOCH_EXIT(et); } void Index: sys/dev/et/if_et.c =================================================================== --- sys/dev/et/if_et.c +++ sys/dev/et/if_et.c @@ -1180,6 +1180,7 @@ static void et_intr(void *xsc) { + struct epoch_tracker et; struct et_softc *sc; struct ifnet *ifp; uint32_t status; @@ -1205,8 +1206,11 @@ ET_UNLOCK(sc); return; } - if (status & ET_INTR_RXDMA) + if (status & ET_INTR_RXDMA) { + NET_EPOCH_ENTER(et); et_rxeof(sc); + NET_EPOCH_EXIT(et); + } if (status & (ET_INTR_TXDMA | ET_INTR_TIMER)) et_txeof(sc); if (status & ET_INTR_TIMER) Index: sys/dev/ffec/if_ffec.c =================================================================== --- sys/dev/ffec/if_ffec.c +++ sys/dev/ffec/if_ffec.c @@ -1285,6 +1285,7 @@ static void ffec_intr(void *arg) { + struct epoch_tracker et; struct ffec_softc *sc; uint32_t ier; @@ -1301,7 +1302,9 @@ if (ier & FEC_IER_RXF) { WR4(sc, FEC_IER_REG, FEC_IER_RXF); + NET_EPOCH_ENTER(et); ffec_rxfinish_locked(sc); + NET_EPOCH_EXIT(et); } /* Index: sys/dev/fxp/if_fxp.c =================================================================== --- sys/dev/fxp/if_fxp.c +++ sys/dev/fxp/if_fxp.c @@ -1680,6 +1680,7 @@ static int fxp_poll(if_t ifp, enum poll_cmd cmd, int count) { + struct epoch_tracker et; struct fxp_softc *sc = if_getsoftc(ifp); uint8_t statack; int rx_npkts = 0; @@ -1706,7 +1707,9 @@ CSR_WRITE_1(sc, FXP_CSR_SCB_STATACK, tmp); statack |= tmp; } + NET_EPOCH_ENTER(et); rx_npkts = fxp_intr_body(sc, ifp, statack, count); + NET_EPOCH_EXIT(et); FXP_UNLOCK(sc); return (rx_npkts); } @@ -1718,6 +1721,7 @@ static void fxp_intr(void *xsc) { + struct epoch_tracker et; struct fxp_softc *sc = xsc; if_t ifp = sc->ifp; uint8_t statack; @@ -1750,8 +1754,11 @@ * First ACK all the interrupts in this pass. */ CSR_WRITE_1(sc, FXP_CSR_SCB_STATACK, statack); - if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) + if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) { + NET_EPOCH_ENTER(et); fxp_intr_body(sc, ifp, statack, -1); + NET_EPOCH_EXIT(et); + } } FXP_UNLOCK(sc); } Index: sys/dev/gem/if_gem.c =================================================================== --- sys/dev/gem/if_gem.c +++ sys/dev/gem/if_gem.c @@ -1524,11 +1524,14 @@ static void gem_rint_timeout(void *arg) { + struct epoch_tracker et; struct gem_softc *sc = arg; GEM_LOCK_ASSERT(sc, MA_OWNED); + NET_EPOCH_ENTER(et); gem_rint(sc); + NET_EPOCH_EXIT(et); } #endif @@ -1728,6 +1731,7 @@ void gem_intr(void *v) { + struct epoch_tracker et; struct gem_softc *sc = v; uint32_t status, status2; @@ -1771,8 +1775,11 @@ (GEM_INTR_RX_TAG_ERR | GEM_INTR_PERR | GEM_INTR_BERR)) != 0) gem_eint(sc, status); - if ((status & (GEM_INTR_RX_DONE | GEM_INTR_RX_NOBUF)) != 0) + if ((status & (GEM_INTR_RX_DONE | GEM_INTR_RX_NOBUF)) != 0) { + NET_EPOCH_ENTER(et); gem_rint(sc); + NET_EPOCH_EXIT(et); + } if ((status & (GEM_INTR_TX_EMPTY | GEM_INTR_TX_INTME)) != 0) gem_tint(sc); Index: sys/dev/gxemul/ether/if_gx.c =================================================================== --- sys/dev/gxemul/ether/if_gx.c +++ sys/dev/gxemul/ether/if_gx.c @@ -348,9 +348,11 @@ static void gx_rx_intr(void *arg) { + struct epoch_tracker et; struct gx_softc *sc = arg; GXEMUL_ETHER_LOCK(sc); + NET_EPOCH_ENTER(et); for (;;) { uint64_t status, length; struct mbuf *m; @@ -376,8 +378,7 @@ if (m == NULL) { device_printf(sc->sc_dev, "no memory for receive mbuf.\n"); if_inc_counter(sc->sc_ifp, IFCOUNTER_IQDROPS, 1); - GXEMUL_ETHER_UNLOCK(sc); - return; + break; } /* Align incoming frame so IP headers are aligned. */ @@ -396,5 +397,6 @@ GXEMUL_ETHER_LOCK(sc); } + NET_EPOCH_EXIT(et); GXEMUL_ETHER_UNLOCK(sc); } Index: sys/dev/hme/if_hme.c =================================================================== --- sys/dev/hme/if_hme.c +++ sys/dev/hme/if_hme.c @@ -1340,6 +1340,7 @@ hme_intr(void *v) { struct hme_softc *sc = (struct hme_softc *)v; + struct epoch_tracker et; u_int32_t status; HME_LOCK(sc); @@ -1349,8 +1350,11 @@ if ((status & HME_SEB_STAT_ALL_ERRORS) != 0) hme_eint(sc, status); - if ((status & HME_SEB_STAT_RXTOHOST) != 0) + if ((status & HME_SEB_STAT_RXTOHOST) != 0) { + NET_EPOCH_ENTER(et); hme_rint(sc); + NET_EPOCH_EXIT(et); + } if ((status & (HME_SEB_STAT_TXALL | HME_SEB_STAT_HOSTTOTX)) != 0) hme_tint(sc); Index: sys/dev/ipw/if_ipw.c =================================================================== --- sys/dev/ipw/if_ipw.c +++ sys/dev/ipw/if_ipw.c @@ -1376,6 +1376,7 @@ static void ipw_intr(void *arg) { + struct epoch_tracker et; struct ipw_softc *sc = arg; uint32_t r; @@ -1399,8 +1400,11 @@ if (r & IPW_INTR_FW_INIT_DONE) wakeup(sc); - if (r & IPW_INTR_RX_TRANSFER) + if (r & IPW_INTR_RX_TRANSFER) { + NET_EPOCH_ENTER(et); ipw_rx_intr(sc); + NET_EPOCH_EXIT(et); + } if (r & IPW_INTR_TX_TRANSFER) ipw_tx_intr(sc); Index: sys/dev/iwi/if_iwi.c =================================================================== --- sys/dev/iwi/if_iwi.c +++ sys/dev/iwi/if_iwi.c @@ -1659,6 +1659,7 @@ static void iwi_intr(void *arg) { + struct epoch_tracker et; struct iwi_softc *sc = arg; uint32_t r; IWI_LOCK_DECL; @@ -1704,8 +1705,11 @@ if (r & IWI_INTR_TX4_DONE) iwi_tx_intr(sc, &sc->txq[3]); - if (r & IWI_INTR_RX_DONE) + if (r & IWI_INTR_RX_DONE) { + NET_EPOCH_ENTER(et); iwi_rx_intr(sc); + NET_EPOCH_EXIT(et); + } if (r & IWI_INTR_PARITY_ERROR) { /* XXX rate-limit */ Index: sys/dev/iwm/if_iwm.c =================================================================== --- sys/dev/iwm/if_iwm.c +++ sys/dev/iwm/if_iwm.c @@ -5655,10 +5655,12 @@ static void iwm_notif_intr(struct iwm_softc *sc) { + struct epoch_tracker et; int count; uint32_t wreg; uint16_t hw; + NET_EPOCH_ENTER(et); bus_dmamap_sync(sc->rxq.stat_dma.tag, sc->rxq.stat_dma.map, BUS_DMASYNC_POSTREAD); @@ -5697,6 +5699,7 @@ */ hw = (hw == 0) ? count - 1 : hw - 1; IWM_WRITE(sc, wreg, rounddown2(hw, 8)); + NET_EPOCH_EXIT(et); } static void Index: sys/dev/iwn/if_iwn.c =================================================================== --- sys/dev/iwn/if_iwn.c +++ sys/dev/iwn/if_iwn.c @@ -4273,6 +4273,7 @@ static void iwn_intr(void *arg) { + struct epoch_tracker et; struct iwn_softc *sc = arg; uint32_t r1, r2, tmp; @@ -4340,6 +4341,7 @@ } if ((r1 & (IWN_INT_FH_RX | IWN_INT_SW_RX | IWN_INT_RX_PERIODIC)) || (r2 & IWN_FH_INT_RX)) { + NET_EPOCH_ENTER(et); if (sc->sc_flags & IWN_FLAG_USE_ICT) { if (r1 & (IWN_INT_FH_RX | IWN_INT_SW_RX)) IWN_WRITE(sc, IWN_FH_INT, IWN_FH_INT_RX); @@ -4352,6 +4354,7 @@ } } else iwn_notif_intr(sc); + NET_EPOCH_EXIT(et); } if ((r1 & IWN_INT_FH_TX) || (r2 & IWN_FH_INT_TX)) { Index: sys/dev/jme/if_jme.c =================================================================== --- sys/dev/jme/if_jme.c +++ sys/dev/jme/if_jme.c @@ -2360,6 +2360,7 @@ static void jme_int_task(void *arg, int pending) { + struct epoch_tracker et; struct jme_softc *sc; struct ifnet *ifp; uint32_t status; @@ -2386,7 +2387,9 @@ more = 0; if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0) { if ((status & (INTR_RXQ_COAL | INTR_RXQ_COAL_TO)) != 0) { + NET_EPOCH_ENTER(et); more = jme_rxintr(sc, sc->jme_process_limit); + NET_EPOCH_EXIT(et); if (more != 0) sc->jme_morework = 1; } @@ -3003,6 +3006,7 @@ static void jme_stop(struct jme_softc *sc) { + struct epoch_tracker et; struct ifnet *ifp; struct jme_txdesc *txd; struct jme_rxdesc *rxd; @@ -3033,7 +3037,9 @@ jme_stop_tx(sc); /* Reclaim Rx/Tx buffers that have been completed. */ + NET_EPOCH_ENTER(et); jme_rxintr(sc, JME_RX_RING_CNT); + NET_EPOCH_EXIT(et); if (sc->jme_cdata.jme_rxhead != NULL) m_freem(sc->jme_cdata.jme_rxhead); JME_RXCHAIN_RESET(sc); Index: sys/dev/le/am7990.c =================================================================== --- sys/dev/le/am7990.c +++ sys/dev/le/am7990.c @@ -389,6 +389,7 @@ void am7990_intr(void *arg) { + struct epoch_tracker et; struct lance_softc *sc = arg; struct ifnet *ifp = sc->sc_ifp; uint16_t isr; @@ -472,8 +473,11 @@ if_link_state_change(ifp, LINK_STATE_UP); sc->sc_flags |= LE_CARRIER; - if (isr & LE_C0_RINT) + if (isr & LE_C0_RINT) { + NET_EPOCH_ENTER(et); am7990_rint(sc); + NET_EPOCH_EXIT(et); + } if (isr & LE_C0_TINT) am7990_tint(sc); Index: sys/dev/lge/if_lge.c =================================================================== --- sys/dev/lge/if_lge.c +++ sys/dev/lge/if_lge.c @@ -1068,6 +1068,7 @@ lge_intr(arg) void *arg; { + struct epoch_tracker et; struct lge_softc *sc; struct ifnet *ifp; u_int32_t status; @@ -1097,8 +1098,11 @@ if ((status & (LGE_ISR_TXCMDFIFO_EMPTY|LGE_ISR_TXDMA_DONE))) lge_txeof(sc); - if (status & LGE_ISR_RXDMA_DONE) + if (status & LGE_ISR_RXDMA_DONE) { + NET_EPOCH_ENTER(et); lge_rxeof(sc, LGE_RX_DMACNT(status)); + NET_EPOCH_EXIT(et); + } if (status & LGE_ISR_RXCMDFIFO_EMPTY) lge_rxeoc(sc); Index: sys/dev/mge/if_mge.c =================================================================== --- sys/dev/mge/if_mge.c +++ sys/dev/mge/if_mge.c @@ -743,6 +743,7 @@ static int mge_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) { + struct epoch_tracker et; struct mge_softc *sc = ifp->if_softc; uint32_t int_cause, int_cause_ext; int rx_npkts = 0; @@ -769,7 +770,9 @@ } + NET_EPOCH_ENTER(et); rx_npkts = mge_intr_rx_locked(sc, count); + NET_EPOCH_EXIT(et); MGE_RECEIVE_UNLOCK(sc); MGE_TRANSMIT_LOCK(sc); @@ -1306,6 +1309,8 @@ mge_intr_rx_check(struct mge_softc *sc, uint32_t int_cause, uint32_t int_cause_ext) { + struct epoch_tracker et; + /* Check for resource error */ if (int_cause & MGE_PORT_INT_RXERRQ0) { mge_reinit_rx(sc); @@ -1319,7 +1324,9 @@ if (int_cause || int_cause_ext) { MGE_WRITE(sc, MGE_PORT_INT_CAUSE, ~int_cause); MGE_WRITE(sc, MGE_PORT_INT_CAUSE_EXT, ~int_cause_ext); + NET_EPOCH_ENTER(et); mge_intr_rx_locked(sc, -1); + NET_EPOCH_EXIT(et); } } Index: sys/dev/msk/if_msk.c =================================================================== --- sys/dev/msk/if_msk.c +++ sys/dev/msk/if_msk.c @@ -3584,6 +3584,7 @@ static int msk_handle_events(struct msk_softc *sc) { + struct epoch_tracker et; struct msk_if_softc *sc_if; int rxput[2]; struct msk_stat_desc *sd; @@ -3630,11 +3631,13 @@ case OP_RXSTAT: if (!(sc_if->msk_ifp->if_drv_flags & IFF_DRV_RUNNING)) break; + NET_EPOCH_ENTER(et); if (sc_if->msk_framesize > (MCLBYTES - MSK_RX_BUF_ALIGN)) msk_jumbo_rxeof(sc_if, status, control, len); else msk_rxeof(sc_if, status, control, len); + NET_EPOCH_EXIT(et); rxprog++; /* * Because there is no way to sync single Rx LE Index: sys/dev/mwl/if_mwl.c =================================================================== --- sys/dev/mwl/if_mwl.c +++ sys/dev/mwl/if_mwl.c @@ -2608,6 +2608,7 @@ static void mwl_rx_proc(void *arg, int npending) { + struct epoch_tracker et; struct mwl_softc *sc = arg; struct ieee80211com *ic = &sc->sc_ic; struct mwl_rxbuf *bf; @@ -2621,6 +2622,8 @@ void *newdata; int16_t nf; + NET_EPOCH_ENTER(et); + DPRINTF(sc, MWL_DEBUG_RX_PROC, "%s: pending %u rdptr 0x%x wrptr 0x%x\n", __func__, npending, RD4(sc, sc->sc_hwspecs.rxDescRead), RD4(sc, sc->sc_hwspecs.rxDescWrite)); @@ -2819,6 +2822,8 @@ rx_stop: sc->sc_rxnext = bf; + NET_EPOCH_EXIT(et); + if (mbufq_first(&sc->sc_snd) != NULL) { /* NB: kick fw; the tx thread may have been preempted */ mwl_hal_txstart(sc->sc_mh, 0); Index: sys/dev/mxge/if_mxge.c =================================================================== --- sys/dev/mxge/if_mxge.c +++ sys/dev/mxge/if_mxge.c @@ -2792,12 +2792,14 @@ static inline void mxge_clean_rx_done(struct mxge_slice_state *ss) { + struct epoch_tracker et; mxge_rx_done_t *rx_done = &ss->rx_done; int limit = 0; uint16_t length; uint16_t checksum; int lro; + NET_EPOCH_ENTER(et); lro = ss->sc->ifp->if_capenable & IFCAP_LRO; while (rx_done->entry[rx_done->idx].length != 0) { length = ntohs(rx_done->entry[rx_done->idx].length); @@ -2817,6 +2819,7 @@ #if defined(INET) || defined (INET6) tcp_lro_flush_all(&ss->lc); #endif + NET_EPOCH_EXIT(et); } Index: sys/dev/my/if_my.c =================================================================== --- sys/dev/my/if_my.c +++ sys/dev/my/if_my.c @@ -1264,6 +1264,7 @@ static void my_intr(void *arg) { + struct epoch_tracker et; struct my_softc *sc; struct ifnet *ifp; u_int32_t status; @@ -1286,8 +1287,11 @@ else break; - if (status & MY_RI) /* receive interrupt */ + if (status & MY_RI) { /* receive interrupt */ + NET_EPOCH_ENTER(et); my_rxeof(sc); + NET_EPOCH_EXIT(et); + } if ((status & MY_RBU) || (status & MY_RxErr)) { /* rx buffer unavailable or rx error */ Index: sys/dev/neta/if_mvneta.c =================================================================== --- sys/dev/neta/if_mvneta.c +++ sys/dev/neta/if_mvneta.c @@ -2946,9 +2946,11 @@ STATIC int mvneta_rx(struct mvneta_softc *sc, int q, int count) { + struct epoch_tracker et; uint32_t prxs, npkt; int more; + NET_EPOCH_ENTER(et); more = 0; mvneta_rx_lockq(sc, q); prxs = MVNETA_READ(sc, MVNETA_PRXS(q)); @@ -2963,6 +2965,7 @@ mvneta_rx_queue(sc, q, npkt); out: mvneta_rx_unlockq(sc, q); + NET_EPOCH_EXIT(et); return more; } Index: sys/dev/netfpga10g/nf10bmac/if_nf10bmac.c =================================================================== --- sys/dev/netfpga10g/nf10bmac/if_nf10bmac.c +++ sys/dev/netfpga10g/nf10bmac/if_nf10bmac.c @@ -611,6 +611,7 @@ static void nf10bmac_intr(void *arg) { + struct epoch_tracker et; struct nf10bmac_softc *sc; struct ifnet *ifp; int rx_npkts; @@ -629,6 +630,7 @@ /* NF10BMAC_RX_INTR_DISABLE(sc); */ NF10BMAC_RX_INTR_CLEAR_DIS(sc); + NET_EPOCH_ENTER(et); /* We only have an RX interrupt and no status information. */ rx_npkts = 0; while (rx_npkts < NF10BMAC_MAX_PKTS) { @@ -639,6 +641,7 @@ if (c == 0) break; } + NET_EPOCH_EXIT(et); if (ifp->if_drv_flags & IFF_DRV_RUNNING) { /* Re-enable interrupts. */ @@ -655,6 +658,7 @@ static int nf10bmac_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) { + struct epoch_tracker et; struct nf10bmac_softc *sc; int rx_npkts = 0; @@ -665,6 +669,7 @@ return (rx_npkts); } + NET_EPOCH_ENTER(et); while (rx_npkts < count) { int c; @@ -673,6 +678,8 @@ if (c == 0) break; } + NET_EPOCH_EXIT(et); + nf10bmac_start_locked(ifp); if (rx_npkts > 0 || cmd == POLL_AND_CHECK_STATUS) { Index: sys/dev/netmap/if_ptnet.c =================================================================== --- sys/dev/netmap/if_ptnet.c +++ sys/dev/netmap/if_ptnet.c @@ -1318,6 +1318,7 @@ static void ptnet_rx_intr(void *opaque) { + struct epoch_tracker et; struct ptnet_queue *pq = opaque; struct ptnet_softc *sc = pq->sc; unsigned int unused; @@ -1334,7 +1335,9 @@ /* Like vtnet, if_igb and if_em drivers when using MSI-X interrupts, * receive-side processing is executed directly in the interrupt * service routine. Alternatively, we may schedule the taskqueue. */ + NET_EPOCH_ENTER(et); ptnet_rx_eof(pq, PTNET_RX_BUDGET, true); + NET_EPOCH_EXIT(et); } static void @@ -1922,9 +1925,13 @@ ptnet_rx_task(void *context, int pending) { struct ptnet_queue *pq = context; + struct epoch_tracker et; DBG(nm_prlim(1, "%s: pq #%u\n", __func__, pq->kring_id)); + + NET_EPOCH_ENTER(et); ptnet_rx_eof(pq, PTNET_RX_BUDGET, true); + NET_EPOCH_EXIT(et); } static void @@ -1942,6 +1949,7 @@ static int ptnet_poll(if_t ifp, enum poll_cmd cmd, int budget) { + struct epoch_tracker et; struct ptnet_softc *sc = if_getsoftc(ifp); unsigned int queue_budget; unsigned int count = 0; @@ -1969,8 +1977,10 @@ rcnt += ptnet_drain_transmit_queue(pq, queue_budget, false); } else { + NET_EPOCH_ENTER(et); rcnt += ptnet_rx_eof(pq, queue_budget, false); + NET_EPOCH_EXIT(et); } } Index: sys/dev/nfe/if_nfe.c =================================================================== --- sys/dev/nfe/if_nfe.c +++ sys/dev/nfe/if_nfe.c @@ -1639,6 +1639,7 @@ static int nfe_poll(if_t ifp, enum poll_cmd cmd, int count) { + struct epoch_tracker et; struct nfe_softc *sc = if_getsoftc(ifp); uint32_t r; int rx_npkts = 0; @@ -1650,10 +1651,12 @@ return (rx_npkts); } + NET_EPOCH_ENTER(et); if (sc->nfe_framesize > MCLBYTES - ETHER_HDR_LEN) rx_npkts = nfe_jrxeof(sc, count, &rx_npkts); else rx_npkts = nfe_rxeof(sc, count, &rx_npkts); + NET_EPOCH_EXIT(et); nfe_txeof(sc); if (!if_sendq_empty(ifp)) nfe_start_locked(ifp); @@ -1885,6 +1888,7 @@ static void nfe_int_task(void *arg, int pending) { + struct epoch_tracker et; struct nfe_softc *sc = arg; if_t ifp = sc->nfe_ifp; uint32_t r; @@ -1921,11 +1925,13 @@ } domore = 0; + NET_EPOCH_ENTER(et); /* check Rx ring */ if (sc->nfe_framesize > MCLBYTES - ETHER_HDR_LEN) domore = nfe_jrxeof(sc, sc->nfe_process_limit, NULL); else domore = nfe_rxeof(sc, sc->nfe_process_limit, NULL); + NET_EPOCH_EXIT(et); /* check Tx ring */ nfe_txeof(sc); Index: sys/dev/nge/if_nge.c =================================================================== --- sys/dev/nge/if_nge.c +++ sys/dev/nge/if_nge.c @@ -512,6 +512,7 @@ static void nge_miibus_statchg(device_t dev) { + struct epoch_tracker et; struct nge_softc *sc; struct mii_data *mii; struct ifnet *ifp; @@ -550,7 +551,9 @@ device_printf(sc->nge_dev, "%s: unable to stop Tx/Rx MAC\n", __func__); nge_txeof(sc); + NET_EPOCH_ENTER(et); nge_rxeof(sc); + NET_EPOCH_EXIT(et); if (sc->nge_head != NULL) { m_freem(sc->nge_head); sc->nge_head = sc->nge_tail = NULL; @@ -1757,6 +1760,7 @@ static int nge_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) { + struct epoch_tracker et; struct nge_softc *sc; int rx_npkts = 0; @@ -1776,7 +1780,9 @@ * and then call the interrupt routine. */ sc->rxcycles = count; + NET_EPOCH_ENTER(et); rx_npkts = nge_rxeof(sc); + NET_EPOCH_EXIT(et); nge_txeof(sc); if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) nge_start_locked(ifp); @@ -1787,8 +1793,11 @@ /* Reading the ISR register clears all interrupts. */ status = CSR_READ_4(sc, NGE_ISR); - if ((status & (NGE_ISR_RX_ERR|NGE_ISR_RX_OFLOW)) != 0) + if ((status & (NGE_ISR_RX_ERR|NGE_ISR_RX_OFLOW)) != 0) { + NET_EPOCH_ENTER(et); rx_npkts += nge_rxeof(sc); + NET_EPOCH_EXIT(et); + } if ((status & NGE_ISR_RX_IDLE) != 0) NGE_SETBIT(sc, NGE_CSR, NGE_CSR_RX_ENABLE); @@ -1806,6 +1815,7 @@ static void nge_intr(void *arg) { + struct epoch_tracker et; struct nge_softc *sc; struct ifnet *ifp; uint32_t status; @@ -1844,8 +1854,11 @@ if ((status & (NGE_ISR_RX_DESC_OK | NGE_ISR_RX_ERR | NGE_ISR_RX_OFLOW | NGE_ISR_RX_FIFO_OFLOW | - NGE_ISR_RX_IDLE | NGE_ISR_RX_OK)) != 0) + NGE_ISR_RX_IDLE | NGE_ISR_RX_OK)) != 0) { + NET_EPOCH_ENTER(et); nge_rxeof(sc); + NET_EPOCH_EXIT(et); + } if ((status & NGE_ISR_RX_IDLE) != 0) NGE_SETBIT(sc, NGE_CSR, NGE_CSR_RX_ENABLE); Index: sys/dev/ral/rt2560.c =================================================================== --- sys/dev/ral/rt2560.c +++ sys/dev/ral/rt2560.c @@ -1323,6 +1323,7 @@ void rt2560_intr(void *arg) { + struct epoch_tracker et; struct rt2560_softc *sc = arg; uint32_t r; @@ -1359,7 +1360,9 @@ rt2560_decryption_intr(sc); if (r & RT2560_RX_DONE) { + NET_EPOCH_ENTER(et); rt2560_rx_intr(sc); + NET_EPOCH_EXIT(et); rt2560_encryption_intr(sc); } Index: sys/dev/ral/rt2661.c =================================================================== --- sys/dev/ral/rt2661.c +++ sys/dev/ral/rt2661.c @@ -1124,6 +1124,7 @@ void rt2661_intr(void *arg) { + struct epoch_tracker et; struct rt2661_softc *sc = arg; uint32_t r1, r2; @@ -1148,8 +1149,11 @@ if (r1 & RT2661_MGT_DONE) rt2661_tx_dma_intr(sc, &sc->mgtq); - if (r1 & RT2661_RX_DONE) + if (r1 & RT2661_RX_DONE) { + NET_EPOCH_ENTER(et); rt2661_rx_intr(sc); + NET_EPOCH_EXIT(et); + } if (r1 & RT2661_TX0_DMA_DONE) rt2661_tx_dma_intr(sc, &sc->txq[0]); Index: sys/dev/ral/rt2860.c =================================================================== --- sys/dev/ral/rt2860.c +++ sys/dev/ral/rt2860.c @@ -1392,6 +1392,7 @@ void rt2860_intr(void *arg) { + struct epoch_tracker et; struct rt2860_softc *sc = arg; uint32_t r; @@ -1419,8 +1420,11 @@ if (r & RT2860_TX_DONE_INT5) rt2860_tx_intr(sc, 5); - if (r & RT2860_RX_DONE_INT) + if (r & RT2860_RX_DONE_INT) { + NET_EPOCH_ENTER(et); rt2860_rx_intr(sc); + NET_EPOCH_EXIT(et); + } if (r & RT2860_TX_DONE_INT4) rt2860_tx_intr(sc, 4); Index: sys/dev/re/if_re.c =================================================================== --- sys/dev/re/if_re.c +++ sys/dev/re/if_re.c @@ -2518,13 +2518,16 @@ static int re_poll_locked(struct ifnet *ifp, enum poll_cmd cmd, int count) { + struct epoch_tracker et; struct rl_softc *sc = ifp->if_softc; int rx_npkts; RL_LOCK_ASSERT(sc); sc->rxcycles = count; + NET_EPOCH_ENTER(et); re_rxeof(sc, &rx_npkts); + NET_EPOCH_EXIT(et); re_txeof(sc); if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) @@ -2603,7 +2606,7 @@ } #endif - if (status & (RL_ISR_RX_OK|RL_ISR_RX_ERR|RL_ISR_FIFO_OFLOW)) { + if (status & (RL_ISR_RX_OK|RL_ISR_RX_ERR|RL_ISR_FIFO_OFLOW)) { NET_EPOCH_ENTER(et); rval = re_rxeof(sc, NULL); NET_EPOCH_EXIT(et); @@ -2650,6 +2653,7 @@ static void re_intr_msi(void *xsc) { + struct epoch_tracker et; struct rl_softc *sc; struct ifnet *ifp; uint16_t intrs, status; @@ -2683,7 +2687,9 @@ if (status & (RL_ISR_TIMEOUT_EXPIRED | RL_ISR_RX_OK | RL_ISR_RX_ERR | RL_ISR_FIFO_OFLOW | RL_ISR_RX_OVERRUN)) { + NET_EPOCH_ENTER(et); re_rxeof(sc, NULL); + NET_EPOCH_EXIT(et); if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0) { if (sc->rl_int_rx_mod != 0 && (status & (RL_ISR_RX_OK | RL_ISR_RX_ERR | @@ -3564,6 +3570,7 @@ re_watchdog(struct rl_softc *sc) { struct ifnet *ifp; + struct epoch_tracker et; RL_LOCK_ASSERT(sc); @@ -3583,7 +3590,9 @@ if_printf(ifp, "watchdog timeout\n"); if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); + NET_EPOCH_ENTER(et); re_rxeof(sc, NULL); + NET_EPOCH_EXIT(et); ifp->if_drv_flags &= ~IFF_DRV_RUNNING; re_init_locked(sc); if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) @@ -4138,6 +4147,7 @@ static int re_debugnet_poll(struct ifnet *ifp, int count) { + struct epoch_tracker et; struct rl_softc *sc; int error; @@ -4147,7 +4157,9 @@ return (EBUSY); re_txeof(sc); + NET_EPOCH_ENTER(et); error = re_rxeof(sc, NULL); + NET_EPOCH_ENTER(et); if (error != 0 && error != EAGAIN) return (error); return (0); Index: sys/dev/rl/if_rl.c =================================================================== --- sys/dev/rl/if_rl.c +++ sys/dev/rl/if_rl.c @@ -1453,13 +1453,16 @@ static int rl_poll_locked(struct ifnet *ifp, enum poll_cmd cmd, int count) { + struct epoch_tracker et; struct rl_softc *sc = ifp->if_softc; int rx_npkts; RL_LOCK_ASSERT(sc); sc->rxcycles = count; + NET_EPOCH_ENTER(et); rx_npkts = rl_rxeof(sc); + NET_EPOCH_EXIT(et); rl_txeof(sc); if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) @@ -1489,6 +1492,7 @@ static void rl_intr(void *arg) { + struct epoch_tracker et; struct rl_softc *sc = arg; struct ifnet *ifp = sc->rl_ifp; uint16_t status; @@ -1516,8 +1520,11 @@ for (count = 16; count > 0; count--) { CSR_WRITE_2(sc, RL_ISR, status); if (ifp->if_drv_flags & IFF_DRV_RUNNING) { - if (status & (RL_ISR_RX_OK | RL_ISR_RX_ERR)) + if (status & (RL_ISR_RX_OK | RL_ISR_RX_ERR)) { + NET_EPOCH_ENTER(et); rl_rxeof(sc); + NET_EPOCH_EXIT(et); + } if (status & (RL_ISR_TX_OK | RL_ISR_TX_ERR)) rl_txeof(sc); if (status & RL_ISR_SYSTEM_ERR) { @@ -1894,6 +1901,7 @@ static void rl_watchdog(struct rl_softc *sc) { + struct epoch_tracker et; RL_LOCK_ASSERT(sc); @@ -1904,7 +1912,9 @@ if_inc_counter(sc->rl_ifp, IFCOUNTER_OERRORS, 1); rl_txeof(sc); + NET_EPOCH_ENTER(et); rl_rxeof(sc); + NET_EPOCH_EXIT(et); sc->rl_ifp->if_drv_flags &= ~IFF_DRV_RUNNING; rl_init_locked(sc); } Index: sys/dev/rt/if_rt.c =================================================================== --- sys/dev/rt/if_rt.c +++ sys/dev/rt/if_rt.c @@ -1701,6 +1701,7 @@ static void rt_rx_done_task(void *context, int pending) { + struct epoch_tracker et; struct rt_softc *sc; struct ifnet *ifp; int again; @@ -1715,7 +1716,9 @@ sc->intr_pending_mask &= ~sc->int_rx_done_mask; + NET_EPOCH_ENTER(et); again = rt_rx_eof(sc, &sc->rx_ring[0], sc->rx_process_limit); + NET_EPOCH_EXIT(et); RT_SOFTC_LOCK(sc); Index: sys/dev/rtwn/pci/rtwn_pci_rx.c =================================================================== --- sys/dev/rtwn/pci/rtwn_pci_rx.c +++ sys/dev/rtwn/pci/rtwn_pci_rx.c @@ -35,6 +35,7 @@ #include #include #include +#include #include #include @@ -324,8 +325,10 @@ struct rtwn_rx_ring *ring = &pc->rx_ring; struct rtwn_rx_stat_pci *rx_desc; struct rtwn_rx_data *rx_data; + struct epoch_tracker et; int len; + NET_EPOCH_ENTER(et); bus_dmamap_sync(ring->desc_dmat, ring->desc_map, BUS_DMASYNC_POSTREAD); for (;;) { @@ -367,6 +370,7 @@ if (le32toh(rx_desc->rxdw0) & RTWN_RXDW0_OWN) ring->cur = (ring->cur + 1) % RTWN_PCI_RX_LIST_COUNT; } + NET_EPOCH_EXIT(et); } void Index: sys/dev/sbni/if_sbni.c =================================================================== --- sys/dev/sbni/if_sbni.c +++ sys/dev/sbni/if_sbni.c @@ -900,6 +900,7 @@ static void indicate_pkt(struct sbni_softc *sc) { + struct epoch_tracker et; struct ifnet *ifp = sc->ifp; struct mbuf *m; @@ -909,7 +910,9 @@ sc->rx_buf_p = NULL; SBNI_UNLOCK(sc); + NET_EPOCH_ENTER(et); (*ifp->if_input)(ifp, m); + NET_EPOCH_EXIT(et); SBNI_LOCK(sc); } Index: sys/dev/sfxge/sfxge_ev.c =================================================================== --- sys/dev/sfxge/sfxge_ev.c +++ sys/dev/sfxge/sfxge_ev.c @@ -51,6 +51,7 @@ static void sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop) { + struct epoch_tracker et; struct sfxge_softc *sc; unsigned int index; struct sfxge_rxq *rxq; @@ -82,8 +83,11 @@ } while (txq != NULL); } - if (rxq->pending != rxq->completed) + if (rxq->pending != rxq->completed) { + NET_EPOCH_ENTER(et); sfxge_rx_qcomplete(rxq, eop); + NET_EPOCH_EXIT(et); + } } static struct sfxge_rxq * Index: sys/dev/sfxge/sfxge_rx.c =================================================================== --- sys/dev/sfxge/sfxge_rx.c +++ sys/dev/sfxge/sfxge_rx.c @@ -934,6 +934,7 @@ static void sfxge_rx_qstop(struct sfxge_softc *sc, unsigned int index) { + struct epoch_tracker et; struct sfxge_rxq *rxq; struct sfxge_evq *evq; unsigned int count; @@ -992,7 +993,9 @@ } rxq->pending = rxq->added; + NET_EPOCH_ENTER(et); sfxge_rx_qcomplete(rxq, B_TRUE); + NET_EPOCH_EXIT(et); KASSERT(rxq->completed == rxq->pending, ("rxq->completed != rxq->pending")); Index: sys/dev/sge/if_sge.c =================================================================== --- sys/dev/sge/if_sge.c +++ sys/dev/sge/if_sge.c @@ -1331,6 +1331,7 @@ static void sge_intr(void *arg) { + struct epoch_tracker et; struct sge_softc *sc; struct ifnet *ifp; uint32_t status; @@ -1362,7 +1363,10 @@ if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) break; if ((status & (INTR_RX_DONE | INTR_RX_IDLE)) != 0) { + NET_EPOCH_ENTER(et); sge_rxeof(sc); + NET_EPOCH_EXIT(et); + /* Wakeup Rx MAC. */ if ((status & INTR_RX_IDLE) != 0) CSR_WRITE_4(sc, RX_CTL, Index: sys/dev/sis/if_sis.c =================================================================== --- sys/dev/sis/if_sis.c +++ sys/dev/sis/if_sis.c @@ -1647,6 +1647,7 @@ static int sis_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) { + struct epoch_tracker et; struct sis_softc *sc = ifp->if_softc; int rx_npkts = 0; @@ -1664,7 +1665,9 @@ * and then call the interrupt routine */ sc->rxcycles = count; + NET_EPOCH_ENTER(et); rx_npkts = sis_rxeof(sc); + NET_EPOCH_EXIT(et); sis_txeof(sc); if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) sis_startl(ifp); @@ -1695,6 +1698,7 @@ static void sis_intr(void *arg) { + struct epoch_tracker et; struct sis_softc *sc; struct ifnet *ifp; uint32_t status; @@ -1730,8 +1734,11 @@ sis_txeof(sc); if (status & (SIS_ISR_RX_DESC_OK | SIS_ISR_RX_OK | - SIS_ISR_RX_ERR | SIS_ISR_RX_IDLE)) + SIS_ISR_RX_ERR | SIS_ISR_RX_IDLE)) { + NET_EPOCH_ENTER(et); sis_rxeof(sc); + NET_EPOCH_EXIT(et); + } if (status & SIS_ISR_RX_OFLOW) if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); Index: sys/dev/sk/if_sk.c =================================================================== --- sys/dev/sk/if_sk.c +++ sys/dev/sk/if_sk.c @@ -3094,6 +3094,7 @@ sk_intr(xsc) void *xsc; { + struct epoch_tracker et; struct sk_softc *sc = xsc; struct sk_if_softc *sc_if0, *sc_if1; struct ifnet *ifp0 = NULL, *ifp1 = NULL; @@ -3116,18 +3117,22 @@ for (; (status &= sc->sk_intrmask) != 0;) { /* Handle receive interrupts first. */ if (status & SK_ISR_RX1_EOF) { + NET_EPOCH_ENTER(et); if (ifp0->if_mtu > SK_MAX_FRAMELEN) sk_jumbo_rxeof(sc_if0); else sk_rxeof(sc_if0); + NET_EPOCH_EXIT(et); CSR_WRITE_4(sc, SK_BMU_RX_CSR0, SK_RXBMU_CLR_IRQ_EOF|SK_RXBMU_RX_START); } if (status & SK_ISR_RX2_EOF) { + NET_EPOCH_ENTER(et); if (ifp1->if_mtu > SK_MAX_FRAMELEN) sk_jumbo_rxeof(sc_if1); else sk_rxeof(sc_if1); + NET_EPOCH_EXIT(et); CSR_WRITE_4(sc, SK_BMU_RX_CSR1, SK_RXBMU_CLR_IRQ_EOF|SK_RXBMU_RX_START); } Index: sys/dev/smc/if_smc.c =================================================================== --- sys/dev/smc/if_smc.c +++ sys/dev/smc/if_smc.c @@ -673,6 +673,7 @@ static void smc_task_rx(void *context, int pending) { + struct epoch_tracker et; u_int packet, status, len; uint8_t *data; struct ifnet *ifp; @@ -774,6 +775,7 @@ SMC_UNLOCK(sc); + NET_EPOCH_ENTER(et); while (mhead != NULL) { m = mhead; mhead = mhead->m_next; @@ -781,6 +783,7 @@ if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); (*ifp->if_input)(ifp, m); } + NET_EPOCH_EXIT(et); } #ifdef DEVICE_POLLING Index: sys/dev/ste/if_ste.c =================================================================== --- sys/dev/ste/if_ste.c +++ sys/dev/ste/if_ste.c @@ -476,12 +476,15 @@ static int ste_poll_locked(struct ifnet *ifp, enum poll_cmd cmd, int count) { + struct epoch_tracker et; struct ste_softc *sc = ifp->if_softc; int rx_npkts; STE_LOCK_ASSERT(sc); + NET_EPOCH_ENTER(et); rx_npkts = ste_rxeof(sc, count); + NET_EPOCH_EXIT(et); ste_txeof(sc); ste_txeoc(sc); if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) @@ -538,7 +541,11 @@ } if ((status & (STE_ISR_SOFTINTR | STE_ISR_RX_DMADONE)) != 0) { + struct epoch_tracker et; + + NET_EPOCH_ENTER(et); ste_rxeof(sc, -1); + NET_EPOCH_EXIT(et); /* * The controller has no ability to Rx interrupt * moderation feature. Receiving 64 bytes frames @@ -1956,6 +1963,7 @@ static void ste_watchdog(struct ste_softc *sc) { + struct epoch_tracker et; struct ifnet *ifp; ifp = sc->ste_ifp; @@ -1969,7 +1977,9 @@ ste_txeof(sc); ste_txeoc(sc); + NET_EPOCH_ENTER(et); ste_rxeof(sc, -1); + NET_EPOCH_EXIT(et); ifp->if_drv_flags &= ~IFF_DRV_RUNNING; ste_init_locked(sc); Index: sys/dev/stge/if_stge.c =================================================================== --- sys/dev/stge/if_stge.c +++ sys/dev/stge/if_stge.c @@ -1452,6 +1452,7 @@ static void stge_intr(void *arg) { + struct epoch_tracker et; struct stge_softc *sc; struct ifnet *ifp; int reinit; @@ -1486,7 +1487,10 @@ /* Receive interrupts. */ if ((status & IS_RxDMAComplete) != 0) { + NET_EPOCH_ENTER(et); stge_rxeof(sc); + NET_EPOCH_EXIT(et); + if ((status & IS_RFDListEnd) != 0) CSR_WRITE_4(sc, STGE_DMACtrl, DMAC_RxDMAPollNow); @@ -1780,6 +1784,7 @@ static int stge_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) { + struct epoch_tracker et; struct stge_softc *sc; uint16_t status; int rx_npkts; @@ -1793,7 +1798,9 @@ } sc->sc_cdata.stge_rxcycles = count; + NET_EPOCH_ENTER(et); rx_npkts = stge_rxeof(sc); + NET_EPOCH_EXIT(et); stge_txeof(sc); if (cmd == POLL_AND_CHECK_STATUS) { Index: sys/dev/ti/if_ti.c =================================================================== --- sys/dev/ti/if_ti.c +++ sys/dev/ti/if_ti.c @@ -2959,6 +2959,7 @@ static void ti_intr(void *xsc) { + struct epoch_tracker et; struct ti_softc *sc; struct ifnet *ifp; @@ -2978,8 +2979,10 @@ if (ifp->if_drv_flags & IFF_DRV_RUNNING) { bus_dmamap_sync(sc->ti_cdata.ti_status_tag, sc->ti_cdata.ti_status_map, BUS_DMASYNC_POSTREAD); + NET_EPOCH_ENTER(et); /* Check RX return ring producer/consumer */ ti_rxeof(sc); + NET_EPOCH_EXIT(et); /* Check TX ring producer/consumer */ ti_txeof(sc); Index: sys/dev/tsec/if_tsec.c =================================================================== --- sys/dev/tsec/if_tsec.c +++ sys/dev/tsec/if_tsec.c @@ -896,6 +896,7 @@ static int tsec_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) { + struct epoch_tracker et; uint32_t ie; struct tsec_softc *sc = ifp->if_softc; int rx_npkts; @@ -920,7 +921,9 @@ TSEC_GLOBAL_TO_RECEIVE_LOCK(sc); + NET_EPOCH_ENTER(et); rx_npkts = tsec_receive_intr_locked(sc, count); + NET_EPOCH_EXIT(et); TSEC_RECEIVE_UNLOCK(sc); @@ -1415,6 +1418,7 @@ tsec_receive_intr(void *arg) { struct tsec_softc *sc = arg; + struct epoch_tracker et; TSEC_RECEIVE_LOCK(sc); @@ -1427,7 +1431,9 @@ /* Confirm the interrupt was received by driver */ TSEC_WRITE(sc, TSEC_REG_IEVENT, TSEC_IEVENT_RXB | TSEC_IEVENT_RXF); + NET_EPOCH_ENTER(et); tsec_receive_intr_locked(sc, -1); + NET_EPOCH_EXIT(et); TSEC_RECEIVE_UNLOCK(sc); } Index: sys/dev/usb/wlan/if_rum.c =================================================================== --- sys/dev/usb/wlan/if_rum.c +++ sys/dev/usb/wlan/if_rum.c @@ -1168,6 +1168,7 @@ struct ieee80211com *ic = &sc->sc_ic; struct ieee80211_frame_min *wh; struct ieee80211_node *ni; + struct epoch_tracker et; struct mbuf *m = NULL; struct usb_page_cache *pc; uint32_t flags; @@ -1286,6 +1287,7 @@ else ni = NULL; + NET_EPOCH_ENTER(et); if (ni != NULL) { (void) ieee80211_input(ni, m, rssi, RT2573_NOISE_FLOOR); @@ -1293,6 +1295,7 @@ } else (void) ieee80211_input_all(ic, m, rssi, RT2573_NOISE_FLOOR); + NET_EPOCH_EXIT(et); } RUM_LOCK(sc); rum_start(sc); Index: sys/dev/usb/wlan/if_run.c =================================================================== --- sys/dev/usb/wlan/if_run.c +++ sys/dev/usb/wlan/if_run.c @@ -2811,6 +2811,7 @@ struct ieee80211com *ic = &sc->sc_ic; struct ieee80211_frame *wh; struct ieee80211_node *ni; + struct epoch_tracker et; struct rt2870_rxd *rxd; struct rt2860_rxwi *rxwi; uint32_t flags; @@ -2929,12 +2930,14 @@ } } + NET_EPOCH_ENTER(et); if (ni != NULL) { (void)ieee80211_input(ni, m, rssi, nf); ieee80211_free_node(ni); } else { (void)ieee80211_input_all(ic, m, rssi, nf); } + NET_EPOCH_EXIT(et); return; Index: sys/dev/usb/wlan/if_uath.c =================================================================== --- sys/dev/usb/wlan/if_uath.c +++ sys/dev/usb/wlan/if_uath.c @@ -2705,6 +2705,7 @@ struct ieee80211com *ic = &sc->sc_ic; struct ieee80211_frame *wh; struct ieee80211_node *ni; + struct epoch_tracker et; struct mbuf *m = NULL; struct uath_data *data; struct uath_rx_desc *desc = NULL; @@ -2751,6 +2752,7 @@ ni = ieee80211_find_rxnode(ic, (struct ieee80211_frame_min *)wh); nf = -95; /* XXX */ + NET_EPOCH_ENTER(et); if (ni != NULL) { (void) ieee80211_input(ni, m, (int)be32toh(desc->rssi), nf); @@ -2759,6 +2761,7 @@ } else (void) ieee80211_input_all(ic, m, (int)be32toh(desc->rssi), nf); + NET_EPOCH_EXIT(et); m = NULL; desc = NULL; } Index: sys/dev/usb/wlan/if_upgt.c =================================================================== --- sys/dev/usb/wlan/if_upgt.c +++ sys/dev/usb/wlan/if_upgt.c @@ -2211,6 +2211,7 @@ struct ieee80211com *ic = &sc->sc_ic; struct ieee80211_frame *wh; struct ieee80211_node *ni; + struct epoch_tracker et; struct mbuf *m = NULL; struct upgt_data *data; int8_t nf; @@ -2248,12 +2249,14 @@ ni = ieee80211_find_rxnode(ic, (struct ieee80211_frame_min *)wh); nf = -95; /* XXX */ + NET_EPOCH_ENTER(et); if (ni != NULL) { (void) ieee80211_input(ni, m, rssi, nf); /* node is no longer needed */ ieee80211_free_node(ni); } else (void) ieee80211_input_all(ic, m, rssi, nf); + NET_EPOCH_EXIT(et); m = NULL; } UPGT_LOCK(sc); Index: sys/dev/usb/wlan/if_ural.c =================================================================== --- sys/dev/usb/wlan/if_ural.c +++ sys/dev/usb/wlan/if_ural.c @@ -851,6 +851,7 @@ struct ural_softc *sc = usbd_xfer_softc(xfer); struct ieee80211com *ic = &sc->sc_ic; struct ieee80211_node *ni; + struct epoch_tracker et; struct mbuf *m = NULL; struct usb_page_cache *pc; uint32_t flags; @@ -931,11 +932,13 @@ if (m) { ni = ieee80211_find_rxnode(ic, mtod(m, struct ieee80211_frame_min *)); + NET_EPOCH_ENTER(et); if (ni != NULL) { (void) ieee80211_input(ni, m, rssi, nf); ieee80211_free_node(ni); } else (void) ieee80211_input_all(ic, m, rssi, nf); + NET_EPOCH_EXIT(et); } RAL_LOCK(sc); ural_start(sc); Index: sys/dev/usb/wlan/if_urtw.c =================================================================== --- sys/dev/usb/wlan/if_urtw.c +++ sys/dev/usb/wlan/if_urtw.c @@ -4042,6 +4042,7 @@ struct urtw_softc *sc = usbd_xfer_softc(xfer); struct ieee80211com *ic = &sc->sc_ic; struct ieee80211_node *ni; + struct epoch_tracker et; struct mbuf *m = NULL; struct urtw_data *data; int8_t nf = -95; @@ -4085,12 +4086,14 @@ } else ni = NULL; + NET_EPOCH_ENTER(et); if (ni != NULL) { (void) ieee80211_input(ni, m, rssi, nf); /* node is no longer needed */ ieee80211_free_node(ni); } else (void) ieee80211_input_all(ic, m, rssi, nf); + NET_EPOCH_EXIT(et); m = NULL; } URTW_LOCK(sc); Index: sys/dev/usb/wlan/if_zyd.c =================================================================== --- sys/dev/usb/wlan/if_zyd.c +++ sys/dev/usb/wlan/if_zyd.c @@ -2223,6 +2223,7 @@ struct zyd_softc *sc = usbd_xfer_softc(xfer); struct ieee80211com *ic = &sc->sc_ic; struct ieee80211_node *ni; + struct epoch_tracker et; struct zyd_rx_desc desc; struct mbuf *m; struct usb_page_cache *pc; @@ -2278,6 +2279,7 @@ * "ieee80211_input" here, and not some lines up! */ ZYD_UNLOCK(sc); + NET_EPOCH_ENTER(et); for (i = 0; i < sc->sc_rx_count; i++) { rssi = sc->sc_rx_data[i].rssi; m = sc->sc_rx_data[i].m; @@ -2293,6 +2295,7 @@ } else (void)ieee80211_input_all(ic, m, rssi, nf); } + NET_EPOCH_EXIT(et); ZYD_LOCK(sc); zyd_start(sc); break; Index: sys/dev/vge/if_vge.c =================================================================== --- sys/dev/vge/if_vge.c +++ sys/dev/vge/if_vge.c @@ -1707,6 +1707,7 @@ static int vge_poll (struct ifnet *ifp, enum poll_cmd cmd, int count) { + struct epoch_tracker et; struct vge_softc *sc = ifp->if_softc; int rx_npkts = 0; @@ -1714,7 +1715,9 @@ if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) goto done; + NET_EPOCH_ENTER(et); rx_npkts = vge_rxeof(sc, count); + NET_EPOCH_EXIT(et); vge_txeof(sc); if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) @@ -1739,7 +1742,10 @@ } if (status & (VGE_ISR_RXOFLOW|VGE_ISR_RXNODESC)) { + struct epoch_tracker et; + NET_EPOCH_ENTER(et); vge_rxeof(sc, count); + NET_EPOCH_EXIT(et); CSR_WRITE_1(sc, VGE_RXQCSRS, VGE_RXQCSR_RUN); CSR_WRITE_1(sc, VGE_RXQCSRS, VGE_RXQCSR_WAK); } @@ -1753,6 +1759,7 @@ static void vge_intr(void *arg) { + struct epoch_tracker et; struct vge_softc *sc; struct ifnet *ifp; uint32_t status; @@ -1786,10 +1793,15 @@ if (status == 0xFFFFFFFF || (status & VGE_INTRS) == 0) goto done; if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0) { - if (status & (VGE_ISR_RXOK|VGE_ISR_RXOK_HIPRIO)) + if (status & (VGE_ISR_RXOK|VGE_ISR_RXOK_HIPRIO)) { + NET_EPOCH_ENTER(et); vge_rxeof(sc, VGE_RX_DESC_CNT); + NET_EPOCH_EXIT(et); + } if (status & (VGE_ISR_RXOFLOW|VGE_ISR_RXNODESC)) { + NET_EPOCH_ENTER(et); vge_rxeof(sc, VGE_RX_DESC_CNT); + NET_EPOCH_EXIT(et); CSR_WRITE_1(sc, VGE_RXQCSRS, VGE_RXQCSR_RUN); CSR_WRITE_1(sc, VGE_RXQCSRS, VGE_RXQCSR_WAK); } @@ -2378,6 +2390,7 @@ static void vge_watchdog(void *arg) { + struct epoch_tracker et; struct vge_softc *sc; struct ifnet *ifp; @@ -2393,7 +2406,9 @@ if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); vge_txeof(sc); + NET_EPOCH_ENTER(et); vge_rxeof(sc, VGE_RX_DESC_CNT); + NET_EPOCH_EXIT(et); ifp->if_drv_flags &= ~IFF_DRV_RUNNING; vge_init_locked(sc); Index: sys/dev/vnic/nicvf_queues.c =================================================================== --- sys/dev/vnic/nicvf_queues.c +++ sys/dev/vnic/nicvf_queues.c @@ -731,6 +731,7 @@ static int nicvf_cq_intr_handler(struct nicvf *nic, uint8_t cq_idx) { + struct epoch_tracker et; struct mbuf *mbuf; struct ifnet *ifp; int processed_cqe, work_done = 0, tx_done = 0; @@ -821,6 +822,7 @@ NICVF_CMP_UNLOCK(cq); + NET_EPOCH_ENTER(et); ifp = nic->ifp; /* Push received MBUFs to the stack */ while (!buf_ring_empty(cq->rx_br)) { @@ -828,6 +830,7 @@ if (__predict_true(mbuf != NULL)) (*ifp->if_input)(ifp, mbuf); } + NET_EPOCH_EXIT(et); return (cmp_err); } Index: sys/dev/vr/if_vr.c =================================================================== --- sys/dev/vr/if_vr.c +++ sys/dev/vr/if_vr.c @@ -1602,6 +1602,7 @@ static int vr_poll_locked(struct ifnet *ifp, enum poll_cmd cmd, int count) { + struct epoch_tracker et; struct vr_softc *sc; int rx_npkts; @@ -1610,7 +1611,9 @@ VR_LOCK_ASSERT(sc); sc->rxcycles = count; + NET_EPOCH_ENTER(et); rx_npkts = vr_rxeof(sc); + NET_EPOCH_EXIT(et); vr_txeof(sc); if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) vr_start_locked(ifp); @@ -1695,6 +1698,7 @@ static void vr_int_task(void *arg, int npending) { + struct epoch_tracker et; struct vr_softc *sc; struct ifnet *ifp; uint16_t status; @@ -1730,7 +1734,9 @@ return; } } + NET_EPOCH_ENTER(et); vr_rxeof(sc); + NET_EPOCH_EXIT(et); if ((status & (VR_ISR_RX_NOBUF | VR_ISR_RX_OFLOW)) != 0) { #ifdef VR_SHOW_ERRORS device_printf(sc->vr_dev, "%s: receive error = 0x%b\n", Index: sys/dev/vte/if_vte.c =================================================================== --- sys/dev/vte/if_vte.c +++ sys/dev/vte/if_vte.c @@ -1337,6 +1337,7 @@ static void vte_intr(void *arg) { + struct epoch_tracker et; struct vte_softc *sc; struct ifnet *ifp; uint16_t status; @@ -1360,8 +1361,11 @@ if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) break; if ((status & (MISR_RX_DONE | MISR_RX_DESC_UNAVAIL | - MISR_RX_FIFO_FULL)) != 0) + MISR_RX_FIFO_FULL)) != 0) { + NET_EPOCH_ENTER(et); vte_rxeof(sc); + NET_EPOCH_EXIT(et); + } if ((status & MISR_TX_DONE) != 0) vte_txeof(sc); if ((status & MISR_EVENT_CNT_OFLOW) != 0) Index: sys/dev/wi/if_wi.c =================================================================== --- sys/dev/wi/if_wi.c +++ sys/dev/wi/if_wi.c @@ -557,6 +557,7 @@ void wi_intr(void *arg) { + struct epoch_tracker et; struct wi_softc *sc = arg; u_int16_t status; @@ -574,8 +575,11 @@ CSR_WRITE_2(sc, WI_INT_EN, 0); status = CSR_READ_2(sc, WI_EVENT_STAT); - if (status & WI_EV_RX) + if (status & WI_EV_RX) { + NET_EPOCH_ENTER(et); wi_rx_intr(sc); + NET_EPOCH_EXIT(et); + } if (status & WI_EV_ALLOC) wi_tx_intr(sc); if (status & WI_EV_TX_EXC) Index: sys/dev/wpi/if_wpi.c =================================================================== --- sys/dev/wpi/if_wpi.c +++ sys/dev/wpi/if_wpi.c @@ -2181,6 +2181,7 @@ { struct ieee80211com *ic = &sc->sc_ic; struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps); + struct epoch_tracker et; uint32_t hw; bus_dmamap_sync(sc->shared_dma.tag, sc->shared_dma.map, @@ -2189,6 +2190,7 @@ hw = le32toh(sc->shared->next) & 0xfff; hw = (hw == 0) ? WPI_RX_RING_COUNT - 1 : hw - 1; + NET_EPOCH_ENTER(et); while (sc->rxq.cur != hw) { sc->rxq.cur = (sc->rxq.cur + 1) % WPI_RX_RING_COUNT; @@ -2216,7 +2218,7 @@ if (__predict_false(sc->sc_running == 0)) { /* wpi_stop() was called. */ - return; + goto done; } break; @@ -2304,7 +2306,7 @@ device_printf(sc->sc_dev, "microcontroller initialization failed\n"); wpi_stop_locked(sc); - return; + goto done; } /* Save the address of the error log in SRAM. */ sc->errptr = le32toh(uc->errptr); @@ -2326,7 +2328,7 @@ WPI_NT_UNLOCK(sc); ieee80211_runtask(ic, &sc->sc_radiooff_task); - return; + goto done; } break; } @@ -2373,6 +2375,8 @@ sc->sc_update_rx_ring(sc); } } +done: + NET_EPOCH_EXIT(et); } /* Index: sys/dev/xen/netback/netback.c =================================================================== --- sys/dev/xen/netback/netback.c +++ sys/dev/xen/netback/netback.c @@ -1429,6 +1429,7 @@ static void xnb_intr(void *arg) { + struct epoch_tracker et; struct xnb_softc *xnb; struct ifnet *ifp; netif_tx_back_ring_t *txb; @@ -1439,6 +1440,7 @@ txb = &xnb->ring_configs[XNB_RING_TYPE_TX].back_ring.tx_ring; mtx_lock(&xnb->tx_lock); + NET_EPOCH_ENTER(et); do { int notify; req_prod_local = txb->sring->req_prod; @@ -1464,6 +1466,7 @@ txb->sring->req_event = txb->req_cons + 1; xen_mb(); } while (txb->sring->req_prod != req_prod_local) ; + NET_EPOCH_EXIT(et); mtx_unlock(&xnb->tx_lock); xnb_start(ifp); Index: sys/dev/xilinx/if_xae.c =================================================================== --- sys/dev/xilinx/if_xae.c +++ sys/dev/xilinx/if_xae.c @@ -208,6 +208,7 @@ static int xae_xdma_rx_intr(void *arg, xdma_transfer_status_t *status) { + struct epoch_tracker et; xdma_transfer_status_t st; struct xae_softc *sc; struct ifnet *ifp; @@ -221,6 +222,7 @@ XAE_LOCK(sc); + NET_EPOCH_ENTER(et); ifp = sc->ifp; cnt_processed = 0; @@ -245,6 +247,7 @@ } xae_rx_enqueue(sc, cnt_processed); + NET_EPOCH_EXIT(et); XAE_UNLOCK(sc); Index: sys/dev/xl/if_xl.c =================================================================== --- sys/dev/xl/if_xl.c +++ sys/dev/xl/if_xl.c @@ -1975,11 +1975,15 @@ static void xl_rxeof_task(void *arg, int pending) { + struct epoch_tracker et; struct xl_softc *sc = (struct xl_softc *)arg; XL_LOCK(sc); - if (sc->xl_ifp->if_drv_flags & IFF_DRV_RUNNING) + if (sc->xl_ifp->if_drv_flags & IFF_DRV_RUNNING) { + NET_EPOCH_ENTER(et); xl_rxeof(sc); + NET_EPOCH_EXIT(et); + } XL_UNLOCK(sc); } @@ -2150,6 +2154,7 @@ static void xl_intr(void *arg) { + struct epoch_tracker et; struct xl_softc *sc = arg; struct ifnet *ifp = sc->xl_ifp; u_int16_t status; @@ -2173,10 +2178,12 @@ break; if (status & XL_STAT_UP_COMPLETE) { + NET_EPOCH_ENTER(et); if (xl_rxeof(sc) == 0) { while (xl_rx_resync(sc)) xl_rxeof(sc); } + NET_EPOCH_EXIT(et); } if (status & XL_STAT_DOWN_COMPLETE) { @@ -2229,13 +2236,16 @@ static int xl_poll_locked(struct ifnet *ifp, enum poll_cmd cmd, int count) { + struct epoch_tracker et; struct xl_softc *sc = ifp->if_softc; int rx_npkts; XL_LOCK_ASSERT(sc); sc->rxcycles = count; + NET_EPOCH_ENTER(et); rx_npkts = xl_rxeof(sc); + NET_EPOCH_EXIT(et); if (sc->xl_type == XL_TYPE_905B) xl_txeof_90xB(sc); else @@ -3098,6 +3108,7 @@ static int xl_watchdog(struct xl_softc *sc) { + struct epoch_tracker et; struct ifnet *ifp = sc->xl_ifp; u_int16_t status = 0; int misintr; @@ -3107,7 +3118,9 @@ if (sc->xl_wdog_timer == 0 || --sc->xl_wdog_timer != 0) return (0); + NET_EPOCH_ENTER(et); xl_rxeof(sc); + NET_EPOCH_EXIT(et); xl_txeoc(sc); misintr = 0; if (sc->xl_type == XL_TYPE_905B) { Index: sys/net/netisr.c =================================================================== --- sys/net/netisr.c +++ sys/net/netisr.c @@ -920,6 +920,7 @@ static void swi_net(void *arg) { + struct epoch_tracker et; #ifdef NETISR_LOCKING struct rm_priotracker tracker; #endif @@ -928,6 +929,7 @@ nwsp = arg; + NET_EPOCH_ENTER(et); #ifdef DEVICE_POLLING KASSERT(nws_count == 1, ("%s: device_polling but nws_count != 1", __func__)); @@ -958,6 +960,7 @@ #ifdef DEVICE_POLLING netisr_pollmore(); #endif + NET_EPOCH_EXIT(et); } static int