Index: sys/net/if_epair.c =================================================================== --- sys/net/if_epair.c +++ sys/net/if_epair.c @@ -53,7 +53,7 @@ #include #include #include -#include +#include #include #include #include @@ -90,21 +90,19 @@ #define EPAIR_LOCK() mtx_lock(&epair_n_index_mtx) #define EPAIR_UNLOCK() mtx_unlock(&epair_n_index_mtx) -static void *swi_cookie[MAXCPU]; /* swi(9). */ -static STAILQ_HEAD(, epair_softc) swi_sc[MAXCPU]; - static struct mtx epair_n_index_mtx; struct epair_softc { - struct ifnet *ifp; /* This ifp. */ - struct ifnet *oifp; /* other ifp of pair. */ - void *swi_cookie; /* swi(9). */ - struct buf_ring *rxring[2]; - volatile int ridx; /* 0 || 1 */ - struct ifmedia media; /* Media config (fake). */ - uint32_t cpuidx; + struct ifnet *ifp; /* This ifp. */ + struct ifnet *oifp; /* other ifp of pair. */ + struct buf_ring *rxring[2]; + volatile int ridx; /* 0 || 1 */ + struct task tx_task; + struct ifmedia media; /* Media config (fake). */ STAILQ_ENTRY(epair_softc) entry; }; +static struct taskqueue *epair_taskq; + static void epair_clear_mbuf(struct mbuf *m) { @@ -121,57 +119,39 @@ static void epair_if_input(struct epair_softc *sc, int ridx) { - struct epoch_tracker et; struct ifnet *ifp; struct mbuf *m; ifp = sc->ifp; - NET_EPOCH_ENTER(et); - do { - m = buf_ring_dequeue_sc(sc->rxring[ridx]); + CURVNET_SET(ifp->if_vnet); + while (! buf_ring_empty(sc->rxring[ridx])) { + m = buf_ring_dequeue_mc(sc->rxring[ridx]); if (m == NULL) - break; + continue; MPASS((m->m_pkthdr.csum_flags & CSUM_SND_TAG) == 0); (*ifp->if_input)(ifp, m); - - } while (1); - NET_EPOCH_EXIT(et); + } + CURVNET_RESTORE(); } static void -epair_sintr(struct epair_softc *sc) +epair_tx_start_deferred(void *arg, int pending) { + struct epair_softc *sc = (struct epair_softc *)arg; int ridx, nidx; if_ref(sc->ifp); do { - ridx = sc->ridx; + ridx = atomic_load_int(&sc->ridx); nidx = (ridx == 0) ? 1 : 0; } while (!atomic_cmpset_int(&sc->ridx, ridx, nidx)); epair_if_input(sc, ridx); - if_rele(sc->ifp); -} - -static void -epair_intr(void *arg) -{ - struct epair_softc *sc; - uint32_t cpuidx; + if (! buf_ring_empty(sc->rxring[nidx])) + taskqueue_enqueue(epair_taskq, &sc->tx_task); - cpuidx = (uintptr_t)arg; - /* If this is a problem, this is a read-mostly situation. */ - EPAIR_LOCK(); - STAILQ_FOREACH(sc, &swi_sc[cpuidx], entry) { - /* Do this lockless. */ - if (buf_ring_empty(sc->rxring[sc->ridx])) - continue; - epair_sintr(sc); - } - EPAIR_UNLOCK(); - - return; + if_rele(sc->ifp); } static int @@ -207,8 +187,9 @@ ret = buf_ring_enqueue(osc->rxring[ridx], m); if (ret != 0) { /* Ring is full. */ + if_inc_counter(ifp, IFCOUNTER_OQDROPS, 1); m_freem(m); - return (0); + goto done; } if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); @@ -223,9 +204,9 @@ /* Someone else received the packet. */ if_inc_counter(oifp, IFCOUNTER_IPACKETS, 1); - /* Kick the interrupt handler for the first packet. */ - if (was_empty && osc->swi_cookie != NULL) - swi_sched(osc->swi_cookie, 0); +done: + if (was_empty) + taskqueue_enqueue(epair_taskq, &osc->tx_task); return (0); } @@ -500,8 +481,10 @@ ifc_free_unit(ifc, unit); return (ENOSPC); } - sca->rxring[0] = buf_ring_alloc(RXRSIZE, M_EPAIR, M_WAITOK,NULL); + sca->rxring[0] = buf_ring_alloc(RXRSIZE, M_EPAIR, M_WAITOK, NULL); sca->rxring[1] = buf_ring_alloc(RXRSIZE, M_EPAIR, M_WAITOK, NULL); + sca->ridx = 0; + NET_TASK_INIT(&sca->tx_task, 0, epair_tx_start_deferred, sca); scb = malloc(sizeof(struct epair_softc), M_EPAIR, M_WAITOK | M_ZERO); scb->ifp = if_alloc(IFT_ETHER); @@ -514,6 +497,8 @@ } scb->rxring[0] = buf_ring_alloc(RXRSIZE, M_EPAIR, M_WAITOK, NULL); scb->rxring[1] = buf_ring_alloc(RXRSIZE, M_EPAIR, M_WAITOK, NULL); + scb->ridx = 0; + NET_TASK_INIT(&scb->tx_task, 0, epair_tx_start_deferred, scb); /* * Cross-reference the interfaces so we will be able to free both. @@ -528,41 +513,6 @@ #else hash = 0; #endif - if (swi_cookie[hash] == NULL) { - void *cookie; - - EPAIR_UNLOCK(); - error = swi_add(NULL, epairname, - epair_intr, (void *)(uintptr_t)hash, - SWI_NET, INTR_MPSAFE, &cookie); - if (error) { - buf_ring_free(scb->rxring[0], M_EPAIR); - buf_ring_free(scb->rxring[1], M_EPAIR); - if_free(scb->ifp); - free(scb, M_EPAIR); - buf_ring_free(sca->rxring[0], M_EPAIR); - buf_ring_free(sca->rxring[1], M_EPAIR); - if_free(sca->ifp); - free(sca, M_EPAIR); - ifc_free_unit(ifc, unit); - return (ENOSPC); - } - EPAIR_LOCK(); - /* Recheck under lock even though a race is very unlikely. */ - if (swi_cookie[hash] == NULL) { - swi_cookie[hash] = cookie; - } else { - EPAIR_UNLOCK(); - (void) swi_remove(cookie); - EPAIR_LOCK(); - } - } - sca->cpuidx = hash; - STAILQ_INSERT_TAIL(&swi_sc[hash], sca, entry); - sca->swi_cookie = swi_cookie[hash]; - scb->cpuidx = hash; - STAILQ_INSERT_TAIL(&swi_sc[hash], scb, entry); - scb->swi_cookie = swi_cookie[hash]; EPAIR_UNLOCK(); /* Initialise pseudo media types. */ @@ -665,10 +615,9 @@ static void epair_drain_rings(struct epair_softc *sc) { - int ridx; struct mbuf *m; - for (ridx = 0; ridx < 2; ridx++) { + for (int ridx = 0; ridx < 2; ridx++) { do { m = buf_ring_dequeue_sc(sc->rxring[ridx]); if (m == NULL) @@ -707,14 +656,6 @@ ether_ifdetach(ifp); ether_ifdetach(oifp); - /* Second stop interrupt handler. */ - EPAIR_LOCK(); - STAILQ_REMOVE(&swi_sc[sca->cpuidx], sca, epair_softc, entry); - STAILQ_REMOVE(&swi_sc[scb->cpuidx], scb, epair_softc, entry); - EPAIR_UNLOCK(); - sca->swi_cookie = NULL; - scb->swi_cookie = NULL; - /* Third free any queued packets and all the resources. */ CURVNET_SET_QUIET(oifp->if_vnet); epair_drain_rings(scb); @@ -762,34 +703,42 @@ VNET_SYSUNINIT(vnet_epair_uninit, SI_SUB_INIT_IF, SI_ORDER_ANY, vnet_epair_uninit, NULL); +static int +epair_mod_init() +{ + char name[32] = { "epair_task" }; + + epair_taskq = taskqueue_create(name, M_WAITOK, + taskqueue_thread_enqueue, + &epair_taskq); + taskqueue_start_threads(&epair_taskq, 1, PI_NET, "%s", name); + + return (0); +} + +static void +epair_mod_cleanup() +{ + taskqueue_drain_all(epair_taskq); + taskqueue_free(epair_taskq); +} + static int epair_modevent(module_t mod, int type, void *data) { - int i; + int ret; switch (type) { case MOD_LOAD: - for (i = 0; i < MAXCPU; i++) { - swi_cookie[i] = NULL; - STAILQ_INIT(&swi_sc[i]); - } EPAIR_LOCK_INIT(); + ret = epair_mod_init(); + if (ret != 0) + return (ret); if (bootverbose) printf("%s: %s initialized.\n", __func__, epairname); break; case MOD_UNLOAD: - EPAIR_LOCK(); - for (i = 0; i < MAXCPU; i++) { - if (!STAILQ_EMPTY(&swi_sc[i])) { - printf("%s: swi_sc[%d] active\n", __func__, i); - EPAIR_UNLOCK(); - return (EBUSY); - } - } - EPAIR_UNLOCK(); - for (i = 0; i < MAXCPU; i++) - if (swi_cookie[i] != NULL) - (void) swi_remove(swi_cookie[i]); + epair_mod_cleanup(); EPAIR_LOCK_DESTROY(); if (bootverbose) printf("%s: %s unloaded.\n", __func__, epairname);