diff --git a/sys/net/if_epair.c b/sys/net/if_epair.c --- a/sys/net/if_epair.c +++ b/sys/net/if_epair.c @@ -101,15 +101,16 @@ #define EPAIR_LOCK() mtx_lock(&epair_n_index_mtx) #define EPAIR_UNLOCK() mtx_unlock(&epair_n_index_mtx) -#define BIT_QUEUE_TASK 0 -#define BIT_MBUF_QUEUED 1 - struct epair_softc; struct epair_queue { + struct mtx mtx; + struct mbufq q; int id; - struct buf_ring *rxring[2]; - volatile int ridx; /* 0 || 1 */ - volatile long state; /* taskqueue coordination */ + enum { + EPAIR_QUEUE_IDLE, + EPAIR_QUEUE_WAKING, + EPAIR_QUEUE_RUNNING, + } state; struct task tx_task; struct epair_softc *sc; }; @@ -145,44 +146,49 @@ } static void -epair_if_input(struct epair_softc *sc, struct epair_queue *q, int ridx) +epair_tx_start_deferred(void *arg, int pending) { - struct ifnet *ifp; - struct mbuf *m; + struct epair_queue *q = (struct epair_queue *)arg; + if_t ifp; + struct mbuf *m, *n; + bool resched; - ifp = sc->ifp; + ifp = q->sc->ifp; + + if_ref(ifp); CURVNET_SET(ifp->if_vnet); - while (! buf_ring_empty(q->rxring[ridx])) { - m = buf_ring_dequeue_mc(q->rxring[ridx]); - if (m == NULL) - continue; - MPASS((m->m_pkthdr.csum_flags & CSUM_SND_TAG) == 0); - (*ifp->if_input)(ifp, m); + mtx_lock(&q->mtx); + m = mbufq_flush(&q->q); + q->state = EPAIR_QUEUE_RUNNING; + mtx_unlock(&q->mtx); + while (m != NULL) { + n = STAILQ_NEXT(m, m_stailqpkt); + m->m_nextpkt = NULL; + if_input(ifp, m); + m = n; } - CURVNET_RESTORE(); -} -static void -epair_tx_start_deferred(void *arg, int pending) -{ - struct epair_queue *q = (struct epair_queue *)arg; - struct epair_softc *sc = q->sc; - int ridx, nidx; - - if_ref(sc->ifp); - ridx = atomic_load_int(&q->ridx); - do { - nidx = (ridx == 0) ? 1 : 0; - } while (!atomic_fcmpset_int(&q->ridx, &ridx, nidx)); - epair_if_input(sc, q, ridx); - - atomic_clear_long(&q->state, (1 << BIT_QUEUE_TASK)); - if (atomic_testandclear_long(&q->state, BIT_MBUF_QUEUED)) + /* + * Avoid flushing the queue more than once per task. We can otherwise + * end up starving ourselves in a multi-epair routing configuration. + */ + mtx_lock(&q->mtx); + if (mbufq_len(&q->q) > 0) { + resched = true; + q->state = EPAIR_QUEUE_WAKING; + } else { + resched = false; + q->state = EPAIR_QUEUE_IDLE; + } + mtx_unlock(&q->mtx); + + if (resched) taskqueue_enqueue(epair_tasks.tq[q->id], &q->tx_task); - if_rele(sc->ifp); + CURVNET_RESTORE(); + if_rele(ifp); } static struct epair_queue * @@ -236,9 +242,9 @@ static void epair_menq(struct mbuf *m, struct epair_softc *osc) { + struct epair_queue *q; struct ifnet *ifp, *oifp; - int len, ret; - int ridx; + int error, len; bool mcast; /* @@ -254,32 +260,26 @@ len = m->m_pkthdr.len; mcast = (m->m_flags & (M_BCAST | M_MCAST)) != 0; - struct epair_queue *q = epair_select_queue(osc, m); + q = epair_select_queue(osc, m); - atomic_set_long(&q->state, (1 << BIT_MBUF_QUEUED)); - ridx = atomic_load_int(&q->ridx); - ret = buf_ring_enqueue(q->rxring[ridx], m); - if (ret != 0) { - /* Ring is full. */ - if_inc_counter(ifp, IFCOUNTER_OQDROPS, 1); - m_freem(m); - return; + mtx_lock(&q->mtx); + if (q->state == EPAIR_QUEUE_IDLE) { + q->state = EPAIR_QUEUE_WAKING; + taskqueue_enqueue(epair_tasks.tq[q->id], &q->tx_task); } + error = mbufq_enqueue(&q->q, m); + mtx_unlock(&q->mtx); - if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); - /* - * IFQ_HANDOFF_ADJ/ip_handoff() update statistics, - * but as we bypass all this we have to duplicate - * the logic another time. - */ - if_inc_counter(ifp, IFCOUNTER_OBYTES, len); - if (mcast) - if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1); - /* Someone else received the packet. */ - if_inc_counter(oifp, IFCOUNTER_IPACKETS, 1); - - if (!atomic_testandset_long(&q->state, BIT_QUEUE_TASK)) - taskqueue_enqueue(epair_tasks.tq[q->id], &q->tx_task); + if (error != 0) { + m_freem(m); + if_inc_counter(ifp, IFCOUNTER_OQDROPS, 1); + } else { + if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); + if_inc_counter(ifp, IFCOUNTER_OBYTES, len); + if (mcast) + if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1); + if_inc_counter(oifp, IFCOUNTER_IPACKETS, 1); + } } static void @@ -514,10 +514,9 @@ for (int i = 0; i < sc->num_queues; i++) { struct epair_queue *q = &sc->queues[i]; q->id = i; - q->rxring[0] = buf_ring_alloc(RXRSIZE, M_EPAIR, M_WAITOK, NULL); - q->rxring[1] = buf_ring_alloc(RXRSIZE, M_EPAIR, M_WAITOK, NULL); - q->ridx = 0; - q->state = 0; + q->state = EPAIR_QUEUE_IDLE; + mtx_init(&q->mtx, "epairq", NULL, MTX_DEF | MTX_NEW); + mbufq_init(&q->q, RXRSIZE); q->sc = sc; NET_TASK_INIT(&q->tx_task, 0, epair_tx_start_deferred, q); } @@ -610,8 +609,7 @@ ifmedia_removeall(&sc->media); for (int i = 0; i < sc->num_queues; i++) { struct epair_queue *q = &sc->queues[i]; - buf_ring_free(q->rxring[0], M_EPAIR); - buf_ring_free(q->rxring[1], M_EPAIR); + mtx_destroy(&q->mtx); } free(sc->queues, M_EPAIR); free(sc, M_EPAIR); @@ -756,18 +754,18 @@ static void epair_drain_rings(struct epair_softc *sc) { - int ridx; - struct mbuf *m; + for (int i = 0; i < sc->num_queues; i++) { + struct epair_queue *q; + struct mbuf *m, *n; - for (ridx = 0; ridx < 2; ridx++) { - for (int i = 0; i < sc->num_queues; i++) { - struct epair_queue *q = &sc->queues[i]; - do { - m = buf_ring_dequeue_sc(q->rxring[ridx]); - if (m == NULL) - break; - m_freem(m); - } while (1); + q = &sc->queues[i]; + mtx_lock(&q->mtx); + m = mbufq_flush(&q->q); + mtx_unlock(&q->mtx); + + for (; m != NULL; m = n) { + n = m->m_nextpkt; + m_freem(m); } } }