Index: sys/net/iflib.c =================================================================== --- sys/net/iflib.c +++ sys/net/iflib.c @@ -6382,9 +6382,10 @@ mp_ring_state_handler(SYSCTL_HANDLER_ARGS) { int rc; - uint16_t *state = ((uint16_t *)oidp->oid_arg1); + struct ifmp_ring *r = ((struct ifmp_ring *)oidp->oid_arg1); struct sbuf *sb; const char *ring_state = "UNKNOWN"; + uint16_t state[4]; /* XXX needed ? */ rc = sysctl_wire_old_buffer(req, 0); @@ -6395,6 +6396,7 @@ MPASS(sb != NULL); if (sb == NULL) return (ENOMEM); + *(uint64_t *)state = ifmp_ring_get_state(r); if (state[3] <= 3) ring_state = ring_states[state[3]]; @@ -6595,7 +6597,7 @@ CTLFLAG_RD, &txq->ift_cleaned, "total cleaned"); SYSCTL_ADD_PROC(ctx_list, queue_list, OID_AUTO, "ring_state", - CTLTYPE_STRING | CTLFLAG_RD, __DEVOLATILE(uint64_t *, &txq->ift_br->state), + CTLTYPE_STRING | CTLFLAG_RD, txq->ift_br, 0, mp_ring_state_handler, "A", "soft ring state"); SYSCTL_ADD_COUNTER_U64(ctx_list, queue_list, OID_AUTO, "r_enqueues", CTLFLAG_RD, &txq->ift_br->enqueues, Index: sys/net/mp_ring.h =================================================================== --- sys/net/mp_ring.h +++ sys/net/mp_ring.h @@ -45,7 +45,10 @@ #endif struct ifmp_ring { - volatile uint64_t state __aligned(CACHE_LINE_SIZE); +#ifndef MP_RING_NO_64BIT_ATOMICS + volatile +#endif + uint64_t state __aligned(CACHE_LINE_SIZE); int size __aligned(CACHE_LINE_SIZE); void * cookie; @@ -72,4 +75,11 @@ void ifmp_ring_reset_stats(struct ifmp_ring *); int ifmp_ring_is_idle(struct ifmp_ring *); int ifmp_ring_is_stalled(struct ifmp_ring *r); + +#ifdef MP_RING_NO_64BIT_ATOMICS +uint64_t ifmp_ring_get_state(struct ifmp_ring *r); +#else +#define ifmp_ring_get_state(r) (r->state) +#endif + #endif Index: sys/net/mp_ring.c =================================================================== --- sys/net/mp_ring.c +++ sys/net/mp_ring.c @@ -98,6 +98,18 @@ } #ifdef MP_RING_NO_64BIT_ATOMICS +/* Hopefully the compiler inlines this... */ +uint64_t +ifmp_ring_get_state(struct ifmp_ring *r) +{ + uint64_t ret; + + mtx_lock(&r->lock); + ret = r->state; + mtx_unlock(&r->lock); + return (ret); +}; + static void drain_ring_locked(struct ifmp_ring *r, union ring_state os, uint16_t prev, int budget) { @@ -117,7 +129,9 @@ while (cidx != pidx) { /* Items from cidx to pidx are available for consumption. */ + mtx_unlock(&r->lock); n = r->drain(r, cidx, pidx); + mtx_lock(&r->lock); if (n == 0) { os.state = ns.state = r->state; ns.cidx = cidx; @@ -348,21 +362,10 @@ } ns.state = os.state; ns.pidx_head = increment_idx(r, os.pidx_head, n); - r->state = ns.state; pidx_start = os.pidx_head; pidx_stop = ns.pidx_head; - /* - * Wait for other producers who got in ahead of us to enqueue their - * items, one producer at a time. It is our turn when the ring's - * pidx_tail reaches the beginning of our reservation (pidx_start). - */ - while (ns.pidx_tail != pidx_start) { - cpu_spinwait(); - ns.state = r->state; - } - - /* Now it is our turn to fill up the area we reserved earlier. */ + /* Now fill up the ring. */ i = pidx_start; do { r->items[i] = *items++; @@ -371,16 +374,14 @@ } while (i != pidx_stop); /* - * Update the ring's pidx_tail. The release style atomic guarantees - * that the items are visible to any thread that sees the updated pidx. + * Update the ring's pidx_tail. */ - os.state = ns.state = r->state; + os.state = ns.state; ns.pidx_tail = pidx_stop; if (abdicate) { if (os.flags == IDLE) ns.flags = ABDICATED; - } - else { + } else { ns.flags = BUSY; } r->state = ns.state; @@ -488,19 +489,24 @@ { union ring_state os, ns; +#ifdef MP_RING_NO_64BIT_ATOMICS + mtx_lock(&r->lock); +#endif os.state = r->state; if ((os.flags != STALLED && os.flags != ABDICATED) || // Only continue in STALLED and ABDICATED os.pidx_head != os.pidx_tail || // Require work to be available - (os.flags != ABDICATED && r->can_drain(r) == 0)) // Can either drain, or everyone left + (os.flags != ABDICATED && r->can_drain(r) == 0)) { // Can either drain, or everyone left +#ifdef MP_RING_NO_64BIT_ATOMICS + mtx_unlock(&r->lock); +#endif return; + } MPASS(os.cidx != os.pidx_tail); /* implied by STALLED */ ns.state = os.state; ns.flags = BUSY; - #ifdef MP_RING_NO_64BIT_ATOMICS - mtx_lock(&r->lock); if (r->state != os.state) { mtx_unlock(&r->lock); return; @@ -538,7 +544,7 @@ { union ring_state s; - s.state = r->state; + s.state = ifmp_ring_get_state(r); if (s.pidx_head == s.pidx_tail && s.pidx_tail == s.cidx && s.flags == IDLE) return (1); @@ -551,7 +557,7 @@ { union ring_state s; - s.state = r->state; + s.state = ifmp_ring_get_state(r); if (s.pidx_head == s.pidx_tail && s.flags == STALLED) return (1);