Index: sys/sys/buf_ring.h =================================================================== --- sys/sys/buf_ring.h +++ sys/sys/buf_ring.h @@ -94,18 +94,50 @@ * - Some effort is made to avoid extra memory barriers by using one * atomic_<>_acq_32 operation to guarantee the value of multiple values */ +#ifdef DEBUG_BUFRING +#define DUPLICATE_CHECK(br, buf) \ + do { \ + int i; \ + for (i = br->br_cons_head; i != br->br_prod_head; \ + i = ((i + 1) & br->br_cons_mask)) \ + if(br->br_ring[i] == buf) \ + panic("buf=%p already enqueue at %d prod=%d cons=%d", \ + buf, i, br->br_prod_tail, br->br_cons_tail); \ + } while (0) +#define DEQUEUE_VALIDATE(br, cons_head) \ + do { \ + br->br_ring[cons_head] = NULL; \ + if (!mtx_owned(br->br_lock)) \ + panic("lock not held on single consumer dequeue"); \ + if (br->br_cons_tail != cons_head) \ + panic("inconsistent list cons_tail=%d cons_head=%d", \ + br->br_cons_tail, cons_head); \ + } while (0) +#else +#define DUPLICATE_CHECK(br, buf) +#define DEQUEUE_VALIDATE(br, cons_head) +#endif + +#ifdef PREFETCH_DEFINED +#define DEQUEUE_PREFETCH(br, cons_head, cons_next, prod_tail) \ + do { \ + uint32_t cons_next_next = (cons_head + 2) & br->br_cons_mask; \ + if (cons_next != prod_tail) { \ + prefetch(br->br_ring[cons_next]); \ + if (cons_next_next != prod_tail) \ + prefetch(br->br_ring[cons_next_next]); \ + } \ + } while (0) +#else +#define DEQUEUE_PREFETCH(br, cons_head, cons_next, prod_tail) +#endif + static __inline int buf_ring_enqueue(struct buf_ring *br, void *buf) { uint32_t prod_head, prod_next, cons_tail; -#ifdef DEBUG_BUFRING - int i; - for (i = br->br_cons_head; i != br->br_prod_head; - i = ((i + 1) & br->br_cons_mask)) - if(br->br_ring[i] == buf) - panic("buf=%p already enqueue at %d prod=%d cons=%d", - buf, i, br->br_prod_tail, br->br_cons_tail); -#endif + + DUPLICATE_CHECK(br, buf); critical_enter(); cons_tail = atomic_load_acq_32(&br->br_cons_tail); do { @@ -198,9 +230,6 @@ buf_ring_dequeue_sc(struct buf_ring *br) { uint32_t cons_head, cons_next; -#ifdef PREFETCH_DEFINED - uint32_t cons_next_next; -#endif uint32_t prod_tail; void *buf; @@ -210,25 +239,11 @@ return (NULL); cons_next = (cons_head + 1) & br->br_cons_mask; -#ifdef PREFETCH_DEFINED - cons_next_next = (cons_head + 2) & br->br_cons_mask; - if (cons_next != prod_tail) { - prefetch(br->br_ring[cons_next]); - if (cons_next_next != prod_tail) - prefetch(br->br_ring[cons_next_next]); - } -#endif + DEQUEUE_PREFETCH(br, cons_head, cons_next, prod_tail); br->br_cons_head = cons_next; buf = br->br_ring[cons_head]; -#ifdef DEBUG_BUFRING - br->br_ring[cons_head] = NULL; - if (!mtx_owned(br->br_lock)) - panic("lock not held on single consumer dequeue"); - if (br->br_cons_tail != cons_head) - panic("inconsistent list cons_tail=%d cons_head=%d", - br->br_cons_tail, cons_head); -#endif + DEQUEUE_VALIDATE(br, cons_head); /* do atomic_store_rel_32 to assure that producer has * up to date view */