diff --git a/sys/dev/gve/gve_adminq.h b/sys/dev/gve/gve_adminq.h --- a/sys/dev/gve/gve_adminq.h +++ b/sys/dev/gve/gve_adminq.h @@ -377,7 +377,8 @@ _Static_assert(sizeof(struct stats) == 16, "gve: bad admin queue struct length"); -/* These are control path types for PTYPE which are the same as the data path +/* + * These are control path types for PTYPE which are the same as the data path * types. */ struct gve_ptype_entry { diff --git a/sys/dev/gve/gve_dqo.h b/sys/dev/gve/gve_dqo.h --- a/sys/dev/gve/gve_dqo.h +++ b/sys/dev/gve/gve_dqo.h @@ -208,9 +208,14 @@ #define GVE_TX_METADATA_VERSION_DQO 0 +/* Used to access the generation bit within a TX completion descriptor. */ +#define GVE_TX_DESC_DQO_GEN_BYTE_OFFSET 1 +#define GVE_TX_DESC_DQO_GEN_BIT_MASK 0x80 + /* TX completion descriptor */ struct gve_tx_compl_desc_dqo { - /* For types 0-4 this is the TX queue ID associated with this + /* + * For types 0-4 this is the TX queue ID associated with this * completion. */ uint16_t id:11; @@ -222,12 +227,14 @@ /* Flipped by HW to notify the descriptor is populated. */ uint16_t generation:1; union { - /* For descriptor completions, this is the last index fetched + /* + * For descriptor completions, this is the last index fetched * by HW + 1. */ __le16 tx_head; - /* For packet completions, this is the completion tag set on the + /* + * For packet completions, this is the completion tag set on the * TX packet descriptors. */ __le16 completion_tag; @@ -258,6 +265,10 @@ _Static_assert(sizeof(struct gve_rx_desc_dqo) == 32, "gve: bad dqo desc struct length"); +/* Used to access the generation bit within an RX completion descriptor. */ +#define GVE_RX_DESC_DQO_GEN_BYTE_OFFSET 5 +#define GVE_RX_DESC_DQO_GEN_BIT_MASK 0x40 + /* Descriptor for HW to notify SW of new packets received on RX queue. */ struct gve_rx_compl_desc_dqo { /* Must be 1 */ @@ -266,7 +277,8 @@ /* Packet originated from this system rather than the network. */ uint8_t loopback:1; - /* Set when IPv6 packet contains a destination options header or routing + /* + * Set when IPv6 packet contains a destination options header or routing * header. */ uint8_t ipv6_ex_add:1; diff --git a/sys/dev/gve/gve_rx_dqo.c b/sys/dev/gve/gve_rx_dqo.c --- a/sys/dev/gve/gve_rx_dqo.c +++ b/sys/dev/gve/gve_rx_dqo.c @@ -962,6 +962,19 @@ rx->ctx = (struct gve_rx_ctx){}; } +static uint8_t +gve_rx_get_gen_bit(uint8_t *desc) +{ + uint8_t byte; + + /* + * Prevent generation bit from being read after the rest of the + * descriptor. + */ + byte = atomic_load_acq_8(desc + GVE_RX_DESC_DQO_GEN_BYTE_OFFSET); + return ((byte & GVE_RX_DESC_DQO_GEN_BIT_MASK) != 0); +} + static bool gve_rx_cleanup_dqo(struct gve_priv *priv, struct gve_rx_ring *rx, int budget) { @@ -971,17 +984,14 @@ NET_EPOCH_ASSERT(); while (work_done < budget) { - bus_dmamap_sync(rx->dqo.compl_ring_mem.tag, rx->dqo.compl_ring_mem.map, + bus_dmamap_sync(rx->dqo.compl_ring_mem.tag, + rx->dqo.compl_ring_mem.map, BUS_DMASYNC_POSTREAD); compl_desc = &rx->dqo.compl_ring[rx->dqo.tail]; - if (compl_desc->generation == rx->dqo.cur_gen_bit) + if (gve_rx_get_gen_bit((uint8_t *)compl_desc) == + rx->dqo.cur_gen_bit) break; - /* - * Prevent generation bit from being read after the rest of the - * descriptor. - */ - atomic_thread_fence_acq(); rx->cnt++; rx->dqo.tail = (rx->dqo.tail + 1) & rx->dqo.mask; diff --git a/sys/dev/gve/gve_tx_dqo.c b/sys/dev/gve/gve_tx_dqo.c --- a/sys/dev/gve/gve_tx_dqo.c +++ b/sys/dev/gve/gve_tx_dqo.c @@ -1029,6 +1029,19 @@ gve_tx_clear_compl_ring_dqo(tx); } +static uint8_t +gve_tx_get_gen_bit(uint8_t *desc) +{ + uint8_t byte; + + /* + * Prevent generation bit from being read after the rest of the + * descriptor. + */ + byte = atomic_load_acq_8(desc + GVE_TX_DESC_DQO_GEN_BYTE_OFFSET); + return ((byte & GVE_TX_DESC_DQO_GEN_BIT_MASK) != 0); +} + static bool gve_tx_cleanup_dqo(struct gve_priv *priv, struct gve_tx_ring *tx, int budget) { @@ -1041,20 +1054,16 @@ uint16_t type; while (work_done < budget) { - bus_dmamap_sync(tx->dqo.compl_ring_mem.tag, tx->dqo.compl_ring_mem.map, + bus_dmamap_sync(tx->dqo.compl_ring_mem.tag, + tx->dqo.compl_ring_mem.map, BUS_DMASYNC_POSTREAD); compl_desc = &tx->dqo.compl_ring[tx->dqo.compl_head]; - if (compl_desc->generation == tx->dqo.cur_gen_bit) + if (gve_tx_get_gen_bit((uint8_t *)compl_desc) == + tx->dqo.cur_gen_bit) break; - /* - * Prevent generation bit from being read after the rest of the - * descriptor. - */ - atomic_thread_fence_acq(); type = compl_desc->type; - if (type == GVE_COMPL_TYPE_DQO_DESC) { /* This is the last descriptor fetched by HW plus one */ tx_head = le16toh(compl_desc->tx_head);