Differential D46690 Diff 144949 sys/dev/gve/gve_rx.c

Changeset View

Standalone View

sys/dev/gve/gve_rx.c

/*- /*-

* SPDX-License-Identifier: BSD-3-Clause * SPDX-License-Identifier: BSD-3-Clause

* *

* Redistribution and use in source and binary forms, with or without modification, * Redistribution and use in source and binary forms, with or without modification,

* are permitted provided that the following conditions are met: * are permitted provided that the following conditions are met:

* *

* 1. Redistributions of source code must retain the above copyright notice, this * 1. Redistributions of source code must retain the above copyright notice, this

* list of conditions and the following disclaimer. * list of conditions and the following disclaimer.

* *

* 2. Redistributions in binary form must reproduce the above copyright notice, * 2. Redistributions in binary form must reproduce the above copyright notice,

Show All 12 Lines

* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;

* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON

* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT

* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS

* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

*/ */

#include "gve.h" #include "gve.h"

#include "gve_adminq.h" #include "gve_adminq.h"

#include "gve_dqo.h"

static void static void

gve_rx_free_ring(struct gve_priv *priv, int i) gve_rx_free_ring_gqi(struct gve_priv *priv, int i)

{ {

struct gve_rx_ring *rx = &priv->rx[i]; struct gve_rx_ring *rx = &priv->rx[i];

struct gve_ring_com *com = &rx->com;

/* Safe to call even if never allocated */

gve_free_counters((counter_u64_t *)&rx->stats, NUM_RX_STATS);

if (rx->page_info != NULL) { if (rx->page_info != NULL) {

free(rx->page_info, M_GVE); free(rx->page_info, M_GVE);

rx->page_info = NULL; rx->page_info = NULL;

} }

if (rx->data_ring != NULL) { if (rx->data_ring != NULL) {

gve_dma_free_coherent(&rx->data_ring_mem); gve_dma_free_coherent(&rx->data_ring_mem);

rx->data_ring = NULL; rx->data_ring = NULL;

} }

if (rx->desc_ring != NULL) { if (rx->desc_ring != NULL) {

gve_dma_free_coherent(&rx->desc_ring_mem); gve_dma_free_coherent(&rx->desc_ring_mem);

rx->desc_ring = NULL; rx->desc_ring = NULL;

} }

}

static void

gve_rx_free_ring(struct gve_priv *priv, int i)

{

struct gve_rx_ring *rx = &priv->rx[i];

struct gve_ring_com *com = &rx->com;

/* Safe to call even if never allocated */

gve_free_counters((counter_u64_t *)&rx->stats, NUM_RX_STATS);

if (gve_is_gqi(priv))

gve_rx_free_ring_gqi(priv, i);

else

gve_rx_free_ring_dqo(priv, i);

if (com->q_resources != NULL) { if (com->q_resources != NULL) {

gve_dma_free_coherent(&com->q_resources_mem); gve_dma_free_coherent(&com->q_resources_mem);

com->q_resources = NULL; com->q_resources = NULL;

} }

static void static void

gve_prefill_rx_slots(struct gve_rx_ring *rx) gve_prefill_rx_slots(struct gve_rx_ring *rx)

Show All 12 Lines for (i = 0; i < com->priv->rx_desc_cnt; i++) {

bus_dmamap_sync(dma->tag, dma->map, BUS_DMASYNC_PREREAD); bus_dmamap_sync(dma->tag, dma->map, BUS_DMASYNC_PREREAD);

} }

bus_dmamap_sync(rx->data_ring_mem.tag, rx->data_ring_mem.map, bus_dmamap_sync(rx->data_ring_mem.tag, rx->data_ring_mem.map,

BUS_DMASYNC_PREWRITE); BUS_DMASYNC_PREWRITE);

} }

static int static int

gve_rx_alloc_ring(struct gve_priv *priv, int i) gve_rx_alloc_ring_gqi(struct gve_priv *priv, int i)

{ {

struct gve_rx_ring *rx = &priv->rx[i]; struct gve_rx_ring *rx = &priv->rx[i];

struct gve_ring_com *com = &rx->com; struct gve_ring_com *com = &rx->com;

int err; int err;

com->priv = priv; err = gve_dma_alloc_coherent(priv,

com->id = i; sizeof(struct gve_rx_desc) * priv->rx_desc_cnt,

CACHE_LINE_SIZE, &rx->desc_ring_mem);

if (err != 0) {

device_printf(priv->dev,

"Failed to alloc desc ring for rx ring %d", i);

goto abort;

}

rx->mask = priv->rx_pages_per_qpl - 1; rx->mask = priv->rx_pages_per_qpl - 1;

rx->desc_ring = rx->desc_ring_mem.cpu_addr;

com->qpl = &priv->qpls[priv->tx_cfg.max_queues + i]; com->qpl = &priv->qpls[priv->tx_cfg.max_queues + i];

if (com->qpl == NULL) { if (com->qpl == NULL) {

device_printf(priv->dev, "No QPL left for rx ring %d", i); device_printf(priv->dev, "No QPL left for rx ring %d", i);

return (ENOMEM); return (ENOMEM);

} }

rx->page_info = malloc(priv->rx_desc_cnt * sizeof(*rx->page_info), M_GVE, rx->page_info = malloc(priv->rx_desc_cnt * sizeof(*rx->page_info),

M_WAITOK | M_ZERO); M_GVE, M_WAITOK | M_ZERO);

err = gve_dma_alloc_coherent(priv,

sizeof(union gve_rx_data_slot) * priv->rx_desc_cnt,

CACHE_LINE_SIZE, &rx->data_ring_mem);

if (err != 0) {

device_printf(priv->dev,

"Failed to alloc data ring for rx ring %d", i);

goto abort;

}

rx->data_ring = rx->data_ring_mem.cpu_addr;

gve_prefill_rx_slots(rx);

return (0);

abort:

gve_rx_free_ring_gqi(priv, i);

return (err);

}

static int

gve_rx_alloc_ring(struct gve_priv *priv, int i)

{

struct gve_rx_ring *rx = &priv->rx[i];

struct gve_ring_com *com = &rx->com;

int err;

com->priv = priv;

com->id = i;

gve_alloc_counters((counter_u64_t *)&rx->stats, NUM_RX_STATS); gve_alloc_counters((counter_u64_t *)&rx->stats, NUM_RX_STATS);

err = gve_dma_alloc_coherent(priv, sizeof(struct gve_queue_resources), err = gve_dma_alloc_coherent(priv, sizeof(struct gve_queue_resources),

PAGE_SIZE, &com->q_resources_mem); PAGE_SIZE, &com->q_resources_mem);

if (err != 0) { if (err != 0) {

device_printf(priv->dev, "Failed to alloc queue resources for rx ring %d", i); device_printf(priv->dev,

"Failed to alloc queue resources for rx ring %d", i);

goto abort; goto abort;

} }

com->q_resources = com->q_resources_mem.cpu_addr; com->q_resources = com->q_resources_mem.cpu_addr;

err = gve_dma_alloc_coherent(priv, if (gve_is_gqi(priv))

sizeof(struct gve_rx_desc) * priv->rx_desc_cnt, err = gve_rx_alloc_ring_gqi(priv, i);

CACHE_LINE_SIZE, &rx->desc_ring_mem); else

if (err != 0) { err = gve_rx_alloc_ring_dqo(priv, i);

device_printf(priv->dev, "Failed to alloc desc ring for rx ring %d", i); if (err != 0)

goto abort; goto abort;

}

rx->desc_ring = rx->desc_ring_mem.cpu_addr;

err = gve_dma_alloc_coherent(priv,

sizeof(union gve_rx_data_slot) * priv->rx_desc_cnt,

CACHE_LINE_SIZE, &rx->data_ring_mem);

if (err != 0) {

device_printf(priv->dev, "Failed to alloc data ring for rx ring %d", i);

goto abort;

}

rx->data_ring = rx->data_ring_mem.cpu_addr;

gve_prefill_rx_slots(rx);

return (0); return (0);

abort: abort:

gve_rx_free_ring(priv, i); gve_rx_free_ring(priv, i);

return (err); return (err);

} }

int int

▲ Show 20 Lines • Show All 79 Lines • ▼ Show 20 Lines gve_clear_rx_ring(struct gve_priv *priv, int i)

rx->fill_cnt = 0; rx->fill_cnt = 0;

rx->mask = priv->rx_desc_cnt - 1; rx->mask = priv->rx_desc_cnt - 1;

gve_rx_clear_desc_ring(rx); gve_rx_clear_desc_ring(rx);

gve_rx_clear_data_ring(rx); gve_rx_clear_data_ring(rx);

} }

static void static void

gve_start_rx_ring(struct gve_priv *priv, int i) gve_start_rx_ring(struct gve_priv *priv, int i,

void (cleanup) (void *arg, int pending))

markjUnsubmitted

Done

gve_start_rx_ring(struct gve_priv *priv, int i,

- void (cleanup) (void *arg, int pending))

+ void (cleanup)(void *arg, int pending))

{

struct gve_rx_ring *rx = &priv->rx[i];

Why does the caller have to pass the callback as a parameter? This function operates on both types of rings, it seems natural for it to hardcode the two cleanup routines that are used, so that the caller doesn't always have to check gve_is_gqi(priv).

markj: Why does the caller have to pass the callback as a parameter? This function operates on both…

{ {

struct gve_rx_ring *rx = &priv->rx[i]; struct gve_rx_ring *rx = &priv->rx[i];

struct gve_ring_com *com = &rx->com; struct gve_ring_com *com = &rx->com;

if ((if_getcapenable(priv->ifp) & IFCAP_LRO) != 0) { if ((if_getcapenable(priv->ifp) & IFCAP_LRO) != 0) {

if (tcp_lro_init(&rx->lro) != 0) if (tcp_lro_init(&rx->lro) != 0)

device_printf(priv->dev, "Failed to init lro for rx ring %d", i); device_printf(priv->dev, "Failed to init lro for rx ring %d", i);

rx->lro.ifp = priv->ifp; rx->lro.ifp = priv->ifp;

} }

NET_TASK_INIT(&com->cleanup_task, 0, gve_rx_cleanup_tq, rx); NET_TASK_INIT(&com->cleanup_task, 0, cleanup, rx);

com->cleanup_tq = taskqueue_create_fast("gve rx", M_WAITOK, com->cleanup_tq = taskqueue_create_fast("gve rx", M_WAITOK,

taskqueue_thread_enqueue, &com->cleanup_tq); taskqueue_thread_enqueue, &com->cleanup_tq);

taskqueue_start_threads(&com->cleanup_tq, 1, PI_NET, taskqueue_start_threads(&com->cleanup_tq, 1, PI_NET,

"%s rxq %d", device_get_nameunit(priv->dev), i); "%s rxq %d", device_get_nameunit(priv->dev), i);

if (gve_is_gqi(priv)) {

/* GQ RX bufs are prefilled at ring alloc time */

gve_db_bar_write_4(priv, com->db_offset, rx->fill_cnt); gve_db_bar_write_4(priv, com->db_offset, rx->fill_cnt);

} else {

gve_rx_prefill_buffers_dqo(rx);

} }

}

int int

gve_create_rx_rings(struct gve_priv *priv) gve_create_rx_rings(struct gve_priv *priv)

{ {

struct gve_ring_com *com; struct gve_ring_com *com;

struct gve_rx_ring *rx; struct gve_rx_ring *rx;

int err; int err;

int i; int i;

if (gve_get_state_flag(priv, GVE_STATE_FLAG_RX_RINGS_OK)) if (gve_get_state_flag(priv, GVE_STATE_FLAG_RX_RINGS_OK))

return (0); return (0);

for (i = 0; i < priv->rx_cfg.num_queues; i++) for (i = 0; i < priv->rx_cfg.num_queues; i++) {

if (gve_is_gqi(priv))

gve_clear_rx_ring(priv, i); gve_clear_rx_ring(priv, i);

else

gve_clear_rx_ring_dqo(priv, i);

markjUnsubmitted

Done

It seems a bit more consistent for gve_clear_rx_ring() to call gve_clear_rx_ring_dqo() itself.

markj: It seems a bit more consistent for `gve_clear_rx_ring()` to call `gve_clear_rx_ring_dqo()`…

}

err = gve_adminq_create_rx_queues(priv, priv->rx_cfg.num_queues); err = gve_adminq_create_rx_queues(priv, priv->rx_cfg.num_queues);

if (err != 0) if (err != 0)

return (err); return (err);

bus_dmamap_sync(priv->irqs_db_mem.tag, priv->irqs_db_mem.map, bus_dmamap_sync(priv->irqs_db_mem.tag, priv->irqs_db_mem.map,

BUS_DMASYNC_POSTREAD); BUS_DMASYNC_POSTREAD);

for (i = 0; i < priv->rx_cfg.num_queues; i++) { for (i = 0; i < priv->rx_cfg.num_queues; i++) {

rx = &priv->rx[i]; rx = &priv->rx[i];

com = &rx->com; com = &rx->com;

com->irq_db_offset = 4 * be32toh(priv->irq_db_indices[com->ntfy_id].index); com->irq_db_offset = 4 * be32toh(priv->irq_db_indices[com->ntfy_id].index);

bus_dmamap_sync(com->q_resources_mem.tag, com->q_resources_mem.map, bus_dmamap_sync(com->q_resources_mem.tag, com->q_resources_mem.map,

BUS_DMASYNC_POSTREAD); BUS_DMASYNC_POSTREAD);

com->db_offset = 4 * be32toh(com->q_resources->db_index); com->db_offset = 4 * be32toh(com->q_resources->db_index);

com->counter_idx = be32toh(com->q_resources->counter_index); com->counter_idx = be32toh(com->q_resources->counter_index);

gve_start_rx_ring(priv, i); if (gve_is_gqi(priv))

gve_start_rx_ring(priv, i, gve_rx_cleanup_tq);

else

gve_start_rx_ring(priv, i, gve_rx_cleanup_tq_dqo);

} }

gve_set_state_flag(priv, GVE_STATE_FLAG_RX_RINGS_OK); gve_set_state_flag(priv, GVE_STATE_FLAG_RX_RINGS_OK);

return (0); return (0);

} }

static void static void

gve_stop_rx_ring(struct gve_priv *priv, int i) gve_stop_rx_ring(struct gve_priv *priv, int i)

▲ Show 20 Lines • Show All 394 Lines • Show Last 20 Lines