diff --git a/sys/dev/gve/gve.h b/sys/dev/gve/gve.h --- a/sys/dev/gve/gve.h +++ b/sys/dev/gve/gve.h @@ -542,7 +542,6 @@ struct gve_irq_db *irq_db_indices; enum gve_queue_format queue_format; - struct gve_queue_page_list *qpls; struct gve_queue_config tx_cfg; struct gve_queue_config rx_cfg; uint32_t num_queues; @@ -629,8 +628,9 @@ void gve_db_bar_dqo_write_4(struct gve_priv *priv, bus_size_t offset, uint32_t val); /* QPL (Queue Page List) functions defined in gve_qpl.c */ -int gve_alloc_qpls(struct gve_priv *priv); -void gve_free_qpls(struct gve_priv *priv); +struct gve_queue_page_list *gve_alloc_qpl(struct gve_priv *priv, uint32_t id, + int npages, bool single_kva); +void gve_free_qpl(struct gve_priv *priv, struct gve_queue_page_list *qpl); int gve_register_qpls(struct gve_priv *priv); int gve_unregister_qpls(struct gve_priv *priv); void gve_mextadd_free(struct mbuf *mbuf); diff --git a/sys/dev/gve/gve_main.c b/sys/dev/gve/gve_main.c --- a/sys/dev/gve/gve_main.c +++ b/sys/dev/gve/gve_main.c @@ -482,8 +482,6 @@ gve_free_irqs(priv); gve_free_tx_rings(priv); gve_free_rx_rings(priv); - if (gve_is_qpl(priv)) - gve_free_qpls(priv); } static int @@ -491,12 +489,6 @@ { int err; - if (gve_is_qpl(priv)) { - err = gve_alloc_qpls(priv); - if (err != 0) - goto abort; - } - err = gve_alloc_rx_rings(priv); if (err != 0) goto abort; diff --git a/sys/dev/gve/gve_qpl.c b/sys/dev/gve/gve_qpl.c --- a/sys/dev/gve/gve_qpl.c +++ b/sys/dev/gve/gve_qpl.c @@ -36,28 +36,9 @@ static MALLOC_DEFINE(M_GVE_QPL, "gve qpl", "gve qpl allocations"); -static uint32_t -gve_num_tx_qpls(struct gve_priv *priv) -{ - if (!gve_is_qpl(priv)) - return (0); - - return (priv->tx_cfg.max_queues); -} - -static uint32_t -gve_num_rx_qpls(struct gve_priv *priv) -{ - if (!gve_is_qpl(priv)) - return (0); - - return (priv->rx_cfg.max_queues); -} - -static void -gve_free_qpl(struct gve_priv *priv, uint32_t id) +void +gve_free_qpl(struct gve_priv *priv, struct gve_queue_page_list *qpl) { - struct gve_queue_page_list *qpl = &priv->qpls[id]; int i; for (i = 0; i < qpl->num_dmas; i++) { @@ -92,12 +73,14 @@ if (qpl->dmas != NULL) free(qpl->dmas, M_GVE_QPL); + + free(qpl, M_GVE_QPL); } -static int +struct gve_queue_page_list * gve_alloc_qpl(struct gve_priv *priv, uint32_t id, int npages, bool single_kva) { - struct gve_queue_page_list *qpl = &priv->qpls[id]; + struct gve_queue_page_list *qpl; int err; int i; @@ -105,9 +88,12 @@ device_printf(priv->dev, "Reached max number of registered pages %ju > %ju\n", (uintmax_t)npages + priv->num_registered_pages, (uintmax_t)priv->max_registered_pages); - return (EINVAL); + return (NULL); } + qpl = malloc(sizeof(struct gve_queue_page_list), M_GVE_QPL, + M_WAITOK | M_ZERO); + qpl->id = id; qpl->num_pages = 0; qpl->num_dmas = 0; @@ -163,126 +149,90 @@ priv->num_registered_pages++; } - return (0); + return (qpl); abort: - gve_free_qpl(priv, id); - return (err); + gve_free_qpl(priv, qpl); + return (NULL); } -void -gve_free_qpls(struct gve_priv *priv) -{ - int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv); - int i; - - if (num_qpls == 0) - return; - - if (priv->qpls != NULL) { - for (i = 0; i < num_qpls; i++) - gve_free_qpl(priv, i); - free(priv->qpls, M_GVE_QPL); - priv->qpls = NULL; - } -} - -int gve_alloc_qpls(struct gve_priv *priv) +int +gve_register_qpls(struct gve_priv *priv) { - int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv); - int num_pages; + struct gve_ring_com *com; + struct gve_tx_ring *tx; + struct gve_rx_ring *rx; int err; int i; - if (num_qpls == 0) + if (gve_get_state_flag(priv, GVE_STATE_FLAG_QPLREG_OK)) return (0); - priv->qpls = malloc(num_qpls * sizeof(*priv->qpls), M_GVE_QPL, - M_WAITOK | M_ZERO); - - num_pages = gve_is_gqi(priv) ? - priv->tx_desc_cnt / GVE_QPL_DIVISOR : - GVE_TX_NUM_QPL_PAGES_DQO; - for (i = 0; i < gve_num_tx_qpls(priv); i++) { - err = gve_alloc_qpl(priv, i, num_pages, - /*single_kva=*/true); - if (err != 0) - goto abort; - } - - num_pages = gve_is_gqi(priv) ? priv->rx_desc_cnt : GVE_RX_NUM_QPL_PAGES_DQO; - for (; i < num_qpls; i++) { - err = gve_alloc_qpl(priv, i, num_pages, /*single_kva=*/false); - if (err != 0) - goto abort; - } - - return (0); - -abort: - gve_free_qpls(priv); - return (err); -} - -static int -gve_unregister_n_qpls(struct gve_priv *priv, int n) -{ - int err; - int i; - - for (i = 0; i < n; i++) { - err = gve_adminq_unregister_page_list(priv, priv->qpls[i].id); + /* Register TX qpls */ + for (i = 0; i < priv->tx_cfg.num_queues; i++) { + tx = &priv->tx[i]; + com = &tx->com; + err = gve_adminq_register_page_list(priv, com->qpl); if (err != 0) { device_printf(priv->dev, - "Failed to unregister qpl %d, err: %d\n", - priv->qpls[i].id, err); + "Failed to register qpl %d, err: %d\n", + com->qpl->id, err); + /* Caller schedules a reset when this fails */ + return (err); } } - if (err != 0) - return (err); - - return (0); -} - -int -gve_register_qpls(struct gve_priv *priv) -{ - int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv); - int err; - int i; - - if (gve_get_state_flag(priv, GVE_STATE_FLAG_QPLREG_OK)) - return (0); - - for (i = 0; i < num_qpls; i++) { - err = gve_adminq_register_page_list(priv, &priv->qpls[i]); + /* Register RX qpls */ + for (i = 0; i < priv->rx_cfg.num_queues; i++) { + rx = &priv->rx[i]; + com = &rx->com; + err = gve_adminq_register_page_list(priv, com->qpl); if (err != 0) { device_printf(priv->dev, "Failed to register qpl %d, err: %d\n", - priv->qpls[i].id, err); - goto abort; + com->qpl->id, err); + /* Caller schedules a reset when this fails */ + return (err); } } - gve_set_state_flag(priv, GVE_STATE_FLAG_QPLREG_OK); return (0); - -abort: - gve_unregister_n_qpls(priv, i); - return (err); } int gve_unregister_qpls(struct gve_priv *priv) { - int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv); int err; + int i; + struct gve_ring_com *com; + struct gve_tx_ring *tx; + struct gve_rx_ring *rx; if (!gve_get_state_flag(priv, GVE_STATE_FLAG_QPLREG_OK)) return (0); - err = gve_unregister_n_qpls(priv, num_qpls); + for (i = 0; i < priv->tx_cfg.num_queues; i++) { + tx = &priv->tx[i]; + com = &tx->com; + err = gve_adminq_unregister_page_list(priv, com->qpl->id); + if (err != 0) { + device_printf(priv->dev, + "Failed to unregister qpl %d, err: %d\n", + com->qpl->id, err); + } + } + + for (i = 0; i < priv->rx_cfg.num_queues; i++) { + rx = &priv->rx[i]; + com = &rx->com; + err = gve_adminq_unregister_page_list(priv, com->qpl->id); + if (err != 0) { + device_printf(priv->dev, + "Failed to unregister qpl %d, err: %d\n", + com->qpl->id, err); + } + } + if (err != 0) return (err); diff --git a/sys/dev/gve/gve_rx.c b/sys/dev/gve/gve_rx.c --- a/sys/dev/gve/gve_rx.c +++ b/sys/dev/gve/gve_rx.c @@ -36,6 +36,7 @@ gve_rx_free_ring_gqi(struct gve_priv *priv, int i) { struct gve_rx_ring *rx = &priv->rx[i]; + struct gve_ring_com *com = &rx->com; if (rx->page_info != NULL) { free(rx->page_info, M_GVE); @@ -51,6 +52,11 @@ gve_dma_free_coherent(&rx->desc_ring_mem); rx->desc_ring = NULL; } + + if (com->qpl != NULL) { + gve_free_qpl(priv, com->qpl); + com->qpl = NULL; + } } static void @@ -113,10 +119,13 @@ rx->mask = priv->rx_pages_per_qpl - 1; rx->desc_ring = rx->desc_ring_mem.cpu_addr; - com->qpl = &priv->qpls[priv->tx_cfg.max_queues + i]; + com->qpl = gve_alloc_qpl(priv, i + priv->tx_cfg.max_queues, + priv->rx_desc_cnt, /*single_kva=*/false); if (com->qpl == NULL) { - device_printf(priv->dev, "No QPL left for rx ring %d", i); - return (ENOMEM); + device_printf(priv->dev, + "Failed to alloc QPL for rx ring %d", i); + err = ENOMEM; + goto abort; } rx->page_info = malloc(priv->rx_desc_cnt * sizeof(*rx->page_info), diff --git a/sys/dev/gve/gve_rx_dqo.c b/sys/dev/gve/gve_rx_dqo.c --- a/sys/dev/gve/gve_rx_dqo.c +++ b/sys/dev/gve/gve_rx_dqo.c @@ -58,6 +58,7 @@ gve_rx_free_ring_dqo(struct gve_priv *priv, int i) { struct gve_rx_ring *rx = &priv->rx[i]; + struct gve_ring_com *com = &rx->com; int j; if (rx->dqo.compl_ring != NULL) { @@ -86,6 +87,11 @@ if (!gve_is_qpl(priv) && rx->dqo.buf_dmatag) bus_dma_tag_destroy(rx->dqo.buf_dmatag); + + if (com->qpl != NULL) { + gve_free_qpl(priv, com->qpl); + com->qpl = NULL; + } } int @@ -123,10 +129,13 @@ M_GVE, M_WAITOK | M_ZERO); if (gve_is_qpl(priv)) { - rx->com.qpl = &priv->qpls[priv->tx_cfg.max_queues + i]; + rx->com.qpl = gve_alloc_qpl(priv, i + priv->tx_cfg.max_queues, + GVE_RX_NUM_QPL_PAGES_DQO, /*single_kva=*/false); if (rx->com.qpl == NULL) { - device_printf(priv->dev, "No QPL left for rx ring %d", i); - return (ENOMEM); + device_printf(priv->dev, + "Failed to alloc QPL for rx ring %d", i); + err = ENOMEM; + goto abort; } return (0); } diff --git a/sys/dev/gve/gve_tx.c b/sys/dev/gve/gve_tx.c --- a/sys/dev/gve/gve_tx.c +++ b/sys/dev/gve/gve_tx.c @@ -52,6 +52,7 @@ gve_tx_free_ring_gqi(struct gve_priv *priv, int i) { struct gve_tx_ring *tx = &priv->tx[i]; + struct gve_ring_com *com = &tx->com; if (tx->desc_ring != NULL) { gve_dma_free_coherent(&tx->desc_ring_mem); @@ -62,6 +63,11 @@ free(tx->info, M_GVE); tx->info = NULL; } + + if (com->qpl != NULL) { + gve_free_qpl(priv, com->qpl); + com->qpl = NULL; + } } static void @@ -109,9 +115,11 @@ } tx->desc_ring = tx->desc_ring_mem.cpu_addr; - com->qpl = &priv->qpls[i]; + com->qpl = gve_alloc_qpl(priv, i, priv->tx_desc_cnt / GVE_QPL_DIVISOR, + /*single_kva=*/true); if (com->qpl == NULL) { - device_printf(priv->dev, "No QPL left for tx ring %d\n", i); + device_printf(priv->dev, + "Failed to alloc QPL for tx ring %d\n", i); err = ENOMEM; goto abort; } diff --git a/sys/dev/gve/gve_tx_dqo.c b/sys/dev/gve/gve_tx_dqo.c --- a/sys/dev/gve/gve_tx_dqo.c +++ b/sys/dev/gve/gve_tx_dqo.c @@ -75,6 +75,7 @@ gve_tx_free_ring_dqo(struct gve_priv *priv, int i) { struct gve_tx_ring *tx = &priv->tx[i]; + struct gve_ring_com *com = &tx->com; int j; if (tx->dqo.desc_ring != NULL) { @@ -109,6 +110,11 @@ free(tx->dqo.qpl_bufs, M_GVE); tx->dqo.qpl_bufs = NULL; } + + if (com->qpl != NULL) { + gve_free_qpl(priv, com->qpl); + com->qpl = NULL; + } } static int @@ -210,7 +216,15 @@ if (gve_is_qpl(priv)) { int qpl_buf_cnt; - tx->com.qpl = &priv->qpls[i]; + tx->com.qpl = gve_alloc_qpl(priv, i, GVE_TX_NUM_QPL_PAGES_DQO, + /*single_kva*/false); + if (tx->com.qpl == NULL) { + device_printf(priv->dev, + "Failed to alloc QPL for tx ring %d", i); + err = ENOMEM; + goto abort; + } + qpl_buf_cnt = GVE_TX_BUFS_PER_PAGE_DQO * tx->com.qpl->num_pages;