Differential D46690 Diff 144949 sys/dev/gve/gve_tx.c

Changeset View

Standalone View

sys/dev/gve/gve_tx.c

/*-		/*-
* SPDX-License-Identifier: BSD-3-Clause		* SPDX-License-Identifier: BSD-3-Clause
*		*
* Copyright (c) 2023 Google LLC		* Copyright (c) 2023-2024 Google LLC
*		*
* Redistribution and use in source and binary forms, with or without modification,		* Redistribution and use in source and binary forms, with or without modification,
* are permitted provided that the following conditions are met:		* are permitted provided that the following conditions are met:
*		*
* 1. Redistributions of source code must retain the above copyright notice, this		* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.		* list of conditions and the following disclaimer.
*		*
* 2. Redistributions in binary form must reproduce the above copyright notice,		* 2. Redistributions in binary form must reproduce the above copyright notice,
Show All 12 Lines
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;		* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON		* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT		* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS		* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.		* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/		*/
#include "gve.h"		#include "gve.h"
#include "gve_adminq.h"		#include "gve_adminq.h"
		#include "gve_dqo.h"

#define GVE_GQ_TX_MIN_PKT_DESC_BYTES 182		#define GVE_GQ_TX_MIN_PKT_DESC_BYTES 182

static int		static int
gve_tx_fifo_init(struct gve_priv priv, struct gve_tx_ring tx)		gve_tx_fifo_init(struct gve_priv priv, struct gve_tx_ring tx)
{		{
struct gve_queue_page_list *qpl = tx->com.qpl;		struct gve_queue_page_list *qpl = tx->com.qpl;
struct gve_tx_fifo *fifo = &tx->fifo;		struct gve_tx_fifo *fifo = &tx->fifo;

fifo->size = qpl->num_pages * PAGE_SIZE;		fifo->size = qpl->num_pages * PAGE_SIZE;
fifo->base = qpl->kva;		fifo->base = qpl->kva;
atomic_store_int(&fifo->available, fifo->size);		atomic_store_int(&fifo->available, fifo->size);
fifo->head = 0;		fifo->head = 0;

return (0);		return (0);
}		}

static void		static void
gve_tx_free_ring(struct gve_priv *priv, int i)		gve_tx_free_ring_gqi(struct gve_priv *priv, int i)
{		{
struct gve_tx_ring *tx = &priv->tx[i];		struct gve_tx_ring *tx = &priv->tx[i];
struct gve_ring_com *com = &tx->com;

/* Safe to call even if never alloced */		if (tx->desc_ring != NULL) {
gve_free_counters((counter_u64_t *)&tx->stats, NUM_TX_STATS);		gve_dma_free_coherent(&tx->desc_ring_mem);
		tx->desc_ring = NULL;
if (tx->br != NULL) {
buf_ring_free(tx->br, M_DEVBUF);
tx->br = NULL;
}		}

if (mtx_initialized(&tx->ring_mtx))
mtx_destroy(&tx->ring_mtx);

if (tx->info != NULL) {		if (tx->info != NULL) {
free(tx->info, M_GVE);		free(tx->info, M_GVE);
tx->info = NULL;		tx->info = NULL;
}		}

if (tx->desc_ring != NULL) {
gve_dma_free_coherent(&tx->desc_ring_mem);
tx->desc_ring = NULL;
}		}

		static void
		gve_tx_free_ring(struct gve_priv *priv, int i)
		{
		struct gve_tx_ring *tx = &priv->tx[i];
		struct gve_ring_com *com = &tx->com;

		/* Safe to call even if never alloced */
		gve_free_counters((counter_u64_t *)&tx->stats, NUM_TX_STATS);

		if (mtx_initialized(&tx->ring_mtx))
		mtx_destroy(&tx->ring_mtx);

if (com->q_resources != NULL) {		if (com->q_resources != NULL) {
gve_dma_free_coherent(&com->q_resources_mem);		gve_dma_free_coherent(&com->q_resources_mem);
com->q_resources = NULL;		com->q_resources = NULL;
}		}

		if (tx->br != NULL) {
		buf_ring_free(tx->br, M_DEVBUF);
		tx->br = NULL;
}		}

		if (gve_is_gqi(priv))
		gve_tx_free_ring_gqi(priv, i);
		else
		gve_tx_free_ring_dqo(priv, i);
		}

static int		static int
gve_tx_alloc_ring(struct gve_priv *priv, int i)		gve_tx_alloc_ring_gqi(struct gve_priv *priv, int i)
{		{
struct gve_tx_ring *tx = &priv->tx[i];		struct gve_tx_ring *tx = &priv->tx[i];
struct gve_ring_com *com = &tx->com;		struct gve_ring_com *com = &tx->com;
char mtx_name[16];
int err;		int err;

com->priv = priv;		err = gve_dma_alloc_coherent(priv,
com->id = i;		sizeof(union gve_tx_desc) * priv->tx_desc_cnt,
		CACHE_LINE_SIZE, &tx->desc_ring_mem);
		if (err != 0) {
		device_printf(priv->dev,
		"Failed to alloc desc ring for tx ring %d", i);
		goto abort;
		}
		tx->desc_ring = tx->desc_ring_mem.cpu_addr;

com->qpl = &priv->qpls[i];		com->qpl = &priv->qpls[i];
if (com->qpl == NULL) {		if (com->qpl == NULL) {
device_printf(priv->dev, "No QPL left for tx ring %d\n", i);		device_printf(priv->dev, "No QPL left for tx ring %d\n", i);
return (ENOMEM);		err = ENOMEM;
		goto abort;
}		}

err = gve_tx_fifo_init(priv, tx);		err = gve_tx_fifo_init(priv, tx);
if (err != 0)		if (err != 0)
goto abort;		goto abort;

tx->info = malloc(sizeof(struct gve_tx_buffer_state) * priv->tx_desc_cnt,		tx->info = malloc(
		sizeof(struct gve_tx_buffer_state) * priv->tx_desc_cnt,
M_GVE, M_WAITOK \| M_ZERO);		M_GVE, M_WAITOK \| M_ZERO);
		return (0);

		abort:
		gve_tx_free_ring_gqi(priv, i);
		return (err);
		}

		static int
		gve_tx_alloc_ring(struct gve_priv *priv, int i)
		{
		struct gve_tx_ring *tx = &priv->tx[i];
		struct gve_ring_com *com = &tx->com;
		char mtx_name[16];
		int err;

		com->priv = priv;
		com->id = i;

		if (gve_is_gqi(priv))
		err = gve_tx_alloc_ring_gqi(priv, i);
		else
		err = gve_tx_alloc_ring_dqo(priv, i);
		if (err != 0)
		goto abort;

sprintf(mtx_name, "gvetx%d", i);		sprintf(mtx_name, "gvetx%d", i);
mtx_init(&tx->ring_mtx, mtx_name, NULL, MTX_DEF);		mtx_init(&tx->ring_mtx, mtx_name, NULL, MTX_DEF);

tx->br = buf_ring_alloc(GVE_TX_BUFRING_ENTRIES, M_DEVBUF,		tx->br = buf_ring_alloc(GVE_TX_BUFRING_ENTRIES, M_DEVBUF,
M_WAITOK, &tx->ring_mtx);		M_WAITOK, &tx->ring_mtx);

gve_alloc_counters((counter_u64_t *)&tx->stats, NUM_TX_STATS);		gve_alloc_counters((counter_u64_t *)&tx->stats, NUM_TX_STATS);

err = gve_dma_alloc_coherent(priv, sizeof(struct gve_queue_resources),		err = gve_dma_alloc_coherent(priv, sizeof(struct gve_queue_resources),
PAGE_SIZE, &com->q_resources_mem);		PAGE_SIZE, &com->q_resources_mem);
if (err != 0) {		if (err != 0) {
device_printf(priv->dev, "Failed to alloc queue resources for tx ring %d", i);		device_printf(priv->dev,
		"Failed to alloc queue resources for tx ring %d", i);
goto abort;		goto abort;
}		}
com->q_resources = com->q_resources_mem.cpu_addr;		com->q_resources = com->q_resources_mem.cpu_addr;

err = gve_dma_alloc_coherent(priv,
sizeof(union gve_tx_desc) * priv->tx_desc_cnt,
CACHE_LINE_SIZE, &tx->desc_ring_mem);
if (err != 0) {
device_printf(priv->dev, "Failed to alloc desc ring for tx ring %d", i);
goto abort;
}
tx->desc_ring = tx->desc_ring_mem.cpu_addr;

return (0);		return (0);

abort:		abort:
gve_tx_free_ring(priv, i);		gve_tx_free_ring(priv, i);
return (err);		return (err);
}		}

int		int
gve_alloc_tx_rings(struct gve_priv *priv)		gve_alloc_tx_rings(struct gve_priv *priv)
{		{
int err = 0;		int err = 0;
int i;		int i;

priv->tx = malloc(sizeof(struct gve_tx_ring) * priv->tx_cfg.num_queues,		priv->tx = malloc(sizeof(struct gve_tx_ring) * priv->tx_cfg.num_queues,
M_GVE, M_WAITOK \| M_ZERO);		M_GVE, M_WAITOK \| M_ZERO);

for (i = 0; i < priv->tx_cfg.num_queues; i++) {		for (i = 0; i < priv->tx_cfg.num_queues; i++) {
err = gve_tx_alloc_ring(priv, i);		err = gve_tx_alloc_ring(priv, i);

		markjUnsubmitted Done Inline Actions Spurious whitespace change. markj: Spurious whitespace change.
if (err != 0)		if (err != 0)
goto free_rings;		goto free_rings;

}		}

return (0);		return (0);

free_rings:		free_rings:
▲ Show 20 Lines • Show All 41 Lines • ▼ Show 20 Lines	gve_clear_tx_ring(struct gve_priv *priv, int i)

atomic_store_int(&fifo->available, fifo->size);		atomic_store_int(&fifo->available, fifo->size);
fifo->head = 0;		fifo->head = 0;

gve_tx_clear_desc_ring(tx);		gve_tx_clear_desc_ring(tx);
}		}

static void		static void
gve_start_tx_ring(struct gve_priv *priv, int i)		gve_start_tx_ring(struct gve_priv *priv, int i,
		void (cleanup) (void *arg, int pending))
{		{
struct gve_tx_ring *tx = &priv->tx[i];		struct gve_tx_ring *tx = &priv->tx[i];
struct gve_ring_com *com = &tx->com;		struct gve_ring_com *com = &tx->com;

NET_TASK_INIT(&com->cleanup_task, 0, gve_tx_cleanup_tq, tx);		NET_TASK_INIT(&com->cleanup_task, 0, cleanup, tx);
com->cleanup_tq = taskqueue_create_fast("gve tx", M_WAITOK,		com->cleanup_tq = taskqueue_create_fast("gve tx", M_WAITOK,
taskqueue_thread_enqueue, &com->cleanup_tq);		taskqueue_thread_enqueue, &com->cleanup_tq);
taskqueue_start_threads(&com->cleanup_tq, 1, PI_NET, "%s txq %d",		taskqueue_start_threads(&com->cleanup_tq, 1, PI_NET, "%s txq %d",
device_get_nameunit(priv->dev), i);		device_get_nameunit(priv->dev), i);

TASK_INIT(&tx->xmit_task, 0, gve_xmit_tq, tx);		TASK_INIT(&tx->xmit_task, 0, gve_xmit_tq, tx);
tx->xmit_tq = taskqueue_create_fast("gve tx xmit",		tx->xmit_tq = taskqueue_create_fast("gve tx xmit",
M_WAITOK, taskqueue_thread_enqueue, &tx->xmit_tq);		M_WAITOK, taskqueue_thread_enqueue, &tx->xmit_tq);
taskqueue_start_threads(&tx->xmit_tq, 1, PI_NET, "%s txq %d xmit",		taskqueue_start_threads(&tx->xmit_tq, 1, PI_NET, "%s txq %d xmit",
device_get_nameunit(priv->dev), i);		device_get_nameunit(priv->dev), i);
}		}

int		int
gve_create_tx_rings(struct gve_priv *priv)		gve_create_tx_rings(struct gve_priv *priv)
{		{
struct gve_ring_com *com;		struct gve_ring_com *com;
struct gve_tx_ring *tx;		struct gve_tx_ring *tx;
int err;		int err;
int i;		int i;

if (gve_get_state_flag(priv, GVE_STATE_FLAG_TX_RINGS_OK))		if (gve_get_state_flag(priv, GVE_STATE_FLAG_TX_RINGS_OK))
return (0);		return (0);

for (i = 0; i < priv->tx_cfg.num_queues; i++)		for (i = 0; i < priv->tx_cfg.num_queues; i++) {
		if (gve_is_gqi(priv))
gve_clear_tx_ring(priv, i);		gve_clear_tx_ring(priv, i);
		else
		gve_clear_tx_ring_dqo(priv, i);
		}

err = gve_adminq_create_tx_queues(priv, priv->tx_cfg.num_queues);		err = gve_adminq_create_tx_queues(priv, priv->tx_cfg.num_queues);
if (err != 0)		if (err != 0)
return (err);		return (err);

bus_dmamap_sync(priv->irqs_db_mem.tag, priv->irqs_db_mem.map,		bus_dmamap_sync(priv->irqs_db_mem.tag, priv->irqs_db_mem.map,
BUS_DMASYNC_POSTREAD);		BUS_DMASYNC_POSTREAD);

for (i = 0; i < priv->tx_cfg.num_queues; i++) {		for (i = 0; i < priv->tx_cfg.num_queues; i++) {
tx = &priv->tx[i];		tx = &priv->tx[i];
com = &tx->com;		com = &tx->com;

com->irq_db_offset = 4 * be32toh(priv->irq_db_indices[com->ntfy_id].index);		com->irq_db_offset = 4 * be32toh(priv->irq_db_indices[com->ntfy_id].index);

bus_dmamap_sync(com->q_resources_mem.tag, com->q_resources_mem.map,		bus_dmamap_sync(com->q_resources_mem.tag, com->q_resources_mem.map,
BUS_DMASYNC_POSTREAD);		BUS_DMASYNC_POSTREAD);
com->db_offset = 4 * be32toh(com->q_resources->db_index);		com->db_offset = 4 * be32toh(com->q_resources->db_index);
com->counter_idx = be32toh(com->q_resources->counter_index);		com->counter_idx = be32toh(com->q_resources->counter_index);

gve_start_tx_ring(priv, i);		if (gve_is_gqi(priv))
		gve_start_tx_ring(priv, i, gve_tx_cleanup_tq);
		else
		gve_start_tx_ring(priv, i, gve_tx_cleanup_tq_dqo);
}		}

gve_set_state_flag(priv, GVE_STATE_FLAG_TX_RINGS_OK);		gve_set_state_flag(priv, GVE_STATE_FLAG_TX_RINGS_OK);
return (0);		return (0);
}		}

static void		static void
gve_stop_tx_ring(struct gve_priv *priv, int i)		gve_stop_tx_ring(struct gve_priv *priv, int i)
▲ Show 20 Lines • Show All 424 Lines • ▼ Show 20 Lines
}		}

static void		static void
gve_xmit_br(struct gve_tx_ring *tx)		gve_xmit_br(struct gve_tx_ring *tx)
{		{
struct gve_priv *priv = tx->com.priv;		struct gve_priv *priv = tx->com.priv;
struct ifnet *ifp = priv->ifp;		struct ifnet *ifp = priv->ifp;
struct mbuf *mbuf;		struct mbuf *mbuf;
		int err;

while ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0 &&		while ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0 &&
(mbuf = drbr_peek(ifp, tx->br)) != NULL) {		(mbuf = drbr_peek(ifp, tx->br)) != NULL) {

if (__predict_false(gve_xmit(tx, mbuf) != 0)) {		if (gve_is_gqi(priv))
		err = gve_xmit(tx, mbuf);
		else {
		/*
		* gve_xmit_dqo might attempt to defrag the mbuf chain.
		* The reference is passed in so that in the case of
		* errors, the new mbuf chain is what's put back on the br.
		*/
		err = gve_xmit_dqo(tx, &mbuf);
		}

		if (__predict_false(err != 0 && mbuf != NULL)) {
drbr_putback(ifp, tx->br, mbuf);		drbr_putback(ifp, tx->br, mbuf);
taskqueue_enqueue(tx->xmit_tq, &tx->xmit_task);		taskqueue_enqueue(tx->xmit_tq, &tx->xmit_task);
break;		break;
}		}

drbr_advance(ifp, tx->br);		drbr_advance(ifp, tx->br);
BPF_MTAP(ifp, mbuf);		BPF_MTAP(ifp, mbuf);

bus_dmamap_sync(tx->desc_ring_mem.tag, tx->desc_ring_mem.map,		bus_dmamap_sync(tx->desc_ring_mem.tag, tx->desc_ring_mem.map,
BUS_DMASYNC_PREWRITE);		BUS_DMASYNC_PREWRITE);

		if (gve_is_gqi(priv))
gve_db_bar_write_4(priv, tx->com.db_offset, tx->req);		gve_db_bar_write_4(priv, tx->com.db_offset, tx->req);
		else
		gve_db_bar_dqo_write_4(priv, tx->com.db_offset,
		tx->dqo.desc_tail);
}		}
}		}

void		void
gve_xmit_tq(void *arg, int pending)		gve_xmit_tq(void *arg, int pending)
{		{
struct gve_tx_ring tx = (struct gve_tx_ring )arg;		struct gve_tx_ring tx = (struct gve_tx_ring )arg;

▲ Show 20 Lines • Show All 84 Lines • Show Last 20 Lines