D21935.id63030.diff
No OneTemporary
Actions

Size

15 KB

Referenced Files

None

Subscribers

None

D21935.id63030.diff
View Options

	Index: sys/dev/ena/ena.h
	===================================================================
	--- sys/dev/ena/ena.h
	+++ sys/dev/ena/ena.h
	@@ -252,6 +252,9 @@
	struct mbuf *mbuf;
	bus_dmamap_t map;
	struct ena_com_buf ena_buf;
	+#ifdef DEV_NETMAP
	+ uint32_t netmap_buf_idx;
	+#endif /* DEV_NETMAP */
	} __aligned(CACHE_LINE_SIZE);

	struct ena_stats_tx {
	@@ -351,6 +354,10 @@

	/* Used for LLQ */
	uint8_t *push_buf_intermediate_buf;
	+
	+#ifdef DEV_NETMAP
	+ bool initialized;
	+#endif /* DEV_NETMAP */
	} __aligned(CACHE_LINE_SIZE);

	struct ena_stats_dev {
	@@ -470,5 +477,25 @@
	int ena_restore_device(struct ena_adapter *);
	void ena_destroy_device(struct ena_adapter *, bool);
	int ena_refill_rx_bufs(struct ena_ring *, uint32_t);
	+inline int validate_rx_req_id(struct ena_ring *, uint16_t);
	+
	+inline int
	+validate_rx_req_id(struct ena_ring *rx_ring, uint16_t req_id)
	+{
	+ if (likely(req_id < rx_ring->ring_size))
	+ return (0);
	+
	+ device_printf(rx_ring->adapter->pdev, "Invalid rx req_id: %hu\n",
	+ req_id);
	+ counter_u64_add(rx_ring->rx_stats.bad_req_id, 1);
	+
	+ /* Trigger device reset */
	+ if (likely(!ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, rx_ring->adapter))) {
	+ rx_ring->adapter->reset_reason = ENA_REGS_RESET_INV_RX_REQ_ID;
	+ ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, rx_ring->adapter);
	+ }
	+
	+ return (EFAULT);
	+}

	#endif /* !(ENA_H) */
	Index: sys/dev/ena/ena.c
	===================================================================
	--- sys/dev/ena/ena.c
	+++ sys/dev/ena/ena.c
	@@ -764,6 +764,11 @@

	size = sizeof(struct ena_rx_buffer) * rx_ring->ring_size;

	+#ifdef DEV_NETMAP
	+ ena_netmap_reset_rx_ring(adapter, qid);
	+ rx_ring->initialized = false;
	+#endif /* DEV_NETMAP */
	+
	/*
	* Alloc extra element so in rx path
	* we can always prefetch rx_info + 1
	@@ -1008,8 +1013,12 @@

	req_id = rx_ring->free_rx_ids[next_to_use];
	rx_info = &rx_ring->rx_buffer_info[req_id];
	-
	- rc = ena_alloc_rx_mbuf(adapter, rx_ring, rx_info);
	+#ifdef DEV_NETMAP
	+ if (adapter->ifp->if_capenable & IFCAP_NETMAP)
	+ rc = ena_netmap_alloc_rx_slot(adapter, rx_ring, rx_info);
	+ else
	+#endif /* DEV_NETMAP */
	+ rc = ena_alloc_rx_mbuf(adapter, rx_ring, rx_info);
	if (unlikely(rc != 0)) {
	ena_trace(ENA_WARNING,
	"failed to alloc buffer for rx queue %d\n",
	@@ -1054,6 +1063,14 @@

	if (rx_info->mbuf != NULL)
	ena_free_rx_mbuf(adapter, rx_ring, rx_info);
	+#ifdef DEV_NETMAP
	+ if (((if_getflags(adapter->ifp) & IFF_DYING) == 0) &&
	+ (adapter->ifp->if_capenable & IFCAP_NETMAP)) {
	+ if (rx_info->netmap_buf_idx != 0)
	+ ena_netmap_free_rx_slot(adapter, rx_ring,
	+ rx_info);
	+ }
	+#endif /* DEV_NETMAP */
	}
	}

	@@ -1072,10 +1089,12 @@
	rx_ring = &adapter->rx_ring[i];
	bufs_num = rx_ring->ring_size - 1;
	rc = ena_refill_rx_bufs(rx_ring, bufs_num);
	-
	if (unlikely(rc != bufs_num))
	ena_trace(ENA_WARNING, "refilling Queue %d failed. "
	"Allocated %d buffers from: %d\n", i, rc, bufs_num);
	+#ifdef DEV_NETMAP
	+ rx_ring->initialized = true;
	+#endif /* DEV_NETMAP */
	}
	}

	Index: sys/dev/ena/ena_datapath.c
	===================================================================
	--- sys/dev/ena/ena_datapath.c
	+++ sys/dev/ena/ena_datapath.c
	@@ -32,6 +32,9 @@

	#include "ena.h"
	#include "ena_datapath.h"
	+#ifdef DEV_NETMAP
	+#include "ena_netmap.h"
	+#endif /* DEV_NETMAP */

	/*********************************************************************
	* Static functions prototypes
	@@ -40,7 +43,6 @@
	static int ena_tx_cleanup(struct ena_ring *);
	static int ena_rx_cleanup(struct ena_ring *);
	static inline int validate_tx_req_id(struct ena_ring *, uint16_t);
	-static inline int validate_rx_req_id(struct ena_ring *, uint16_t);
	static void ena_rx_hash_mbuf(struct ena_ring , struct ena_com_rx_ctx ,
	struct mbuf *);
	static struct mbuf* ena_rx_mbuf(struct ena_ring , struct ena_com_rx_buf_info ,
	@@ -206,25 +208,6 @@
	return (EFAULT);
	}

	-static inline int
	-validate_rx_req_id(struct ena_ring *rx_ring, uint16_t req_id)
	-{
	- if (likely(req_id < rx_ring->ring_size))
	- return (0);
	-
	- device_printf(rx_ring->adapter->pdev, "Invalid rx req_id: %hu\n",
	- req_id);
	- counter_u64_add(rx_ring->rx_stats.bad_req_id, 1);
	-
	- /* Trigger device reset */
	- if (likely(!ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, rx_ring->adapter))) {
	- rx_ring->adapter->reset_reason = ENA_REGS_RESET_INV_RX_REQ_ID;
	- ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, rx_ring->adapter);
	- }
	-
	- return (EFAULT);
	-}
	-
	/**
	* ena_tx_cleanup - clear sent packets and corresponding descriptors
	* @tx_ring: ring for which we want to clean packets
	@@ -577,6 +560,9 @@
	unsigned int qid;
	int rc, i;
	int budget = RX_BUDGET;
	+#ifdef DEV_NETMAP
	+ int done;
	+#endif /* DEV_NETMAP */

	adapter = rx_ring->que->adapter;
	ifp = adapter->ifp;
	@@ -586,6 +572,11 @@
	io_sq = &adapter->ena_dev->io_sq_queues[ena_qid];
	next_to_clean = rx_ring->next_to_clean;

	+#ifdef DEV_NETMAP
	+ if (netmap_rx_irq(adapter->ifp, rx_ring->qid, &done) != NM_IRQ_PASS)
	+ return (0);
	+#endif /* DEV_NETMAP */
	+
	ena_trace(ENA_DBG, "rx: qid %d\n", qid);

	do {
	Index: sys/dev/ena/ena_netmap.h
	===================================================================
	--- sys/dev/ena/ena_netmap.h
	+++ sys/dev/ena/ena_netmap.h
	@@ -47,5 +47,10 @@
	#include <dev/netmap/netmap_kern.h>

	int ena_netmap_attach(struct ena_adapter *);
	+int ena_netmap_alloc_rx_slot(struct ena_adapter , struct ena_ring ,
	+ struct ena_rx_buffer *);
	+void ena_netmap_free_rx_slot(struct ena_adapter , struct ena_ring ,
	+ struct ena_rx_buffer *);
	+void ena_netmap_reset_rx_ring(struct ena_adapter *, int);

	#endif /* _ENA_NETMAP_H_ */
	Index: sys/dev/ena/ena_netmap.c
	===================================================================
	--- sys/dev/ena/ena_netmap.c
	+++ sys/dev/ena/ena_netmap.c
	@@ -35,10 +35,37 @@
	#include "ena.h"
	#include "ena_netmap.h"

	+#define ENA_NETMAP_MORE_FRAMES 1
	+#define ENA_NETMAP_NO_MORE_FRAMES 0
	+#define ENA_MAX_FRAMES 16384
	+
	+struct ena_netmap_ctx {
	+ struct netmap_kring *kring;
	+ struct ena_adapter *adapter;
	+ struct netmap_adapter *na;
	+ struct netmap_slot *slots;
	+ struct ena_ring *ring;
	+ struct ena_com_io_cq *io_cq;
	+ struct ena_com_io_sq *io_sq;
	+ u_int nm_i;
	+ uint16_t nt;
	+ uint16_t lim;
	+};
	+
	+/* Netmap callbacks */
	static int ena_netmap_reg(struct netmap_adapter *, int);
	static int ena_netmap_txsync(struct netmap_kring *, int);
	static int ena_netmap_rxsync(struct netmap_kring *, int);

	+/* Helper functions */
	+static int ena_netmap_rx_frames(struct ena_netmap_ctx *);
	+static int ena_netmap_rx_frame(struct ena_netmap_ctx *);
	+static int ena_netmap_rx_load_desc(struct ena_netmap_ctx *, uint16_t,
	+ int *);
	+static void ena_netmap_rx_cleanup(struct ena_netmap_ctx *);
	+static void ena_netmap_fill_ctx(struct netmap_kring *,
	+ struct ena_netmap_ctx *, uint16_t);
	+
	int
	ena_netmap_attach(struct ena_adapter *adapter)
	{
	@@ -61,6 +88,124 @@
	return (netmap_attach(&na));
	}

	+int
	+ena_netmap_alloc_rx_slot(struct ena_adapter *adapter,
	+ struct ena_ring rx_ring, struct ena_rx_buffer rx_info)
	+{
	+ struct netmap_adapter *na = NA(adapter->ifp);
	+ struct netmap_kring *kring;
	+ struct netmap_ring *ring;
	+ struct netmap_slot *slot;
	+ void *addr;
	+ uint64_t paddr;
	+ int nm_i, qid, head, lim, rc;
	+
	+ /* if previously allocated frag is not used */
	+ if (unlikely(rx_info->netmap_buf_idx != 0))
	+ return (0);
	+
	+ qid = rx_ring->qid;
	+ kring = na->rx_rings[qid];
	+ nm_i = kring->nr_hwcur;
	+ head = kring->rhead;
	+
	+ ena_trace(ENA_NETMAP \| ENA_DBG, "nr_hwcur: %d, nr_hwtail: %d, "
	+ "rhead: %d, rcur: %d, rtail: %d\n", kring->nr_hwcur,
	+ kring->nr_hwtail, kring->rhead, kring->rcur, kring->rtail);
	+
	+ if ((nm_i == head) && rx_ring->initialized) {
	+ ena_trace(ENA_NETMAP, "No free slots in netmap ring\n");
	+ return (ENOMEM);
	+ }
	+
	+ ring = kring->ring;
	+ if (ring == NULL) {
	+ device_printf(adapter->pdev, "Rx ring %d is NULL\n", qid);
	+ return (EFAULT);
	+ }
	+ slot = &ring->slot[nm_i];
	+
	+ addr = PNMB(na, slot, &paddr);
	+ if (addr == NETMAP_BUF_BASE(na)) {
	+ device_printf(adapter->pdev, "Bad buff in slot\n");
	+ return (EFAULT);
	+ }
	+
	+ rc = netmap_load_map(na, adapter->rx_buf_tag, rx_info->map, addr);
	+ if (rc != 0) {
	+ ena_trace(ENA_WARNING, "DMA mapping error\n");
	+ return (rc);
	+ }
	+ bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map, BUS_DMASYNC_PREREAD);
	+
	+ rx_info->ena_buf.paddr = paddr;
	+ rx_info->ena_buf.len = ring->nr_buf_size;
	+ rx_info->mbuf = NULL;
	+ rx_info->netmap_buf_idx = slot->buf_idx;
	+
	+ slot->buf_idx = 0;
	+
	+ lim = kring->nkr_num_slots - 1;
	+ kring->nr_hwcur = nm_next(nm_i, lim);
	+
	+ return (0);
	+}
	+
	+void
	+ena_netmap_free_rx_slot(struct ena_adapter *adapter,
	+ struct ena_ring rx_ring, struct ena_rx_buffer rx_info)
	+{
	+ struct netmap_adapter *na;
	+ struct netmap_kring *kring;
	+ struct netmap_slot *slot;
	+ int nm_i, qid, lim;
	+
	+ na = NA(adapter->ifp);
	+ if (na == NULL) {
	+ device_printf(adapter->pdev, "netmap adapter is NULL\n");
	+ return;
	+ }
	+
	+ if (na->rx_rings == NULL) {
	+ device_printf(adapter->pdev, "netmap rings are NULL\n");
	+ return;
	+ }
	+
	+ qid = rx_ring->qid;
	+ kring = na->rx_rings[qid];
	+ if (kring == NULL) {
	+ device_printf(adapter->pdev,
	+ "netmap kernel ring %d is NULL\n", qid);
	+ return;
	+ }
	+
	+ lim = kring->nkr_num_slots - 1;
	+ nm_i = nm_prev(kring->nr_hwcur, lim);
	+
	+ if (kring->nr_mode != NKR_NETMAP_ON)
	+ return;
	+
	+ bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map,
	+ BUS_DMASYNC_POSTREAD);
	+ netmap_unload_map(na, adapter->rx_buf_tag, rx_info->map);
	+
	+ slot = &kring->ring->slot[nm_i];
	+
	+ ENA_ASSERT(slot->buf_idx == 0, "Overwrite slot buf\n");
	+ slot->buf_idx = rx_info->netmap_buf_idx;
	+ slot->flags = NS_BUF_CHANGED;
	+
	+ rx_info->netmap_buf_idx = 0;
	+ kring->nr_hwcur = nm_i;
	+}
	+
	+void
	+ena_netmap_reset_rx_ring(struct ena_adapter *adapter, int qid)
	+{
	+ if (adapter->ifp->if_capenable & IFCAP_NETMAP)
	+ netmap_reset(NA(adapter->ifp), NR_RX, qid, 0);
	+}
	+
	static int
	ena_netmap_reg(struct netmap_adapter *na, int onoff)
	{
	@@ -104,8 +249,192 @@
	static int
	ena_netmap_rxsync(struct netmap_kring *kring, int flags)
	{
	- ena_trace(ENA_NETMAP, "netmap rxsync\n");
	+ struct ena_netmap_ctx ctx;
	+ int rc;
	+
	+ ena_netmap_fill_ctx(kring, &ctx, ENA_IO_RXQ_IDX(kring->ring_id));
	+ ctx.ring = &ctx.adapter->rx_ring[kring->ring_id];
	+
	+ if (ctx.kring->rhead > ctx.lim) {
	+ /* Probably not needed to release slots from RX ring. */
	+ return (netmap_ring_reinit(ctx.kring));
	+ }
	+
	+ if (unlikely((if_getdrvflags(ctx.na->ifp) & IFF_DRV_RUNNING) == 0))
	+ return (0);
	+
	+ if (unlikely(!ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, ctx.adapter)))
	+ return (0);
	+
	+ if ((rc = ena_netmap_rx_frames(&ctx)) != 0)
	+ return (rc);
	+
	+ ena_netmap_rx_cleanup(&ctx);
	+
	+ return (0);
	+}
	+
	+static inline int
	+ena_netmap_rx_frames(struct ena_netmap_ctx *ctx)
	+{
	+ int rc = 0;
	+ int frames_counter = 0;
	+
	+ ctx->nt = ctx->ring->next_to_clean;
	+ ctx->nm_i = ctx->kring->nr_hwtail;
	+
	+ while((rc = ena_netmap_rx_frame(ctx)) == ENA_NETMAP_MORE_FRAMES) {
	+ frames_counter++;
	+ /* In case of multiple frames, it is not an error. */
	+ rc = 0;
	+ if (frames_counter > ENA_MAX_FRAMES) {
	+ device_printf(ctx->adapter->pdev,
	+ "Driver is stuck in the Rx loop\n");
	+ break;
	+ }
	+ };
	+
	+ ctx->kring->nr_hwtail = ctx->nm_i;
	+ ctx->kring->nr_kflags &= ~NKR_PENDINTR;
	+ ctx->ring->next_to_clean = ctx->nt;
	+
	+ return (rc);
	+}
	+
	+static inline int
	+ena_netmap_rx_frame(struct ena_netmap_ctx *ctx)
	+{
	+ struct ena_com_rx_ctx ena_rx_ctx;
	+ int rc, len = 0;
	+ uint16_t buf, nm;
	+
	+ ena_rx_ctx.ena_bufs = ctx->ring->ena_bufs;
	+ ena_rx_ctx.max_bufs = ctx->adapter->max_rx_sgl_size;
	+ bus_dmamap_sync(ctx->io_cq->cdesc_addr.mem_handle.tag,
	+ ctx->io_cq->cdesc_addr.mem_handle.map, BUS_DMASYNC_POSTREAD);
	+
	+ rc = ena_com_rx_pkt(ctx->io_cq, ctx->io_sq, &ena_rx_ctx);
	+ if (unlikely(rc != 0)) {
	+ ena_trace(ENA_ALERT, "Too many desc from the device.\n");
	+ counter_u64_add(ctx->ring->rx_stats.bad_desc_num, 1);
	+ ctx->adapter->reset_reason = ENA_REGS_RESET_TOO_MANY_RX_DESCS;
	+ ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, ctx->adapter);
	+ return (rc);
	+ }
	+ if (unlikely(ena_rx_ctx.descs == 0))
	+ return (ENA_NETMAP_NO_MORE_FRAMES);
	+
	+ ena_trace(ENA_NETMAP \| ENA_DBG, "Rx: q %d got packet from ena. descs #:"
	+ " %d l3 proto %d l4 proto %d hash: %x\n", ctx->ring->qid,
	+ ena_rx_ctx.descs, ena_rx_ctx.l3_proto, ena_rx_ctx.l4_proto,
	+ ena_rx_ctx.hash);
	+
	+ for (buf = 0; buf < ena_rx_ctx.descs; buf++)
	+ if ((rc = ena_netmap_rx_load_desc(ctx, buf, &len)) != 0)
	+ break;
	+ /*
	+ * ena_netmap_rx_load_desc doesn't know the number of descriptors.
	+ * It just set flag NS_MOREFRAG to all slots, then here flag of
	+ * last slot is cleared.
	+ */
	+ ctx->slots[nm_prev(ctx->nm_i, ctx->lim)].flags = NS_BUF_CHANGED;
	+
	+ if (rc != 0) {
	+ goto rx_clear_desc;
	+ }
	+
	+ bus_dmamap_sync(ctx->io_cq->cdesc_addr.mem_handle.tag,
	+ ctx->io_cq->cdesc_addr.mem_handle.map, BUS_DMASYNC_PREREAD);
	+
	+ counter_enter();
	+ counter_u64_add_protected(ctx->ring->rx_stats.bytes, len);
	+ counter_u64_add_protected(ctx->adapter->hw_stats.rx_bytes, len);
	+ counter_u64_add_protected(ctx->ring->rx_stats.cnt, 1);
	+ counter_u64_add_protected(ctx->adapter->hw_stats.rx_packets, 1);
	+ counter_exit();
	+
	+ return (ENA_NETMAP_MORE_FRAMES);
	+
	+rx_clear_desc:
	+ nm = ctx->nm_i;
	+
	+ /* Remove failed packet from ring */
	+ while(buf--) {
	+ ctx->slots[nm].flags = 0;
	+ ctx->slots[nm].len = 0;
	+ nm = nm_prev(nm, ctx->lim);
	+ }
	+
	+ return (rc);
	+}
	+
	+static inline int
	+ena_netmap_rx_load_desc(struct ena_netmap_ctx ctx, uint16_t buf, int len)
	+{
	+ struct ena_rx_buffer *rx_info;
	+ uint16_t req_id;
	+ int rc;
	+
	+ req_id = ctx->ring->ena_bufs[buf].req_id;
	+ rc = validate_rx_req_id(ctx->ring, req_id);
	+ if (unlikely(rc != 0))
	+ return (rc);
	+
	+ rx_info = &ctx->ring->rx_buffer_info[req_id];
	+ bus_dmamap_sync(ctx->adapter->rx_buf_tag, rx_info->map,
	+ BUS_DMASYNC_POSTREAD);
	+ netmap_unload_map(ctx->na, ctx->adapter->rx_buf_tag, rx_info->map);
	+
	+ ENA_ASSERT(ctx->slots[ctx->nm_i].buf_idx == 0, "Rx idx is not 0.\n");
	+
	+ ctx->slots[ctx->nm_i].buf_idx = rx_info->netmap_buf_idx;
	+ rx_info->netmap_buf_idx = 0;
	+ /*
	+ * Set NS_MOREFRAG to all slots.
	+ * Then ena_netmap_rx_frame clears it from last one.
	+ */
	+ ctx->slots[ctx->nm_i].flags \|= NS_MOREFRAG \| NS_BUF_CHANGED;
	+ ctx->slots[ctx->nm_i].len = ctx->ring->ena_bufs[buf].len;
	+ *len += ctx->slots[ctx->nm_i].len;
	+ ctx->ring->free_rx_ids[ctx->nt] = req_id;
	+ ena_trace(ENA_DBG, "rx_info %p, buf_idx %d, paddr %jx, nm: %d\n",
	+ rx_info, ctx->slots[ctx->nm_i].buf_idx,
	+ (uintmax_t)rx_info->ena_buf.paddr, ctx->nm_i);
	+
	+ ctx->nm_i = nm_next(ctx->nm_i, ctx->lim);
	+ ctx->nt = ENA_RX_RING_IDX_NEXT(ctx->nt, ctx->ring->ring_size);
	+
	return (0);
	}

	+static inline void
	+ena_netmap_rx_cleanup(struct ena_netmap_ctx *ctx)
	+{
	+ int refill_required;
	+
	+ refill_required = ctx->kring->rhead - ctx->kring->nr_hwcur;
	+ if (ctx->kring->nr_hwcur != ctx->kring->nr_hwtail)
	+ refill_required -= 1;
	+
	+ if (refill_required == 0)
	+ return;
	+ else if (refill_required < 0)
	+ refill_required += ctx->kring->nkr_num_slots;
	+
	+ ena_refill_rx_bufs(ctx->ring, refill_required);
	+}
	+
	+static inline void
	+ena_netmap_fill_ctx(struct netmap_kring kring, struct ena_netmap_ctx ctx,
	+ uint16_t ena_qid)
	+{
	+ ctx->kring = kring;
	+ ctx->na = kring->na;
	+ ctx->adapter = ctx->na->ifp->if_softc;
	+ ctx->lim = kring->nkr_num_slots - 1;
	+ ctx->io_cq = &ctx->adapter->ena_dev->io_cq_queues[ena_qid];
	+ ctx->io_sq = &ctx->adapter->ena_dev->io_sq_queues[ena_qid];
	+ ctx->slots = kring->ring->slot;
	+}
	+
	#endif /* DEV_NETMAP */

File Metadata

Mime Type: text/plain
Expires: Sat, May 23, 7:23 AM (12 h, 40 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 33443162
Default Alt Text: D21935.id63030.diff (15 KB)

D21935.id63030.diffNo OneTemporaryActions

D21935.id63030.diffView Options

File Metadata

Event Timeline

D21935.id63030.diff
No OneTemporary
Actions

D21935.id63030.diff
View Options