Index: head/sys/dev/ena/ena.h =================================================================== --- head/sys/dev/ena/ena.h +++ head/sys/dev/ena/ena.h @@ -252,6 +252,9 @@ struct mbuf *mbuf; bus_dmamap_t map; struct ena_com_buf ena_buf; +#ifdef DEV_NETMAP + uint32_t netmap_buf_idx; +#endif /* DEV_NETMAP */ } __aligned(CACHE_LINE_SIZE); struct ena_stats_tx { @@ -351,6 +354,10 @@ /* Used for LLQ */ uint8_t *push_buf_intermediate_buf; + +#ifdef DEV_NETMAP + bool initialized; +#endif /* DEV_NETMAP */ } __aligned(CACHE_LINE_SIZE); struct ena_stats_dev { @@ -470,5 +477,25 @@ int ena_restore_device(struct ena_adapter *); void ena_destroy_device(struct ena_adapter *, bool); int ena_refill_rx_bufs(struct ena_ring *, uint32_t); +inline int validate_rx_req_id(struct ena_ring *, uint16_t); + +inline int +validate_rx_req_id(struct ena_ring *rx_ring, uint16_t req_id) +{ + if (likely(req_id < rx_ring->ring_size)) + return (0); + + device_printf(rx_ring->adapter->pdev, "Invalid rx req_id: %hu\n", + req_id); + counter_u64_add(rx_ring->rx_stats.bad_req_id, 1); + + /* Trigger device reset */ + if (likely(!ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, rx_ring->adapter))) { + rx_ring->adapter->reset_reason = ENA_REGS_RESET_INV_RX_REQ_ID; + ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, rx_ring->adapter); + } + + return (EFAULT); +} #endif /* !(ENA_H) */ Index: head/sys/dev/ena/ena.c =================================================================== --- head/sys/dev/ena/ena.c +++ head/sys/dev/ena/ena.c @@ -764,6 +764,11 @@ size = sizeof(struct ena_rx_buffer) * rx_ring->ring_size; +#ifdef DEV_NETMAP + ena_netmap_reset_rx_ring(adapter, qid); + rx_ring->initialized = false; +#endif /* DEV_NETMAP */ + /* * Alloc extra element so in rx path * we can always prefetch rx_info + 1 @@ -1008,8 +1013,12 @@ req_id = rx_ring->free_rx_ids[next_to_use]; rx_info = &rx_ring->rx_buffer_info[req_id]; - - rc = ena_alloc_rx_mbuf(adapter, rx_ring, rx_info); +#ifdef DEV_NETMAP + if (adapter->ifp->if_capenable & IFCAP_NETMAP) + rc = ena_netmap_alloc_rx_slot(adapter, rx_ring, rx_info); + else +#endif /* DEV_NETMAP */ + rc = ena_alloc_rx_mbuf(adapter, rx_ring, rx_info); if (unlikely(rc != 0)) { ena_trace(ENA_WARNING, "failed to alloc buffer for rx queue %d\n", @@ -1054,6 +1063,14 @@ if (rx_info->mbuf != NULL) ena_free_rx_mbuf(adapter, rx_ring, rx_info); +#ifdef DEV_NETMAP + if (((if_getflags(adapter->ifp) & IFF_DYING) == 0) && + (adapter->ifp->if_capenable & IFCAP_NETMAP)) { + if (rx_info->netmap_buf_idx != 0) + ena_netmap_free_rx_slot(adapter, rx_ring, + rx_info); + } +#endif /* DEV_NETMAP */ } } @@ -1072,10 +1089,12 @@ rx_ring = &adapter->rx_ring[i]; bufs_num = rx_ring->ring_size - 1; rc = ena_refill_rx_bufs(rx_ring, bufs_num); - if (unlikely(rc != bufs_num)) ena_trace(ENA_WARNING, "refilling Queue %d failed. " "Allocated %d buffers from: %d\n", i, rc, bufs_num); +#ifdef DEV_NETMAP + rx_ring->initialized = true; +#endif /* DEV_NETMAP */ } } Index: head/sys/dev/ena/ena_datapath.c =================================================================== --- head/sys/dev/ena/ena_datapath.c +++ head/sys/dev/ena/ena_datapath.c @@ -32,6 +32,9 @@ #include "ena.h" #include "ena_datapath.h" +#ifdef DEV_NETMAP +#include "ena_netmap.h" +#endif /* DEV_NETMAP */ /********************************************************************* * Static functions prototypes @@ -40,7 +43,6 @@ static int ena_tx_cleanup(struct ena_ring *); static int ena_rx_cleanup(struct ena_ring *); static inline int validate_tx_req_id(struct ena_ring *, uint16_t); -static inline int validate_rx_req_id(struct ena_ring *, uint16_t); static void ena_rx_hash_mbuf(struct ena_ring *, struct ena_com_rx_ctx *, struct mbuf *); static struct mbuf* ena_rx_mbuf(struct ena_ring *, struct ena_com_rx_buf_info *, @@ -206,25 +208,6 @@ return (EFAULT); } -static inline int -validate_rx_req_id(struct ena_ring *rx_ring, uint16_t req_id) -{ - if (likely(req_id < rx_ring->ring_size)) - return (0); - - device_printf(rx_ring->adapter->pdev, "Invalid rx req_id: %hu\n", - req_id); - counter_u64_add(rx_ring->rx_stats.bad_req_id, 1); - - /* Trigger device reset */ - if (likely(!ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, rx_ring->adapter))) { - rx_ring->adapter->reset_reason = ENA_REGS_RESET_INV_RX_REQ_ID; - ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, rx_ring->adapter); - } - - return (EFAULT); -} - /** * ena_tx_cleanup - clear sent packets and corresponding descriptors * @tx_ring: ring for which we want to clean packets @@ -577,6 +560,9 @@ unsigned int qid; int rc, i; int budget = RX_BUDGET; +#ifdef DEV_NETMAP + int done; +#endif /* DEV_NETMAP */ adapter = rx_ring->que->adapter; ifp = adapter->ifp; @@ -585,6 +571,11 @@ io_cq = &adapter->ena_dev->io_cq_queues[ena_qid]; io_sq = &adapter->ena_dev->io_sq_queues[ena_qid]; next_to_clean = rx_ring->next_to_clean; + +#ifdef DEV_NETMAP + if (netmap_rx_irq(adapter->ifp, rx_ring->qid, &done) != NM_IRQ_PASS) + return (0); +#endif /* DEV_NETMAP */ ena_trace(ENA_DBG, "rx: qid %d\n", qid); Index: head/sys/dev/ena/ena_netmap.h =================================================================== --- head/sys/dev/ena/ena_netmap.h +++ head/sys/dev/ena/ena_netmap.h @@ -47,5 +47,10 @@ #include int ena_netmap_attach(struct ena_adapter *); +int ena_netmap_alloc_rx_slot(struct ena_adapter *, struct ena_ring *, + struct ena_rx_buffer *); +void ena_netmap_free_rx_slot(struct ena_adapter *, struct ena_ring *, + struct ena_rx_buffer *); +void ena_netmap_reset_rx_ring(struct ena_adapter *, int); #endif /* _ENA_NETMAP_H_ */ Index: head/sys/dev/ena/ena_netmap.c =================================================================== --- head/sys/dev/ena/ena_netmap.c +++ head/sys/dev/ena/ena_netmap.c @@ -35,10 +35,37 @@ #include "ena.h" #include "ena_netmap.h" +#define ENA_NETMAP_MORE_FRAMES 1 +#define ENA_NETMAP_NO_MORE_FRAMES 0 +#define ENA_MAX_FRAMES 16384 + +struct ena_netmap_ctx { + struct netmap_kring *kring; + struct ena_adapter *adapter; + struct netmap_adapter *na; + struct netmap_slot *slots; + struct ena_ring *ring; + struct ena_com_io_cq *io_cq; + struct ena_com_io_sq *io_sq; + u_int nm_i; + uint16_t nt; + uint16_t lim; +}; + +/* Netmap callbacks */ static int ena_netmap_reg(struct netmap_adapter *, int); static int ena_netmap_txsync(struct netmap_kring *, int); static int ena_netmap_rxsync(struct netmap_kring *, int); +/* Helper functions */ +static int ena_netmap_rx_frames(struct ena_netmap_ctx *); +static int ena_netmap_rx_frame(struct ena_netmap_ctx *); +static int ena_netmap_rx_load_desc(struct ena_netmap_ctx *, uint16_t, + int *); +static void ena_netmap_rx_cleanup(struct ena_netmap_ctx *); +static void ena_netmap_fill_ctx(struct netmap_kring *, + struct ena_netmap_ctx *, uint16_t); + int ena_netmap_attach(struct ena_adapter *adapter) { @@ -61,6 +88,124 @@ return (netmap_attach(&na)); } +int +ena_netmap_alloc_rx_slot(struct ena_adapter *adapter, + struct ena_ring *rx_ring, struct ena_rx_buffer *rx_info) +{ + struct netmap_adapter *na = NA(adapter->ifp); + struct netmap_kring *kring; + struct netmap_ring *ring; + struct netmap_slot *slot; + void *addr; + uint64_t paddr; + int nm_i, qid, head, lim, rc; + + /* if previously allocated frag is not used */ + if (unlikely(rx_info->netmap_buf_idx != 0)) + return (0); + + qid = rx_ring->qid; + kring = na->rx_rings[qid]; + nm_i = kring->nr_hwcur; + head = kring->rhead; + + ena_trace(ENA_NETMAP | ENA_DBG, "nr_hwcur: %d, nr_hwtail: %d, " + "rhead: %d, rcur: %d, rtail: %d\n", kring->nr_hwcur, + kring->nr_hwtail, kring->rhead, kring->rcur, kring->rtail); + + if ((nm_i == head) && rx_ring->initialized) { + ena_trace(ENA_NETMAP, "No free slots in netmap ring\n"); + return (ENOMEM); + } + + ring = kring->ring; + if (ring == NULL) { + device_printf(adapter->pdev, "Rx ring %d is NULL\n", qid); + return (EFAULT); + } + slot = &ring->slot[nm_i]; + + addr = PNMB(na, slot, &paddr); + if (addr == NETMAP_BUF_BASE(na)) { + device_printf(adapter->pdev, "Bad buff in slot\n"); + return (EFAULT); + } + + rc = netmap_load_map(na, adapter->rx_buf_tag, rx_info->map, addr); + if (rc != 0) { + ena_trace(ENA_WARNING, "DMA mapping error\n"); + return (rc); + } + bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map, BUS_DMASYNC_PREREAD); + + rx_info->ena_buf.paddr = paddr; + rx_info->ena_buf.len = ring->nr_buf_size; + rx_info->mbuf = NULL; + rx_info->netmap_buf_idx = slot->buf_idx; + + slot->buf_idx = 0; + + lim = kring->nkr_num_slots - 1; + kring->nr_hwcur = nm_next(nm_i, lim); + + return (0); +} + +void +ena_netmap_free_rx_slot(struct ena_adapter *adapter, + struct ena_ring *rx_ring, struct ena_rx_buffer *rx_info) +{ + struct netmap_adapter *na; + struct netmap_kring *kring; + struct netmap_slot *slot; + int nm_i, qid, lim; + + na = NA(adapter->ifp); + if (na == NULL) { + device_printf(adapter->pdev, "netmap adapter is NULL\n"); + return; + } + + if (na->rx_rings == NULL) { + device_printf(adapter->pdev, "netmap rings are NULL\n"); + return; + } + + qid = rx_ring->qid; + kring = na->rx_rings[qid]; + if (kring == NULL) { + device_printf(adapter->pdev, + "netmap kernel ring %d is NULL\n", qid); + return; + } + + lim = kring->nkr_num_slots - 1; + nm_i = nm_prev(kring->nr_hwcur, lim); + + if (kring->nr_mode != NKR_NETMAP_ON) + return; + + bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map, + BUS_DMASYNC_POSTREAD); + netmap_unload_map(na, adapter->rx_buf_tag, rx_info->map); + + slot = &kring->ring->slot[nm_i]; + + ENA_ASSERT(slot->buf_idx == 0, "Overwrite slot buf\n"); + slot->buf_idx = rx_info->netmap_buf_idx; + slot->flags = NS_BUF_CHANGED; + + rx_info->netmap_buf_idx = 0; + kring->nr_hwcur = nm_i; +} + +void +ena_netmap_reset_rx_ring(struct ena_adapter *adapter, int qid) +{ + if (adapter->ifp->if_capenable & IFCAP_NETMAP) + netmap_reset(NA(adapter->ifp), NR_RX, qid, 0); +} + static int ena_netmap_reg(struct netmap_adapter *na, int onoff) { @@ -104,8 +249,192 @@ static int ena_netmap_rxsync(struct netmap_kring *kring, int flags) { - ena_trace(ENA_NETMAP, "netmap rxsync\n"); + struct ena_netmap_ctx ctx; + int rc; + + ena_netmap_fill_ctx(kring, &ctx, ENA_IO_RXQ_IDX(kring->ring_id)); + ctx.ring = &ctx.adapter->rx_ring[kring->ring_id]; + + if (ctx.kring->rhead > ctx.lim) { + /* Probably not needed to release slots from RX ring. */ + return (netmap_ring_reinit(ctx.kring)); + } + + if (unlikely((if_getdrvflags(ctx.na->ifp) & IFF_DRV_RUNNING) == 0)) + return (0); + + if (unlikely(!ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, ctx.adapter))) + return (0); + + if ((rc = ena_netmap_rx_frames(&ctx)) != 0) + return (rc); + + ena_netmap_rx_cleanup(&ctx); + return (0); +} + +static inline int +ena_netmap_rx_frames(struct ena_netmap_ctx *ctx) +{ + int rc = 0; + int frames_counter = 0; + + ctx->nt = ctx->ring->next_to_clean; + ctx->nm_i = ctx->kring->nr_hwtail; + + while((rc = ena_netmap_rx_frame(ctx)) == ENA_NETMAP_MORE_FRAMES) { + frames_counter++; + /* In case of multiple frames, it is not an error. */ + rc = 0; + if (frames_counter > ENA_MAX_FRAMES) { + device_printf(ctx->adapter->pdev, + "Driver is stuck in the Rx loop\n"); + break; + } + }; + + ctx->kring->nr_hwtail = ctx->nm_i; + ctx->kring->nr_kflags &= ~NKR_PENDINTR; + ctx->ring->next_to_clean = ctx->nt; + + return (rc); +} + +static inline int +ena_netmap_rx_frame(struct ena_netmap_ctx *ctx) +{ + struct ena_com_rx_ctx ena_rx_ctx; + int rc, len = 0; + uint16_t buf, nm; + + ena_rx_ctx.ena_bufs = ctx->ring->ena_bufs; + ena_rx_ctx.max_bufs = ctx->adapter->max_rx_sgl_size; + bus_dmamap_sync(ctx->io_cq->cdesc_addr.mem_handle.tag, + ctx->io_cq->cdesc_addr.mem_handle.map, BUS_DMASYNC_POSTREAD); + + rc = ena_com_rx_pkt(ctx->io_cq, ctx->io_sq, &ena_rx_ctx); + if (unlikely(rc != 0)) { + ena_trace(ENA_ALERT, "Too many desc from the device.\n"); + counter_u64_add(ctx->ring->rx_stats.bad_desc_num, 1); + ctx->adapter->reset_reason = ENA_REGS_RESET_TOO_MANY_RX_DESCS; + ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, ctx->adapter); + return (rc); + } + if (unlikely(ena_rx_ctx.descs == 0)) + return (ENA_NETMAP_NO_MORE_FRAMES); + + ena_trace(ENA_NETMAP | ENA_DBG, "Rx: q %d got packet from ena. descs #:" + " %d l3 proto %d l4 proto %d hash: %x\n", ctx->ring->qid, + ena_rx_ctx.descs, ena_rx_ctx.l3_proto, ena_rx_ctx.l4_proto, + ena_rx_ctx.hash); + + for (buf = 0; buf < ena_rx_ctx.descs; buf++) + if ((rc = ena_netmap_rx_load_desc(ctx, buf, &len)) != 0) + break; + /* + * ena_netmap_rx_load_desc doesn't know the number of descriptors. + * It just set flag NS_MOREFRAG to all slots, then here flag of + * last slot is cleared. + */ + ctx->slots[nm_prev(ctx->nm_i, ctx->lim)].flags = NS_BUF_CHANGED; + + if (rc != 0) { + goto rx_clear_desc; + } + + bus_dmamap_sync(ctx->io_cq->cdesc_addr.mem_handle.tag, + ctx->io_cq->cdesc_addr.mem_handle.map, BUS_DMASYNC_PREREAD); + + counter_enter(); + counter_u64_add_protected(ctx->ring->rx_stats.bytes, len); + counter_u64_add_protected(ctx->adapter->hw_stats.rx_bytes, len); + counter_u64_add_protected(ctx->ring->rx_stats.cnt, 1); + counter_u64_add_protected(ctx->adapter->hw_stats.rx_packets, 1); + counter_exit(); + + return (ENA_NETMAP_MORE_FRAMES); + +rx_clear_desc: + nm = ctx->nm_i; + + /* Remove failed packet from ring */ + while(buf--) { + ctx->slots[nm].flags = 0; + ctx->slots[nm].len = 0; + nm = nm_prev(nm, ctx->lim); + } + + return (rc); +} + +static inline int +ena_netmap_rx_load_desc(struct ena_netmap_ctx *ctx, uint16_t buf, int *len) +{ + struct ena_rx_buffer *rx_info; + uint16_t req_id; + int rc; + + req_id = ctx->ring->ena_bufs[buf].req_id; + rc = validate_rx_req_id(ctx->ring, req_id); + if (unlikely(rc != 0)) + return (rc); + + rx_info = &ctx->ring->rx_buffer_info[req_id]; + bus_dmamap_sync(ctx->adapter->rx_buf_tag, rx_info->map, + BUS_DMASYNC_POSTREAD); + netmap_unload_map(ctx->na, ctx->adapter->rx_buf_tag, rx_info->map); + + ENA_ASSERT(ctx->slots[ctx->nm_i].buf_idx == 0, "Rx idx is not 0.\n"); + + ctx->slots[ctx->nm_i].buf_idx = rx_info->netmap_buf_idx; + rx_info->netmap_buf_idx = 0; + /* + * Set NS_MOREFRAG to all slots. + * Then ena_netmap_rx_frame clears it from last one. + */ + ctx->slots[ctx->nm_i].flags |= NS_MOREFRAG | NS_BUF_CHANGED; + ctx->slots[ctx->nm_i].len = ctx->ring->ena_bufs[buf].len; + *len += ctx->slots[ctx->nm_i].len; + ctx->ring->free_rx_ids[ctx->nt] = req_id; + ena_trace(ENA_DBG, "rx_info %p, buf_idx %d, paddr %jx, nm: %d\n", + rx_info, ctx->slots[ctx->nm_i].buf_idx, + (uintmax_t)rx_info->ena_buf.paddr, ctx->nm_i); + + ctx->nm_i = nm_next(ctx->nm_i, ctx->lim); + ctx->nt = ENA_RX_RING_IDX_NEXT(ctx->nt, ctx->ring->ring_size); + + return (0); +} + +static inline void +ena_netmap_rx_cleanup(struct ena_netmap_ctx *ctx) +{ + int refill_required; + + refill_required = ctx->kring->rhead - ctx->kring->nr_hwcur; + if (ctx->kring->nr_hwcur != ctx->kring->nr_hwtail) + refill_required -= 1; + + if (refill_required == 0) + return; + else if (refill_required < 0) + refill_required += ctx->kring->nkr_num_slots; + + ena_refill_rx_bufs(ctx->ring, refill_required); +} + +static inline void +ena_netmap_fill_ctx(struct netmap_kring *kring, struct ena_netmap_ctx *ctx, + uint16_t ena_qid) +{ + ctx->kring = kring; + ctx->na = kring->na; + ctx->adapter = ctx->na->ifp->if_softc; + ctx->lim = kring->nkr_num_slots - 1; + ctx->io_cq = &ctx->adapter->ena_dev->io_cq_queues[ena_qid]; + ctx->io_sq = &ctx->adapter->ena_dev->io_sq_queues[ena_qid]; + ctx->slots = kring->ring->slot; } #endif /* DEV_NETMAP */