Changeset View
Changeset View
Standalone View
Standalone View
head/sys/net/iflib.c
Show First 20 Lines • Show All 86 Lines • ▼ Show 20 Lines | |||||
#include <dev/pci/pcivar.h> | #include <dev/pci/pcivar.h> | ||||
#include <dev/pci/pci_private.h> | #include <dev/pci/pci_private.h> | ||||
#include <net/iflib.h> | #include <net/iflib.h> | ||||
#include <net/iflib_private.h> | #include <net/iflib_private.h> | ||||
#include "ifdi_if.h" | #include "ifdi_if.h" | ||||
#if defined(__i386__) || defined(__amd64__) | |||||
#include <sys/memdesc.h> | |||||
#include <machine/bus.h> | |||||
#include <machine/md_var.h> | |||||
#include <machine/specialreg.h> | |||||
#include <x86/include/busdma_impl.h> | |||||
#include <x86/iommu/busdma_dmar.h> | |||||
#endif | |||||
#ifdef PCI_IOV | #ifdef PCI_IOV | ||||
#include <dev/pci/pci_iov.h> | #include <dev/pci/pci_iov.h> | ||||
#endif | #endif | ||||
#include <sys/bitstring.h> | #include <sys/bitstring.h> | ||||
/* | /* | ||||
* enable accounting of every mbuf as it comes in to and goes out of | * enable accounting of every mbuf as it comes in to and goes out of | ||||
* iflib's software descriptor references | * iflib's software descriptor references | ||||
▲ Show 20 Lines • Show All 165 Lines • ▼ Show 20 Lines | |||||
#define IP_ALIGNED(m) ((((uintptr_t)(m)->m_data) & 0x3) == 0x2) | #define IP_ALIGNED(m) ((((uintptr_t)(m)->m_data) & 0x3) == 0x2) | ||||
#define CACHE_PTR_INCREMENT (CACHE_LINE_SIZE/sizeof(void*)) | #define CACHE_PTR_INCREMENT (CACHE_LINE_SIZE/sizeof(void*)) | ||||
#define CACHE_PTR_NEXT(ptr) ((void *)(((uintptr_t)(ptr)+CACHE_LINE_SIZE-1) & (CACHE_LINE_SIZE-1))) | #define CACHE_PTR_NEXT(ptr) ((void *)(((uintptr_t)(ptr)+CACHE_LINE_SIZE-1) & (CACHE_LINE_SIZE-1))) | ||||
#define LINK_ACTIVE(ctx) ((ctx)->ifc_link_state == LINK_STATE_UP) | #define LINK_ACTIVE(ctx) ((ctx)->ifc_link_state == LINK_STATE_UP) | ||||
#define CTX_IS_VF(ctx) ((ctx)->ifc_sctx->isc_flags & IFLIB_IS_VF) | #define CTX_IS_VF(ctx) ((ctx)->ifc_sctx->isc_flags & IFLIB_IS_VF) | ||||
#define RX_SW_DESC_MAP_CREATED (1 << 0) | |||||
#define TX_SW_DESC_MAP_CREATED (1 << 1) | |||||
#define RX_SW_DESC_INUSE (1 << 3) | |||||
#define TX_SW_DESC_MAPPED (1 << 4) | |||||
#define M_TOOBIG M_PROTO1 | |||||
typedef struct iflib_sw_rx_desc_array { | typedef struct iflib_sw_rx_desc_array { | ||||
bus_dmamap_t *ifsd_map; /* bus_dma maps for packet */ | bus_dmamap_t *ifsd_map; /* bus_dma maps for packet */ | ||||
struct mbuf **ifsd_m; /* pkthdr mbufs */ | struct mbuf **ifsd_m; /* pkthdr mbufs */ | ||||
caddr_t *ifsd_cl; /* direct cluster pointer for rx */ | caddr_t *ifsd_cl; /* direct cluster pointer for rx */ | ||||
uint8_t *ifsd_flags; | bus_addr_t *ifsd_ba; /* bus addr of cluster for rx */ | ||||
} iflib_rxsd_array_t; | } iflib_rxsd_array_t; | ||||
typedef struct iflib_sw_tx_desc_array { | typedef struct iflib_sw_tx_desc_array { | ||||
bus_dmamap_t *ifsd_map; /* bus_dma maps for packet */ | bus_dmamap_t *ifsd_map; /* bus_dma maps for packet */ | ||||
struct mbuf **ifsd_m; /* pkthdr mbufs */ | struct mbuf **ifsd_m; /* pkthdr mbufs */ | ||||
uint8_t *ifsd_flags; | |||||
} if_txsd_vec_t; | } if_txsd_vec_t; | ||||
/* magic number that should be high enough for any hardware */ | /* magic number that should be high enough for any hardware */ | ||||
#define IFLIB_MAX_TX_SEGS 128 | #define IFLIB_MAX_TX_SEGS 128 | ||||
/* bnxt supports 64 with hardware LRO enabled */ | /* bnxt supports 64 with hardware LRO enabled */ | ||||
#define IFLIB_MAX_RX_SEGS 64 | #define IFLIB_MAX_RX_SEGS 64 | ||||
#define IFLIB_RX_COPY_THRESH 128 | #define IFLIB_RX_COPY_THRESH 128 | ||||
▲ Show 20 Lines • Show All 624 Lines • ▼ Show 20 Lines | iflib_netmap_txsync(struct netmap_kring *kring, int flags) | ||||
* interrupts on every tx packet are expensive so request | * interrupts on every tx packet are expensive so request | ||||
* them every half ring, or where NS_REPORT is set | * them every half ring, or where NS_REPORT is set | ||||
*/ | */ | ||||
u_int report_frequency = kring->nkr_num_slots >> 1; | u_int report_frequency = kring->nkr_num_slots >> 1; | ||||
/* device-specific */ | /* device-specific */ | ||||
if_ctx_t ctx = ifp->if_softc; | if_ctx_t ctx = ifp->if_softc; | ||||
iflib_txq_t txq = &ctx->ifc_txqs[kring->ring_id]; | iflib_txq_t txq = &ctx->ifc_txqs[kring->ring_id]; | ||||
if (txq->ift_sds.ifsd_map) | |||||
bus_dmamap_sync(txq->ift_desc_tag, txq->ift_ifdi->idi_map, | bus_dmamap_sync(txq->ift_desc_tag, txq->ift_ifdi->idi_map, | ||||
BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); | BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); | ||||
/* | /* | ||||
* First part: process new packets to send. | * First part: process new packets to send. | ||||
* nm_i is the current index in the netmap kring, | * nm_i is the current index in the netmap kring, | ||||
* nic_i is the corresponding index in the NIC ring. | * nic_i is the corresponding index in the NIC ring. | ||||
* | * | ||||
* If we have packets to send (nm_i != head) | * If we have packets to send (nm_i != head) | ||||
▲ Show 20 Lines • Show All 65 Lines • ▼ Show 20 Lines | for (n = 0; nm_i != head; n++) { | ||||
} | } | ||||
slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED); | slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED); | ||||
nm_i = nm_next(nm_i, lim); | nm_i = nm_next(nm_i, lim); | ||||
nic_i = nm_next(nic_i, lim); | nic_i = nm_next(nic_i, lim); | ||||
} | } | ||||
kring->nr_hwcur = nm_i; | kring->nr_hwcur = nm_i; | ||||
/* synchronize the NIC ring */ | /* synchronize the NIC ring */ | ||||
if (txq->ift_sds.ifsd_map) | |||||
bus_dmamap_sync(txq->ift_desc_tag, txq->ift_ifdi->idi_map, | bus_dmamap_sync(txq->ift_desc_tag, txq->ift_ifdi->idi_map, | ||||
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | ||||
/* (re)start the tx unit up to slot nic_i (excluded) */ | /* (re)start the tx unit up to slot nic_i (excluded) */ | ||||
ctx->isc_txd_flush(ctx->ifc_softc, txq->ift_id, nic_i); | ctx->isc_txd_flush(ctx->ifc_softc, txq->ift_id, nic_i); | ||||
} | } | ||||
/* | /* | ||||
* Second part: reclaim buffers for completed transmissions. | * Second part: reclaim buffers for completed transmissions. | ||||
* | * | ||||
▲ Show 20 Lines • Show All 86 Lines • ▼ Show 20 Lines | for (i = 0; i < rxq->ifr_nfl; i++) { | ||||
ri.iri_frags = rxq->ifr_frags; | ri.iri_frags = rxq->ifr_frags; | ||||
ri.iri_qsidx = kring->ring_id; | ri.iri_qsidx = kring->ring_id; | ||||
ri.iri_ifp = ctx->ifc_ifp; | ri.iri_ifp = ctx->ifc_ifp; | ||||
ri.iri_cidx = nic_i; | ri.iri_cidx = nic_i; | ||||
error = ctx->isc_rxd_pkt_get(ctx->ifc_softc, &ri); | error = ctx->isc_rxd_pkt_get(ctx->ifc_softc, &ri); | ||||
ring->slot[nm_i].len = error ? 0 : ri.iri_len - crclen; | ring->slot[nm_i].len = error ? 0 : ri.iri_len - crclen; | ||||
ring->slot[nm_i].flags = 0; | ring->slot[nm_i].flags = 0; | ||||
if (fl->ifl_sds.ifsd_map) | |||||
bus_dmamap_sync(fl->ifl_ifdi->idi_tag, | bus_dmamap_sync(fl->ifl_ifdi->idi_tag, | ||||
fl->ifl_sds.ifsd_map[nic_i], BUS_DMASYNC_POSTREAD); | fl->ifl_sds.ifsd_map[nic_i], BUS_DMASYNC_POSTREAD); | ||||
nm_i = nm_next(nm_i, lim); | nm_i = nm_next(nm_i, lim); | ||||
nic_i = nm_next(nic_i, lim); | nic_i = nm_next(nic_i, lim); | ||||
} | } | ||||
if (n) { /* update the state variables */ | if (n) { /* update the state variables */ | ||||
if (netmap_no_pendintr && !force_update) { | if (netmap_no_pendintr && !force_update) { | ||||
/* diagnostics */ | /* diagnostics */ | ||||
iflib_rx_miss ++; | iflib_rx_miss ++; | ||||
iflib_rx_miss_bufs += n; | iflib_rx_miss_bufs += n; | ||||
▲ Show 20 Lines • Show All 62 Lines • ▼ Show 20 Lines | |||||
iflib_netmap_txq_init(if_ctx_t ctx, iflib_txq_t txq) | iflib_netmap_txq_init(if_ctx_t ctx, iflib_txq_t txq) | ||||
{ | { | ||||
struct netmap_adapter *na = NA(ctx->ifc_ifp); | struct netmap_adapter *na = NA(ctx->ifc_ifp); | ||||
struct netmap_slot *slot; | struct netmap_slot *slot; | ||||
slot = netmap_reset(na, NR_TX, txq->ift_id, 0); | slot = netmap_reset(na, NR_TX, txq->ift_id, 0); | ||||
if (slot == NULL) | if (slot == NULL) | ||||
return; | return; | ||||
if (txq->ift_sds.ifsd_map == NULL) | |||||
return; | |||||
for (int i = 0; i < ctx->ifc_softc_ctx.isc_ntxd[0]; i++) { | for (int i = 0; i < ctx->ifc_softc_ctx.isc_ntxd[0]; i++) { | ||||
/* | /* | ||||
* In netmap mode, set the map for the packet buffer. | * In netmap mode, set the map for the packet buffer. | ||||
* NOTE: Some drivers (not this one) also need to set | * NOTE: Some drivers (not this one) also need to set | ||||
* the physical buffer address in the NIC ring. | * the physical buffer address in the NIC ring. | ||||
* netmap_idx_n2k() maps a nic index, i, into the corresponding | * netmap_idx_n2k() maps a nic index, i, into the corresponding | ||||
* netmap slot index, si | * netmap slot index, si | ||||
▲ Show 20 Lines • Show All 428 Lines • ▼ Show 20 Lines | if ((if_getcapabilities(ctx->ifc_ifp) & IFCAP_TSO) & | ||||
0, /* flags */ | 0, /* flags */ | ||||
NULL, /* lockfunc */ | NULL, /* lockfunc */ | ||||
NULL, /* lockfuncarg */ | NULL, /* lockfuncarg */ | ||||
&txq->ift_tso_desc_tag))) { | &txq->ift_tso_desc_tag))) { | ||||
device_printf(dev,"Unable to allocate TX TSO DMA tag: %d\n", err); | device_printf(dev,"Unable to allocate TX TSO DMA tag: %d\n", err); | ||||
goto fail; | goto fail; | ||||
} | } | ||||
if (!(txq->ift_sds.ifsd_flags = | |||||
(uint8_t *) malloc(sizeof(uint8_t) * | |||||
scctx->isc_ntxd[txq->ift_br_offset], M_IFLIB, M_NOWAIT | M_ZERO))) { | |||||
device_printf(dev, "Unable to allocate tx_buffer memory\n"); | |||||
err = ENOMEM; | |||||
goto fail; | |||||
} | |||||
if (!(txq->ift_sds.ifsd_m = | if (!(txq->ift_sds.ifsd_m = | ||||
(struct mbuf **) malloc(sizeof(struct mbuf *) * | (struct mbuf **) malloc(sizeof(struct mbuf *) * | ||||
scctx->isc_ntxd[txq->ift_br_offset], M_IFLIB, M_NOWAIT | M_ZERO))) { | scctx->isc_ntxd[txq->ift_br_offset], M_IFLIB, M_NOWAIT | M_ZERO))) { | ||||
device_printf(dev, "Unable to allocate tx_buffer memory\n"); | device_printf(dev, "Unable to allocate tx_buffer memory\n"); | ||||
err = ENOMEM; | err = ENOMEM; | ||||
goto fail; | goto fail; | ||||
} | } | ||||
/* Create the descriptor buffer dma maps */ | /* Create the descriptor buffer dma maps */ | ||||
#if defined(ACPI_DMAR) || (! (defined(__i386__) || defined(__amd64__))) | |||||
if ((ctx->ifc_flags & IFC_DMAR) == 0) | |||||
return (0); | |||||
if (!(txq->ift_sds.ifsd_map = | if (!(txq->ift_sds.ifsd_map = | ||||
(bus_dmamap_t *) malloc(sizeof(bus_dmamap_t) * scctx->isc_ntxd[txq->ift_br_offset], M_IFLIB, M_NOWAIT | M_ZERO))) { | (bus_dmamap_t *) malloc(sizeof(bus_dmamap_t) * scctx->isc_ntxd[txq->ift_br_offset], M_IFLIB, M_NOWAIT | M_ZERO))) { | ||||
device_printf(dev, "Unable to allocate tx_buffer map memory\n"); | device_printf(dev, "Unable to allocate tx_buffer map memory\n"); | ||||
err = ENOMEM; | err = ENOMEM; | ||||
goto fail; | goto fail; | ||||
} | } | ||||
for (int i = 0; i < scctx->isc_ntxd[txq->ift_br_offset]; i++) { | for (int i = 0; i < scctx->isc_ntxd[txq->ift_br_offset]; i++) { | ||||
err = bus_dmamap_create(txq->ift_desc_tag, 0, &txq->ift_sds.ifsd_map[i]); | err = bus_dmamap_create(txq->ift_desc_tag, 0, &txq->ift_sds.ifsd_map[i]); | ||||
if (err != 0) { | if (err != 0) { | ||||
device_printf(dev, "Unable to create TX DMA map\n"); | device_printf(dev, "Unable to create TX DMA map\n"); | ||||
goto fail; | goto fail; | ||||
} | } | ||||
} | } | ||||
#endif | |||||
return (0); | return (0); | ||||
fail: | fail: | ||||
/* We free all, it handles case where we are in the middle */ | /* We free all, it handles case where we are in the middle */ | ||||
iflib_tx_structures_free(ctx); | iflib_tx_structures_free(ctx); | ||||
return (err); | return (err); | ||||
} | } | ||||
static void | static void | ||||
Show All 21 Lines | iflib_txq_destroy(iflib_txq_t txq) | ||||
if (txq->ift_sds.ifsd_map != NULL) { | if (txq->ift_sds.ifsd_map != NULL) { | ||||
free(txq->ift_sds.ifsd_map, M_IFLIB); | free(txq->ift_sds.ifsd_map, M_IFLIB); | ||||
txq->ift_sds.ifsd_map = NULL; | txq->ift_sds.ifsd_map = NULL; | ||||
} | } | ||||
if (txq->ift_sds.ifsd_m != NULL) { | if (txq->ift_sds.ifsd_m != NULL) { | ||||
free(txq->ift_sds.ifsd_m, M_IFLIB); | free(txq->ift_sds.ifsd_m, M_IFLIB); | ||||
txq->ift_sds.ifsd_m = NULL; | txq->ift_sds.ifsd_m = NULL; | ||||
} | } | ||||
if (txq->ift_sds.ifsd_flags != NULL) { | |||||
free(txq->ift_sds.ifsd_flags, M_IFLIB); | |||||
txq->ift_sds.ifsd_flags = NULL; | |||||
} | |||||
if (txq->ift_desc_tag != NULL) { | if (txq->ift_desc_tag != NULL) { | ||||
bus_dma_tag_destroy(txq->ift_desc_tag); | bus_dma_tag_destroy(txq->ift_desc_tag); | ||||
txq->ift_desc_tag = NULL; | txq->ift_desc_tag = NULL; | ||||
} | } | ||||
if (txq->ift_tso_desc_tag != NULL) { | if (txq->ift_tso_desc_tag != NULL) { | ||||
bus_dma_tag_destroy(txq->ift_tso_desc_tag); | bus_dma_tag_destroy(txq->ift_tso_desc_tag); | ||||
txq->ift_tso_desc_tag = NULL; | txq->ift_tso_desc_tag = NULL; | ||||
} | } | ||||
▲ Show 20 Lines • Show All 85 Lines • ▼ Show 20 Lines | err = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ | ||||
NULL, /* lockfunc */ | NULL, /* lockfunc */ | ||||
NULL, /* lockarg */ | NULL, /* lockarg */ | ||||
&fl->ifl_desc_tag); | &fl->ifl_desc_tag); | ||||
if (err) { | if (err) { | ||||
device_printf(dev, "%s: bus_dma_tag_create failed %d\n", | device_printf(dev, "%s: bus_dma_tag_create failed %d\n", | ||||
__func__, err); | __func__, err); | ||||
goto fail; | goto fail; | ||||
} | } | ||||
if (!(fl->ifl_sds.ifsd_flags = | |||||
(uint8_t *) malloc(sizeof(uint8_t) * | |||||
scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT | M_ZERO))) { | |||||
device_printf(dev, "Unable to allocate tx_buffer memory\n"); | |||||
err = ENOMEM; | |||||
goto fail; | |||||
} | |||||
if (!(fl->ifl_sds.ifsd_m = | if (!(fl->ifl_sds.ifsd_m = | ||||
(struct mbuf **) malloc(sizeof(struct mbuf *) * | (struct mbuf **) malloc(sizeof(struct mbuf *) * | ||||
scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT | M_ZERO))) { | scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT | M_ZERO))) { | ||||
device_printf(dev, "Unable to allocate tx_buffer memory\n"); | device_printf(dev, "Unable to allocate tx_buffer memory\n"); | ||||
err = ENOMEM; | err = ENOMEM; | ||||
goto fail; | goto fail; | ||||
} | } | ||||
if (!(fl->ifl_sds.ifsd_cl = | if (!(fl->ifl_sds.ifsd_cl = | ||||
(caddr_t *) malloc(sizeof(caddr_t) * | (caddr_t *) malloc(sizeof(caddr_t) * | ||||
scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT | M_ZERO))) { | scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT | M_ZERO))) { | ||||
device_printf(dev, "Unable to allocate tx_buffer memory\n"); | device_printf(dev, "Unable to allocate tx_buffer memory\n"); | ||||
err = ENOMEM; | err = ENOMEM; | ||||
goto fail; | goto fail; | ||||
} | } | ||||
/* Create the descriptor buffer dma maps */ | if (!(fl->ifl_sds.ifsd_ba = | ||||
#if defined(ACPI_DMAR) || (! (defined(__i386__) || defined(__amd64__))) | (bus_addr_t *) malloc(sizeof(bus_addr_t) * | ||||
if ((ctx->ifc_flags & IFC_DMAR) == 0) | scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT | M_ZERO))) { | ||||
continue; | device_printf(dev, "Unable to allocate rx bus addr memory\n"); | ||||
err = ENOMEM; | |||||
goto fail; | |||||
} | |||||
/* Create the descriptor buffer dma maps */ | |||||
if (!(fl->ifl_sds.ifsd_map = | if (!(fl->ifl_sds.ifsd_map = | ||||
(bus_dmamap_t *) malloc(sizeof(bus_dmamap_t) * scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT | M_ZERO))) { | (bus_dmamap_t *) malloc(sizeof(bus_dmamap_t) * scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT | M_ZERO))) { | ||||
device_printf(dev, "Unable to allocate tx_buffer map memory\n"); | device_printf(dev, "Unable to allocate tx_buffer map memory\n"); | ||||
err = ENOMEM; | err = ENOMEM; | ||||
goto fail; | goto fail; | ||||
} | } | ||||
for (int i = 0; i < scctx->isc_nrxd[rxq->ifr_fl_offset]; i++) { | for (int i = 0; i < scctx->isc_nrxd[rxq->ifr_fl_offset]; i++) { | ||||
err = bus_dmamap_create(fl->ifl_desc_tag, 0, &fl->ifl_sds.ifsd_map[i]); | err = bus_dmamap_create(fl->ifl_desc_tag, 0, &fl->ifl_sds.ifsd_map[i]); | ||||
if (err != 0) { | if (err != 0) { | ||||
device_printf(dev, "Unable to create RX buffer DMA map\n"); | device_printf(dev, "Unable to create RX buffer DMA map\n"); | ||||
goto fail; | goto fail; | ||||
} | } | ||||
} | } | ||||
#endif | |||||
} | } | ||||
return (0); | return (0); | ||||
fail: | fail: | ||||
iflib_rx_structures_free(ctx); | iflib_rx_structures_free(ctx); | ||||
return (err); | return (err); | ||||
} | } | ||||
Show All 13 Lines | |||||
{ | { | ||||
struct rxq_refill_cb_arg *cb_arg = arg; | struct rxq_refill_cb_arg *cb_arg = arg; | ||||
cb_arg->error = error; | cb_arg->error = error; | ||||
cb_arg->seg = segs[0]; | cb_arg->seg = segs[0]; | ||||
cb_arg->nseg = nseg; | cb_arg->nseg = nseg; | ||||
} | } | ||||
#ifdef ACPI_DMAR | |||||
#define IS_DMAR(ctx) (ctx->ifc_flags & IFC_DMAR) | |||||
#else | |||||
#define IS_DMAR(ctx) (0) | |||||
#endif | |||||
/** | /** | ||||
* rxq_refill - refill an rxq free-buffer list | * rxq_refill - refill an rxq free-buffer list | ||||
* @ctx: the iflib context | * @ctx: the iflib context | ||||
* @rxq: the free-list to refill | * @rxq: the free-list to refill | ||||
* @n: the number of new buffers to allocate | * @n: the number of new buffers to allocate | ||||
* | * | ||||
* (Re)populate an rxq free-buffer list with up to @n new packet buffers. | * (Re)populate an rxq free-buffer list with up to @n new packet buffers. | ||||
* The caller must assure that @n does not exceed the queue's capacity. | * The caller must assure that @n does not exceed the queue's capacity. | ||||
*/ | */ | ||||
static void | static void | ||||
_iflib_fl_refill(if_ctx_t ctx, iflib_fl_t fl, int count) | _iflib_fl_refill(if_ctx_t ctx, iflib_fl_t fl, int count) | ||||
{ | { | ||||
struct mbuf *m; | struct mbuf *m; | ||||
int idx, frag_idx = fl->ifl_fragidx; | int idx, frag_idx = fl->ifl_fragidx; | ||||
int pidx = fl->ifl_pidx; | int pidx = fl->ifl_pidx; | ||||
caddr_t cl, *sd_cl; | caddr_t cl, *sd_cl; | ||||
struct mbuf **sd_m; | struct mbuf **sd_m; | ||||
uint8_t *sd_flags; | |||||
struct if_rxd_update iru; | struct if_rxd_update iru; | ||||
struct rxq_refill_cb_arg cb_arg; | |||||
bus_dmamap_t *sd_map; | bus_dmamap_t *sd_map; | ||||
int n, i = 0; | int n, i = 0; | ||||
uint64_t bus_addr; | bus_addr_t bus_addr, *sd_ba; | ||||
int err; | int err; | ||||
qidx_t credits; | qidx_t credits; | ||||
sd_m = fl->ifl_sds.ifsd_m; | sd_m = fl->ifl_sds.ifsd_m; | ||||
sd_map = fl->ifl_sds.ifsd_map; | sd_map = fl->ifl_sds.ifsd_map; | ||||
sd_cl = fl->ifl_sds.ifsd_cl; | sd_cl = fl->ifl_sds.ifsd_cl; | ||||
sd_flags = fl->ifl_sds.ifsd_flags; | sd_ba = fl->ifl_sds.ifsd_ba; | ||||
idx = pidx; | idx = pidx; | ||||
credits = fl->ifl_credits; | credits = fl->ifl_credits; | ||||
n = count; | n = count; | ||||
MPASS(n > 0); | MPASS(n > 0); | ||||
MPASS(credits + n <= fl->ifl_size); | MPASS(credits + n <= fl->ifl_size); | ||||
if (pidx < fl->ifl_cidx) | if (pidx < fl->ifl_cidx) | ||||
Show All 13 Lines | while (n--) { | ||||
* initialized after rx. | * initialized after rx. | ||||
* | * | ||||
* If the cluster is still set then we know a minimum sized packet was received | * If the cluster is still set then we know a minimum sized packet was received | ||||
*/ | */ | ||||
bit_ffc_at(fl->ifl_rx_bitmap, frag_idx, fl->ifl_size, &frag_idx); | bit_ffc_at(fl->ifl_rx_bitmap, frag_idx, fl->ifl_size, &frag_idx); | ||||
if ((frag_idx < 0) || (frag_idx >= fl->ifl_size)) | if ((frag_idx < 0) || (frag_idx >= fl->ifl_size)) | ||||
bit_ffc(fl->ifl_rx_bitmap, fl->ifl_size, &frag_idx); | bit_ffc(fl->ifl_rx_bitmap, fl->ifl_size, &frag_idx); | ||||
if ((cl = sd_cl[frag_idx]) == NULL) { | if ((cl = sd_cl[frag_idx]) == NULL) { | ||||
if ((cl = sd_cl[frag_idx] = m_cljget(NULL, M_NOWAIT, fl->ifl_buf_size)) == NULL) | if ((cl = m_cljget(NULL, M_NOWAIT, fl->ifl_buf_size)) == NULL) | ||||
break; | break; | ||||
#if MEMORY_LOGGING | |||||
fl->ifl_cl_enqueued++; | |||||
#endif | |||||
} | |||||
if ((m = m_gethdr(M_NOWAIT, MT_NOINIT)) == NULL) { | |||||
break; | |||||
} | |||||
#if MEMORY_LOGGING | |||||
fl->ifl_m_enqueued++; | |||||
#endif | |||||
DBG_COUNTER_INC(rx_allocs); | |||||
#if defined(__i386__) || defined(__amd64__) | |||||
if (!IS_DMAR(ctx)) { | |||||
bus_addr = pmap_kextract((vm_offset_t)cl); | |||||
} else | |||||
#endif | |||||
{ | |||||
struct rxq_refill_cb_arg cb_arg; | |||||
cb_arg.error = 0; | cb_arg.error = 0; | ||||
MPASS(sd_map != NULL); | MPASS(sd_map != NULL); | ||||
MPASS(sd_map[frag_idx] != NULL); | |||||
err = bus_dmamap_load(fl->ifl_desc_tag, sd_map[frag_idx], | err = bus_dmamap_load(fl->ifl_desc_tag, sd_map[frag_idx], | ||||
cl, fl->ifl_buf_size, _rxq_refill_cb, &cb_arg, 0); | cl, fl->ifl_buf_size, _rxq_refill_cb, &cb_arg, 0); | ||||
bus_dmamap_sync(fl->ifl_desc_tag, sd_map[frag_idx], | bus_dmamap_sync(fl->ifl_desc_tag, sd_map[frag_idx], | ||||
BUS_DMASYNC_PREREAD); | BUS_DMASYNC_PREREAD); | ||||
if (err != 0 || cb_arg.error) { | if (err != 0 || cb_arg.error) { | ||||
/* | /* | ||||
* !zone_pack ? | * !zone_pack ? | ||||
*/ | */ | ||||
if (fl->ifl_zone == zone_pack) | if (fl->ifl_zone == zone_pack) | ||||
uma_zfree(fl->ifl_zone, cl); | uma_zfree(fl->ifl_zone, cl); | ||||
m_free(m); | break; | ||||
n = 0; | |||||
goto done; | |||||
} | } | ||||
bus_addr = cb_arg.seg.ds_addr; | |||||
sd_ba[frag_idx] = bus_addr = cb_arg.seg.ds_addr; | |||||
sd_cl[frag_idx] = cl; | |||||
#if MEMORY_LOGGING | |||||
fl->ifl_cl_enqueued++; | |||||
#endif | |||||
} else { | |||||
bus_addr = sd_ba[frag_idx]; | |||||
} | } | ||||
bit_set(fl->ifl_rx_bitmap, frag_idx); | |||||
sd_flags[frag_idx] |= RX_SW_DESC_INUSE; | |||||
bit_set(fl->ifl_rx_bitmap, frag_idx); | |||||
MPASS(sd_m[frag_idx] == NULL); | MPASS(sd_m[frag_idx] == NULL); | ||||
sd_cl[frag_idx] = cl; | if ((m = m_gethdr(M_NOWAIT, MT_NOINIT)) == NULL) { | ||||
break; | |||||
} | |||||
sd_m[frag_idx] = m; | sd_m[frag_idx] = m; | ||||
#if MEMORY_LOGGING | |||||
fl->ifl_m_enqueued++; | |||||
#endif | |||||
DBG_COUNTER_INC(rx_allocs); | |||||
fl->ifl_rxd_idxs[i] = frag_idx; | fl->ifl_rxd_idxs[i] = frag_idx; | ||||
fl->ifl_bus_addrs[i] = bus_addr; | fl->ifl_bus_addrs[i] = bus_addr; | ||||
fl->ifl_vm_addrs[i] = cl; | fl->ifl_vm_addrs[i] = cl; | ||||
credits++; | credits++; | ||||
i++; | i++; | ||||
MPASS(credits <= fl->ifl_size); | MPASS(credits <= fl->ifl_size); | ||||
if (++idx == fl->ifl_size) { | if (++idx == fl->ifl_size) { | ||||
fl->ifl_gen = 1; | fl->ifl_gen = 1; | ||||
idx = 0; | idx = 0; | ||||
} | } | ||||
if (n == 0 || i == IFLIB_MAX_RX_REFRESH) { | if (n == 0 || i == IFLIB_MAX_RX_REFRESH) { | ||||
iru.iru_pidx = pidx; | iru.iru_pidx = pidx; | ||||
iru.iru_count = i; | iru.iru_count = i; | ||||
ctx->isc_rxd_refill(ctx->ifc_softc, &iru); | ctx->isc_rxd_refill(ctx->ifc_softc, &iru); | ||||
i = 0; | i = 0; | ||||
pidx = idx; | pidx = idx; | ||||
fl->ifl_pidx = idx; | fl->ifl_pidx = idx; | ||||
fl->ifl_credits = credits; | fl->ifl_credits = credits; | ||||
} | } | ||||
} | } | ||||
done: | |||||
if (i) { | if (i) { | ||||
iru.iru_pidx = pidx; | iru.iru_pidx = pidx; | ||||
iru.iru_count = i; | iru.iru_count = i; | ||||
ctx->isc_rxd_refill(ctx->ifc_softc, &iru); | ctx->isc_rxd_refill(ctx->ifc_softc, &iru); | ||||
fl->ifl_pidx = idx; | fl->ifl_pidx = idx; | ||||
fl->ifl_credits = credits; | fl->ifl_credits = credits; | ||||
} | } | ||||
DBG_COUNTER_INC(rxd_flush); | DBG_COUNTER_INC(rxd_flush); | ||||
if (fl->ifl_pidx == 0) | if (fl->ifl_pidx == 0) | ||||
pidx = fl->ifl_size - 1; | pidx = fl->ifl_size - 1; | ||||
else | else | ||||
pidx = fl->ifl_pidx - 1; | pidx = fl->ifl_pidx - 1; | ||||
if (sd_map) | |||||
bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, | bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, | ||||
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | ||||
ctx->isc_rxd_flush(ctx->ifc_softc, fl->ifl_rxq->ifr_id, fl->ifl_id, pidx); | ctx->isc_rxd_flush(ctx->ifc_softc, fl->ifl_rxq->ifr_id, fl->ifl_id, pidx); | ||||
fl->ifl_fragidx = frag_idx; | fl->ifl_fragidx = frag_idx; | ||||
} | } | ||||
static __inline void | static __inline void | ||||
__iflib_fl_refill_lt(if_ctx_t ctx, iflib_fl_t fl, int max) | __iflib_fl_refill_lt(if_ctx_t ctx, iflib_fl_t fl, int max) | ||||
{ | { | ||||
/* we avoid allowing pidx to catch up with cidx as it confuses ixl */ | /* we avoid allowing pidx to catch up with cidx as it confuses ixl */ | ||||
Show All 22 Lines | |||||
static void | static void | ||||
iflib_fl_bufs_free(iflib_fl_t fl) | iflib_fl_bufs_free(iflib_fl_t fl) | ||||
{ | { | ||||
iflib_dma_info_t idi = fl->ifl_ifdi; | iflib_dma_info_t idi = fl->ifl_ifdi; | ||||
uint32_t i; | uint32_t i; | ||||
for (i = 0; i < fl->ifl_size; i++) { | for (i = 0; i < fl->ifl_size; i++) { | ||||
struct mbuf **sd_m = &fl->ifl_sds.ifsd_m[i]; | struct mbuf **sd_m = &fl->ifl_sds.ifsd_m[i]; | ||||
uint8_t *sd_flags = &fl->ifl_sds.ifsd_flags[i]; | |||||
caddr_t *sd_cl = &fl->ifl_sds.ifsd_cl[i]; | caddr_t *sd_cl = &fl->ifl_sds.ifsd_cl[i]; | ||||
if (*sd_flags & RX_SW_DESC_INUSE) { | if (*sd_cl != NULL) { | ||||
if (fl->ifl_sds.ifsd_map != NULL) { | |||||
bus_dmamap_t sd_map = fl->ifl_sds.ifsd_map[i]; | bus_dmamap_t sd_map = fl->ifl_sds.ifsd_map[i]; | ||||
bus_dmamap_unload(fl->ifl_desc_tag, sd_map); | bus_dmamap_unload(fl->ifl_desc_tag, sd_map); | ||||
if (*sd_cl != NULL) | |||||
uma_zfree(fl->ifl_zone, *sd_cl); | |||||
// XXX: Should this get moved out? | // XXX: Should this get moved out? | ||||
if (iflib_in_detach(fl->ifl_rxq->ifr_ctx)) | if (iflib_in_detach(fl->ifl_rxq->ifr_ctx)) | ||||
bus_dmamap_destroy(fl->ifl_desc_tag, sd_map); | bus_dmamap_destroy(fl->ifl_desc_tag, sd_map); | ||||
} | |||||
if (*sd_m != NULL) { | if (*sd_m != NULL) { | ||||
m_init(*sd_m, M_NOWAIT, MT_DATA, 0); | m_init(*sd_m, M_NOWAIT, MT_DATA, 0); | ||||
uma_zfree(zone_mbuf, *sd_m); | uma_zfree(zone_mbuf, *sd_m); | ||||
} | } | ||||
if (*sd_cl != NULL) | |||||
uma_zfree(fl->ifl_zone, *sd_cl); | |||||
*sd_flags = 0; | |||||
} else { | } else { | ||||
MPASS(*sd_cl == NULL); | MPASS(*sd_cl == NULL); | ||||
MPASS(*sd_m == NULL); | MPASS(*sd_m == NULL); | ||||
} | } | ||||
#if MEMORY_LOGGING | #if MEMORY_LOGGING | ||||
fl->ifl_m_dequeued++; | fl->ifl_m_dequeued++; | ||||
fl->ifl_cl_dequeued++; | fl->ifl_cl_dequeued++; | ||||
#endif | #endif | ||||
*sd_cl = NULL; | *sd_cl = NULL; | ||||
*sd_m = NULL; | *sd_m = NULL; | ||||
} | } | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
for (i = 0; i < fl->ifl_size; i++) { | for (i = 0; i < fl->ifl_size; i++) { | ||||
MPASS(fl->ifl_sds.ifsd_flags[i] == 0); | |||||
MPASS(fl->ifl_sds.ifsd_cl[i] == NULL); | MPASS(fl->ifl_sds.ifsd_cl[i] == NULL); | ||||
MPASS(fl->ifl_sds.ifsd_m[i] == NULL); | MPASS(fl->ifl_sds.ifsd_m[i] == NULL); | ||||
} | } | ||||
#endif | #endif | ||||
/* | /* | ||||
* Reset free list values | * Reset free list values | ||||
*/ | */ | ||||
fl->ifl_credits = fl->ifl_cidx = fl->ifl_pidx = fl->ifl_gen = fl->ifl_fragidx = 0; | fl->ifl_credits = fl->ifl_cidx = fl->ifl_pidx = fl->ifl_gen = fl->ifl_fragidx = 0; | ||||
▲ Show 20 Lines • Show All 74 Lines • ▼ Show 20 Lines | if (rxq->ifr_fl != NULL) { | ||||
for (i = 0; i < rxq->ifr_nfl; i++) { | for (i = 0; i < rxq->ifr_nfl; i++) { | ||||
fl = &rxq->ifr_fl[i]; | fl = &rxq->ifr_fl[i]; | ||||
if (fl->ifl_desc_tag != NULL) { | if (fl->ifl_desc_tag != NULL) { | ||||
bus_dma_tag_destroy(fl->ifl_desc_tag); | bus_dma_tag_destroy(fl->ifl_desc_tag); | ||||
fl->ifl_desc_tag = NULL; | fl->ifl_desc_tag = NULL; | ||||
} | } | ||||
free(fl->ifl_sds.ifsd_m, M_IFLIB); | free(fl->ifl_sds.ifsd_m, M_IFLIB); | ||||
free(fl->ifl_sds.ifsd_cl, M_IFLIB); | free(fl->ifl_sds.ifsd_cl, M_IFLIB); | ||||
free(fl->ifl_sds.ifsd_ba, M_IFLIB); | |||||
/* XXX destroy maps first */ | /* XXX destroy maps first */ | ||||
free(fl->ifl_sds.ifsd_map, M_IFLIB); | free(fl->ifl_sds.ifsd_map, M_IFLIB); | ||||
fl->ifl_sds.ifsd_m = NULL; | fl->ifl_sds.ifsd_m = NULL; | ||||
fl->ifl_sds.ifsd_cl = NULL; | fl->ifl_sds.ifsd_cl = NULL; | ||||
fl->ifl_sds.ifsd_ba = NULL; | |||||
fl->ifl_sds.ifsd_map = NULL; | fl->ifl_sds.ifsd_map = NULL; | ||||
} | } | ||||
free(rxq->ifr_fl, M_IFLIB); | free(rxq->ifr_fl, M_IFLIB); | ||||
rxq->ifr_fl = NULL; | rxq->ifr_fl = NULL; | ||||
rxq->ifr_cq_gen = rxq->ifr_cq_cidx = rxq->ifr_cq_pidx = 0; | rxq->ifr_cq_gen = rxq->ifr_cq_cidx = rxq->ifr_cq_pidx = 0; | ||||
} | } | ||||
} | } | ||||
▲ Show 20 Lines • Show All 252 Lines • ▼ Show 20 Lines | rxd_frag_to_sd(iflib_rxq_t rxq, if_rxd_frag_t irf, int unload, if_rxsd_t sd) | ||||
sd->ifsd_m = &fl->ifl_sds.ifsd_m[cidx]; | sd->ifsd_m = &fl->ifl_sds.ifsd_m[cidx]; | ||||
sd->ifsd_cl = &fl->ifl_sds.ifsd_cl[cidx]; | sd->ifsd_cl = &fl->ifl_sds.ifsd_cl[cidx]; | ||||
fl->ifl_credits--; | fl->ifl_credits--; | ||||
#if MEMORY_LOGGING | #if MEMORY_LOGGING | ||||
fl->ifl_m_dequeued++; | fl->ifl_m_dequeued++; | ||||
#endif | #endif | ||||
if (rxq->ifr_ctx->ifc_flags & IFC_PREFETCH) | if (rxq->ifr_ctx->ifc_flags & IFC_PREFETCH) | ||||
prefetch_pkts(fl, cidx); | prefetch_pkts(fl, cidx); | ||||
if (fl->ifl_sds.ifsd_map != NULL) { | |||||
next = (cidx + CACHE_PTR_INCREMENT) & (fl->ifl_size-1); | next = (cidx + CACHE_PTR_INCREMENT) & (fl->ifl_size-1); | ||||
prefetch(&fl->ifl_sds.ifsd_map[next]); | prefetch(&fl->ifl_sds.ifsd_map[next]); | ||||
map = fl->ifl_sds.ifsd_map[cidx]; | map = fl->ifl_sds.ifsd_map[cidx]; | ||||
di = fl->ifl_ifdi; | di = fl->ifl_ifdi; | ||||
next = (cidx + CACHE_LINE_SIZE) & (fl->ifl_size-1); | next = (cidx + CACHE_LINE_SIZE) & (fl->ifl_size-1); | ||||
prefetch(&fl->ifl_sds.ifsd_flags[next]); | |||||
bus_dmamap_sync(di->idi_tag, di->idi_map, | bus_dmamap_sync(di->idi_tag, di->idi_map, | ||||
BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); | BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); | ||||
/* not valid assert if bxe really does SGE from non-contiguous elements */ | /* not valid assert if bxe really does SGE from non-contiguous elements */ | ||||
MPASS(fl->ifl_cidx == cidx); | MPASS(fl->ifl_cidx == cidx); | ||||
if (unload) | if (unload) | ||||
bus_dmamap_unload(fl->ifl_desc_tag, map); | bus_dmamap_unload(fl->ifl_desc_tag, map); | ||||
} | |||||
fl->ifl_cidx = (fl->ifl_cidx + 1) & (fl->ifl_size-1); | fl->ifl_cidx = (fl->ifl_cidx + 1) & (fl->ifl_size-1); | ||||
if (__predict_false(fl->ifl_cidx == 0)) | if (__predict_false(fl->ifl_cidx == 0)) | ||||
fl->ifl_gen = 0; | fl->ifl_gen = 0; | ||||
if (map != NULL) | |||||
bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, | bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, | ||||
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | ||||
bit_clear(fl->ifl_rx_bitmap, cidx); | bit_clear(fl->ifl_rx_bitmap, cidx); | ||||
} | } | ||||
static struct mbuf * | static struct mbuf * | ||||
assemble_segments(iflib_rxq_t rxq, if_rxd_info_t ri, if_rxsd_t sd) | assemble_segments(iflib_rxq_t rxq, if_rxd_info_t ri, if_rxsd_t sd) | ||||
{ | { | ||||
int i, padlen , flags; | int i, padlen , flags; | ||||
struct mbuf *m, *mh, *mt; | struct mbuf *m, *mh, *mt; | ||||
▲ Show 20 Lines • Show All 546 Lines • ▼ Show 20 Lines | |||||
/* | /* | ||||
* If dodgy hardware rejects the scatter gather chain we've handed it | * If dodgy hardware rejects the scatter gather chain we've handed it | ||||
* we'll need to remove the mbuf chain from ifsg_m[] before we can add the | * we'll need to remove the mbuf chain from ifsg_m[] before we can add the | ||||
* m_defrag'd mbufs | * m_defrag'd mbufs | ||||
*/ | */ | ||||
static __noinline struct mbuf * | static __noinline struct mbuf * | ||||
iflib_remove_mbuf(iflib_txq_t txq) | iflib_remove_mbuf(iflib_txq_t txq) | ||||
{ | { | ||||
int ntxd, i, pidx; | int ntxd, pidx; | ||||
struct mbuf *m, *mh, **ifsd_m; | struct mbuf *m, **ifsd_m; | ||||
bus_dmamap_t *ifsd_map; | |||||
pidx = txq->ift_pidx; | |||||
ifsd_m = txq->ift_sds.ifsd_m; | ifsd_m = txq->ift_sds.ifsd_m; | ||||
ntxd = txq->ift_size; | ntxd = txq->ift_size; | ||||
mh = m = ifsd_m[pidx]; | pidx = txq->ift_pidx & (ntxd - 1); | ||||
ifsd_m = txq->ift_sds.ifsd_m; | |||||
ifsd_map = txq->ift_sds.ifsd_map; | |||||
m = ifsd_m[pidx]; | |||||
ifsd_m[pidx] = NULL; | ifsd_m[pidx] = NULL; | ||||
bus_dmamap_unload(txq->ift_desc_tag, ifsd_map[pidx]); | |||||
#if MEMORY_LOGGING | #if MEMORY_LOGGING | ||||
txq->ift_dequeued++; | txq->ift_dequeued++; | ||||
#endif | #endif | ||||
i = 1; | return (m); | ||||
while (m) { | |||||
ifsd_m[(pidx + i) & (ntxd -1)] = NULL; | |||||
#if MEMORY_LOGGING | |||||
txq->ift_dequeued++; | |||||
#endif | |||||
m = m->m_next; | |||||
i++; | |||||
} | } | ||||
return (mh); | |||||
} | |||||
static int | |||||
iflib_busdma_load_mbuf_sg(iflib_txq_t txq, bus_dma_tag_t tag, bus_dmamap_t map, | |||||
struct mbuf **m0, bus_dma_segment_t *segs, int *nsegs, | |||||
int max_segs, int flags) | |||||
{ | |||||
if_ctx_t ctx; | |||||
if_shared_ctx_t sctx; | |||||
if_softc_ctx_t scctx; | |||||
int i, next, pidx, err, ntxd, count; | |||||
struct mbuf *m, *tmp, **ifsd_m; | |||||
m = *m0; | |||||
/* | |||||
* Please don't ever do this | |||||
*/ | |||||
MPASS(__predict_true(m->m_len > 0)); | |||||
ctx = txq->ift_ctx; | |||||
sctx = ctx->ifc_sctx; | |||||
scctx = &ctx->ifc_softc_ctx; | |||||
ifsd_m = txq->ift_sds.ifsd_m; | |||||
ntxd = txq->ift_size; | |||||
pidx = txq->ift_pidx; | |||||
if (map != NULL) { | |||||
uint8_t *ifsd_flags = txq->ift_sds.ifsd_flags; | |||||
err = bus_dmamap_load_mbuf_sg(tag, map, | |||||
*m0, segs, nsegs, BUS_DMA_NOWAIT); | |||||
if (err) | |||||
return (err); | |||||
ifsd_flags[pidx] |= TX_SW_DESC_MAPPED; | |||||
count = 0; | |||||
m = *m0; | |||||
do { | |||||
if (__predict_false(m->m_len <= 0)) { | |||||
tmp = m; | |||||
m = m->m_next; | |||||
tmp->m_next = NULL; | |||||
m_free(tmp); | |||||
continue; | |||||
} | |||||
m = m->m_next; | |||||
count++; | |||||
} while (m != NULL); | |||||
if (count > *nsegs) { | |||||
ifsd_m[pidx] = *m0; | |||||
ifsd_m[pidx]->m_flags |= M_TOOBIG; | |||||
return (0); | |||||
} | |||||
m = *m0; | |||||
count = 0; | |||||
do { | |||||
next = (pidx + count) & (ntxd-1); | |||||
MPASS(ifsd_m[next] == NULL); | |||||
ifsd_m[next] = m; | |||||
count++; | |||||
tmp = m; | |||||
m = m->m_next; | |||||
} while (m != NULL); | |||||
} else { | |||||
int buflen, sgsize, maxsegsz, max_sgsize; | |||||
vm_offset_t vaddr; | |||||
vm_paddr_t curaddr; | |||||
count = i = 0; | |||||
m = *m0; | |||||
if (m->m_pkthdr.csum_flags & CSUM_TSO) | |||||
maxsegsz = scctx->isc_tx_tso_segsize_max; | |||||
else | |||||
maxsegsz = sctx->isc_tx_maxsegsize; | |||||
do { | |||||
if (__predict_false(m->m_len <= 0)) { | |||||
tmp = m; | |||||
m = m->m_next; | |||||
tmp->m_next = NULL; | |||||
m_free(tmp); | |||||
continue; | |||||
} | |||||
buflen = m->m_len; | |||||
vaddr = (vm_offset_t)m->m_data; | |||||
/* | |||||
* see if we can't be smarter about physically | |||||
* contiguous mappings | |||||
*/ | |||||
next = (pidx + count) & (ntxd-1); | |||||
MPASS(ifsd_m[next] == NULL); | |||||
#if MEMORY_LOGGING | |||||
txq->ift_enqueued++; | |||||
#endif | |||||
ifsd_m[next] = m; | |||||
while (buflen > 0) { | |||||
if (i >= max_segs) | |||||
goto err; | |||||
max_sgsize = MIN(buflen, maxsegsz); | |||||
curaddr = pmap_kextract(vaddr); | |||||
sgsize = PAGE_SIZE - (curaddr & PAGE_MASK); | |||||
sgsize = MIN(sgsize, max_sgsize); | |||||
segs[i].ds_addr = curaddr; | |||||
segs[i].ds_len = sgsize; | |||||
vaddr += sgsize; | |||||
buflen -= sgsize; | |||||
i++; | |||||
} | |||||
count++; | |||||
tmp = m; | |||||
m = m->m_next; | |||||
} while (m != NULL); | |||||
*nsegs = i; | |||||
} | |||||
return (0); | |||||
err: | |||||
*m0 = iflib_remove_mbuf(txq); | |||||
return (EFBIG); | |||||
} | |||||
static inline caddr_t | static inline caddr_t | ||||
calc_next_txd(iflib_txq_t txq, int cidx, uint8_t qid) | calc_next_txd(iflib_txq_t txq, int cidx, uint8_t qid) | ||||
{ | { | ||||
qidx_t size; | qidx_t size; | ||||
int ntxd; | int ntxd; | ||||
caddr_t start, end, cur, next; | caddr_t start, end, cur, next; | ||||
ntxd = txq->ift_size; | ntxd = txq->ift_size; | ||||
▲ Show 20 Lines • Show All 54 Lines • ▼ Show 20 Lines | |||||
static int | static int | ||||
iflib_encap(iflib_txq_t txq, struct mbuf **m_headp) | iflib_encap(iflib_txq_t txq, struct mbuf **m_headp) | ||||
{ | { | ||||
if_ctx_t ctx; | if_ctx_t ctx; | ||||
if_shared_ctx_t sctx; | if_shared_ctx_t sctx; | ||||
if_softc_ctx_t scctx; | if_softc_ctx_t scctx; | ||||
bus_dma_segment_t *segs; | bus_dma_segment_t *segs; | ||||
struct mbuf *m_head; | struct mbuf *m_head, **ifsd_m; | ||||
void *next_txd; | void *next_txd; | ||||
bus_dmamap_t map; | bus_dmamap_t map; | ||||
struct if_pkt_info pi; | struct if_pkt_info pi; | ||||
int remap = 0; | int remap = 0; | ||||
int err, nsegs, ndesc, max_segs, pidx, cidx, next, ntxd; | int err, nsegs, ndesc, max_segs, pidx, cidx, next, ntxd; | ||||
bus_dma_tag_t desc_tag; | bus_dma_tag_t desc_tag; | ||||
ctx = txq->ift_ctx; | ctx = txq->ift_ctx; | ||||
Show All 13 Lines | if (ctx->ifc_flags & IFC_PREFETCH) { | ||||
next = (cidx + CACHE_PTR_INCREMENT) & (ntxd-1); | next = (cidx + CACHE_PTR_INCREMENT) & (ntxd-1); | ||||
if (!(ctx->ifc_flags & IFLIB_HAS_TXCQ)) { | if (!(ctx->ifc_flags & IFLIB_HAS_TXCQ)) { | ||||
next_txd = calc_next_txd(txq, cidx, 0); | next_txd = calc_next_txd(txq, cidx, 0); | ||||
prefetch(next_txd); | prefetch(next_txd); | ||||
} | } | ||||
/* prefetch the next cache line of mbuf pointers and flags */ | /* prefetch the next cache line of mbuf pointers and flags */ | ||||
prefetch(&txq->ift_sds.ifsd_m[next]); | prefetch(&txq->ift_sds.ifsd_m[next]); | ||||
if (txq->ift_sds.ifsd_map != NULL) { | |||||
prefetch(&txq->ift_sds.ifsd_map[next]); | prefetch(&txq->ift_sds.ifsd_map[next]); | ||||
next = (cidx + CACHE_LINE_SIZE) & (ntxd-1); | next = (cidx + CACHE_LINE_SIZE) & (ntxd-1); | ||||
prefetch(&txq->ift_sds.ifsd_flags[next]); | |||||
} | } | ||||
} else if (txq->ift_sds.ifsd_map != NULL) | |||||
map = txq->ift_sds.ifsd_map[pidx]; | map = txq->ift_sds.ifsd_map[pidx]; | ||||
ifsd_m = txq->ift_sds.ifsd_m; | |||||
if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { | if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { | ||||
desc_tag = txq->ift_tso_desc_tag; | desc_tag = txq->ift_tso_desc_tag; | ||||
max_segs = scctx->isc_tx_tso_segments_max; | max_segs = scctx->isc_tx_tso_segments_max; | ||||
MPASS(desc_tag != NULL); | MPASS(desc_tag != NULL); | ||||
MPASS(max_segs > 0); | MPASS(max_segs > 0); | ||||
} else { | } else { | ||||
desc_tag = txq->ift_desc_tag; | desc_tag = txq->ift_desc_tag; | ||||
Show All 22 Lines | if (__predict_true((pi.ipi_csum_flags | pi.ipi_vtag))) { | ||||
if (__predict_false((err = iflib_parse_header(txq, &pi, m_headp)) != 0)) { | if (__predict_false((err = iflib_parse_header(txq, &pi, m_headp)) != 0)) { | ||||
DBG_COUNTER_INC(encap_txd_encap_fail); | DBG_COUNTER_INC(encap_txd_encap_fail); | ||||
return (err); | return (err); | ||||
} | } | ||||
m_head = *m_headp; | m_head = *m_headp; | ||||
} | } | ||||
retry: | retry: | ||||
err = iflib_busdma_load_mbuf_sg(txq, desc_tag, map, m_headp, segs, &nsegs, max_segs, BUS_DMA_NOWAIT); | err = bus_dmamap_load_mbuf_sg(desc_tag, map, m_head, segs, &nsegs, | ||||
BUS_DMA_NOWAIT); | |||||
defrag: | defrag: | ||||
if (__predict_false(err)) { | if (__predict_false(err)) { | ||||
switch (err) { | switch (err) { | ||||
case EFBIG: | case EFBIG: | ||||
/* try collapse once and defrag once */ | /* try collapse once and defrag once */ | ||||
if (remap == 0) { | if (remap == 0) { | ||||
m_head = m_collapse(*m_headp, M_NOWAIT, max_segs); | m_head = m_collapse(*m_headp, M_NOWAIT, max_segs); | ||||
/* try defrag if collapsing fails */ | /* try defrag if collapsing fails */ | ||||
Show All 20 Lines | default: | ||||
*m_headp = NULL; | *m_headp = NULL; | ||||
break; | break; | ||||
} | } | ||||
txq->ift_map_failed++; | txq->ift_map_failed++; | ||||
DBG_COUNTER_INC(encap_load_mbuf_fail); | DBG_COUNTER_INC(encap_load_mbuf_fail); | ||||
DBG_COUNTER_INC(encap_txd_encap_fail); | DBG_COUNTER_INC(encap_txd_encap_fail); | ||||
return (err); | return (err); | ||||
} | } | ||||
ifsd_m[pidx] = m_head; | |||||
/* | /* | ||||
* XXX assumes a 1 to 1 relationship between segments and | * XXX assumes a 1 to 1 relationship between segments and | ||||
* descriptors - this does not hold true on all drivers, e.g. | * descriptors - this does not hold true on all drivers, e.g. | ||||
* cxgb | * cxgb | ||||
*/ | */ | ||||
if (__predict_false(nsegs + 2 > TXQ_AVAIL(txq))) { | if (__predict_false(nsegs + 2 > TXQ_AVAIL(txq))) { | ||||
txq->ift_no_desc_avail++; | txq->ift_no_desc_avail++; | ||||
if (map != NULL) | |||||
bus_dmamap_unload(desc_tag, map); | bus_dmamap_unload(desc_tag, map); | ||||
DBG_COUNTER_INC(encap_txq_avail_fail); | DBG_COUNTER_INC(encap_txq_avail_fail); | ||||
DBG_COUNTER_INC(encap_txd_encap_fail); | DBG_COUNTER_INC(encap_txd_encap_fail); | ||||
if ((txq->ift_task.gt_task.ta_flags & TASK_ENQUEUED) == 0) | if ((txq->ift_task.gt_task.ta_flags & TASK_ENQUEUED) == 0) | ||||
GROUPTASK_ENQUEUE(&txq->ift_task); | GROUPTASK_ENQUEUE(&txq->ift_task); | ||||
return (ENOBUFS); | return (ENOBUFS); | ||||
} | } | ||||
/* | /* | ||||
* On Intel cards we can greatly reduce the number of TX interrupts | * On Intel cards we can greatly reduce the number of TX interrupts | ||||
Show All 10 Lines | defrag: | ||||
pi.ipi_segs = segs; | pi.ipi_segs = segs; | ||||
pi.ipi_nsegs = nsegs; | pi.ipi_nsegs = nsegs; | ||||
MPASS(pidx >= 0 && pidx < txq->ift_size); | MPASS(pidx >= 0 && pidx < txq->ift_size); | ||||
#ifdef PKT_DEBUG | #ifdef PKT_DEBUG | ||||
print_pkt(&pi); | print_pkt(&pi); | ||||
#endif | #endif | ||||
if (map != NULL) | |||||
bus_dmamap_sync(desc_tag, map, BUS_DMASYNC_PREWRITE); | bus_dmamap_sync(desc_tag, map, BUS_DMASYNC_PREWRITE); | ||||
if ((err = ctx->isc_txd_encap(ctx->ifc_softc, &pi)) == 0) { | if ((err = ctx->isc_txd_encap(ctx->ifc_softc, &pi)) == 0) { | ||||
if (map != NULL) | |||||
bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map, | bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map, | ||||
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | ||||
DBG_COUNTER_INC(tx_encap); | DBG_COUNTER_INC(tx_encap); | ||||
MPASS(pi.ipi_new_pidx < txq->ift_size); | MPASS(pi.ipi_new_pidx < txq->ift_size); | ||||
ndesc = pi.ipi_new_pidx - pi.ipi_pidx; | ndesc = pi.ipi_new_pidx - pi.ipi_pidx; | ||||
if (pi.ipi_new_pidx < pi.ipi_pidx) { | if (pi.ipi_new_pidx < pi.ipi_pidx) { | ||||
ndesc += txq->ift_size; | ndesc += txq->ift_size; | ||||
txq->ift_gen = 1; | txq->ift_gen = 1; | ||||
} | } | ||||
Show All 37 Lines | defrag_failed: | ||||
*m_headp = NULL; | *m_headp = NULL; | ||||
DBG_COUNTER_INC(encap_txd_encap_fail); | DBG_COUNTER_INC(encap_txd_encap_fail); | ||||
return (ENOMEM); | return (ENOMEM); | ||||
} | } | ||||
static void | static void | ||||
iflib_tx_desc_free(iflib_txq_t txq, int n) | iflib_tx_desc_free(iflib_txq_t txq, int n) | ||||
{ | { | ||||
int hasmap; | |||||
uint32_t qsize, cidx, mask, gen; | uint32_t qsize, cidx, mask, gen; | ||||
struct mbuf *m, **ifsd_m; | struct mbuf *m, **ifsd_m; | ||||
uint8_t *ifsd_flags; | |||||
bus_dmamap_t *ifsd_map; | bus_dmamap_t *ifsd_map; | ||||
bool do_prefetch; | bool do_prefetch; | ||||
cidx = txq->ift_cidx; | cidx = txq->ift_cidx; | ||||
gen = txq->ift_gen; | gen = txq->ift_gen; | ||||
qsize = txq->ift_size; | qsize = txq->ift_size; | ||||
mask = qsize-1; | mask = qsize-1; | ||||
hasmap = txq->ift_sds.ifsd_map != NULL; | |||||
ifsd_flags = txq->ift_sds.ifsd_flags; | |||||
ifsd_m = txq->ift_sds.ifsd_m; | ifsd_m = txq->ift_sds.ifsd_m; | ||||
ifsd_map = txq->ift_sds.ifsd_map; | ifsd_map = txq->ift_sds.ifsd_map; | ||||
do_prefetch = (txq->ift_ctx->ifc_flags & IFC_PREFETCH); | do_prefetch = (txq->ift_ctx->ifc_flags & IFC_PREFETCH); | ||||
while (n-- > 0) { | while (n-- > 0) { | ||||
if (do_prefetch) { | if (do_prefetch) { | ||||
prefetch(ifsd_m[(cidx + 3) & mask]); | prefetch(ifsd_m[(cidx + 3) & mask]); | ||||
prefetch(ifsd_m[(cidx + 4) & mask]); | prefetch(ifsd_m[(cidx + 4) & mask]); | ||||
} | } | ||||
if (ifsd_m[cidx] != NULL) { | if ((m = ifsd_m[cidx]) != NULL) { | ||||
prefetch(&ifsd_m[(cidx + CACHE_PTR_INCREMENT) & mask]); | prefetch(&ifsd_m[(cidx + CACHE_PTR_INCREMENT) & mask]); | ||||
prefetch(&ifsd_flags[(cidx + CACHE_PTR_INCREMENT) & mask]); | |||||
if (hasmap && (ifsd_flags[cidx] & TX_SW_DESC_MAPPED)) { | |||||
/* | |||||
* does it matter if it's not the TSO tag? If so we'll | |||||
* have to add the type to flags | |||||
*/ | |||||
bus_dmamap_unload(txq->ift_desc_tag, ifsd_map[cidx]); | bus_dmamap_unload(txq->ift_desc_tag, ifsd_map[cidx]); | ||||
ifsd_flags[cidx] &= ~TX_SW_DESC_MAPPED; | |||||
} | |||||
if ((m = ifsd_m[cidx]) != NULL) { | |||||
/* XXX we don't support any drivers that batch packets yet */ | /* XXX we don't support any drivers that batch packets yet */ | ||||
MPASS(m->m_nextpkt == NULL); | MPASS(m->m_nextpkt == NULL); | ||||
/* if the number of clusters exceeds the number of segments | |||||
* there won't be space on the ring to save a pointer to each | |||||
* cluster so we simply free the list here | |||||
*/ | |||||
if (m->m_flags & M_TOOBIG) { | |||||
m_freem(m); | m_freem(m); | ||||
} else { | |||||
m_free(m); | |||||
} | |||||
ifsd_m[cidx] = NULL; | ifsd_m[cidx] = NULL; | ||||
#if MEMORY_LOGGING | #if MEMORY_LOGGING | ||||
txq->ift_dequeued++; | txq->ift_dequeued++; | ||||
#endif | #endif | ||||
DBG_COUNTER_INC(tx_frees); | DBG_COUNTER_INC(tx_frees); | ||||
} | } | ||||
} | |||||
if (__predict_false(++cidx == qsize)) { | if (__predict_false(++cidx == qsize)) { | ||||
cidx = 0; | cidx = 0; | ||||
gen = 0; | gen = 0; | ||||
} | } | ||||
} | } | ||||
txq->ift_cidx = cidx; | txq->ift_cidx = cidx; | ||||
txq->ift_gen = gen; | txq->ift_gen = gen; | ||||
} | } | ||||
▲ Show 20 Lines • Show All 981 Lines • ▼ Show 20 Lines | #endif | ||||
if_setcapabilities(ifp, scctx->isc_capabilities | IFCAP_HWSTATS); | if_setcapabilities(ifp, scctx->isc_capabilities | IFCAP_HWSTATS); | ||||
if_setcapenable(ifp, scctx->isc_capenable | IFCAP_HWSTATS); | if_setcapenable(ifp, scctx->isc_capenable | IFCAP_HWSTATS); | ||||
if (scctx->isc_ntxqsets == 0 || (scctx->isc_ntxqsets_max && scctx->isc_ntxqsets_max < scctx->isc_ntxqsets)) | if (scctx->isc_ntxqsets == 0 || (scctx->isc_ntxqsets_max && scctx->isc_ntxqsets_max < scctx->isc_ntxqsets)) | ||||
scctx->isc_ntxqsets = scctx->isc_ntxqsets_max; | scctx->isc_ntxqsets = scctx->isc_ntxqsets_max; | ||||
if (scctx->isc_nrxqsets == 0 || (scctx->isc_nrxqsets_max && scctx->isc_nrxqsets_max < scctx->isc_nrxqsets)) | if (scctx->isc_nrxqsets == 0 || (scctx->isc_nrxqsets_max && scctx->isc_nrxqsets_max < scctx->isc_nrxqsets)) | ||||
scctx->isc_nrxqsets = scctx->isc_nrxqsets_max; | scctx->isc_nrxqsets = scctx->isc_nrxqsets_max; | ||||
#ifdef ACPI_DMAR | |||||
if (dmar_get_dma_tag(device_get_parent(dev), dev) != NULL) | |||||
ctx->ifc_flags |= IFC_DMAR; | |||||
#elif !(defined(__i386__) || defined(__amd64__)) | |||||
/* set unconditionally for !x86 */ | |||||
ctx->ifc_flags |= IFC_DMAR; | |||||
#endif | |||||
main_txq = (sctx->isc_flags & IFLIB_HAS_TXCQ) ? 1 : 0; | main_txq = (sctx->isc_flags & IFLIB_HAS_TXCQ) ? 1 : 0; | ||||
main_rxq = (sctx->isc_flags & IFLIB_HAS_RXCQ) ? 1 : 0; | main_rxq = (sctx->isc_flags & IFLIB_HAS_RXCQ) ? 1 : 0; | ||||
/* XXX change for per-queue sizes */ | /* XXX change for per-queue sizes */ | ||||
device_printf(dev, "using %d tx descriptors and %d rx descriptors\n", | device_printf(dev, "using %d tx descriptors and %d rx descriptors\n", | ||||
scctx->isc_ntxd[main_txq], scctx->isc_nrxd[main_rxq]); | scctx->isc_ntxd[main_txq], scctx->isc_nrxd[main_rxq]); | ||||
for (i = 0; i < sctx->isc_nrxqs; i++) { | for (i = 0; i < sctx->isc_nrxqs; i++) { | ||||
▲ Show 20 Lines • Show All 2,067 Lines • Show Last 20 Lines |