Changeset View
Changeset View
Standalone View
Standalone View
head/sys/net/iflib.c
Show First 20 Lines • Show All 347 Lines • ▼ Show 20 Lines | #endif | ||||
uint16_t ift_id; | uint16_t ift_id; | ||||
struct callout ift_timer; | struct callout ift_timer; | ||||
if_txsd_vec_t ift_sds; | if_txsd_vec_t ift_sds; | ||||
uint8_t ift_qstatus; | uint8_t ift_qstatus; | ||||
uint8_t ift_closed; | uint8_t ift_closed; | ||||
uint8_t ift_update_freq; | uint8_t ift_update_freq; | ||||
struct iflib_filter_info ift_filter_info; | struct iflib_filter_info ift_filter_info; | ||||
bus_dma_tag_t ift_desc_tag; | bus_dma_tag_t ift_buf_tag; | ||||
bus_dma_tag_t ift_tso_desc_tag; | bus_dma_tag_t ift_tso_buf_tag; | ||||
iflib_dma_info_t ift_ifdi; | iflib_dma_info_t ift_ifdi; | ||||
#define MTX_NAME_LEN 16 | #define MTX_NAME_LEN 16 | ||||
char ift_mtx_name[MTX_NAME_LEN]; | char ift_mtx_name[MTX_NAME_LEN]; | ||||
char ift_db_mtx_name[MTX_NAME_LEN]; | char ift_db_mtx_name[MTX_NAME_LEN]; | ||||
bus_dma_segment_t ift_segs[IFLIB_MAX_TX_SEGS] __aligned(CACHE_LINE_SIZE); | bus_dma_segment_t ift_segs[IFLIB_MAX_TX_SEGS] __aligned(CACHE_LINE_SIZE); | ||||
#ifdef IFLIB_DIAGNOSTICS | #ifdef IFLIB_DIAGNOSTICS | ||||
uint64_t ift_cpu_exec_count[256]; | uint64_t ift_cpu_exec_count[256]; | ||||
#endif | #endif | ||||
Show All 18 Lines | #endif | ||||
/* constant */ | /* constant */ | ||||
qidx_t ifl_size; | qidx_t ifl_size; | ||||
uint16_t ifl_buf_size; | uint16_t ifl_buf_size; | ||||
uint16_t ifl_cltype; | uint16_t ifl_cltype; | ||||
uma_zone_t ifl_zone; | uma_zone_t ifl_zone; | ||||
iflib_rxsd_array_t ifl_sds; | iflib_rxsd_array_t ifl_sds; | ||||
iflib_rxq_t ifl_rxq; | iflib_rxq_t ifl_rxq; | ||||
uint8_t ifl_id; | uint8_t ifl_id; | ||||
bus_dma_tag_t ifl_desc_tag; | bus_dma_tag_t ifl_buf_tag; | ||||
iflib_dma_info_t ifl_ifdi; | iflib_dma_info_t ifl_ifdi; | ||||
uint64_t ifl_bus_addrs[IFLIB_MAX_RX_REFRESH] __aligned(CACHE_LINE_SIZE); | uint64_t ifl_bus_addrs[IFLIB_MAX_RX_REFRESH] __aligned(CACHE_LINE_SIZE); | ||||
caddr_t ifl_vm_addrs[IFLIB_MAX_RX_REFRESH]; | caddr_t ifl_vm_addrs[IFLIB_MAX_RX_REFRESH]; | ||||
qidx_t ifl_rxd_idxs[IFLIB_MAX_RX_REFRESH]; | qidx_t ifl_rxd_idxs[IFLIB_MAX_RX_REFRESH]; | ||||
} __aligned(CACHE_LINE_SIZE); | } __aligned(CACHE_LINE_SIZE); | ||||
static inline qidx_t | static inline qidx_t | ||||
get_inuse(int size, qidx_t cidx, qidx_t pidx, uint8_t gen) | get_inuse(int size, qidx_t cidx, qidx_t pidx, uint8_t gen) | ||||
▲ Show 20 Lines • Show All 516 Lines • ▼ Show 20 Lines | iflib_netmap_txsync(struct netmap_kring *kring, int flags) | ||||
* interrupts on every tx packet are expensive so request | * interrupts on every tx packet are expensive so request | ||||
* them every half ring, or where NS_REPORT is set | * them every half ring, or where NS_REPORT is set | ||||
*/ | */ | ||||
u_int report_frequency = kring->nkr_num_slots >> 1; | u_int report_frequency = kring->nkr_num_slots >> 1; | ||||
/* device-specific */ | /* device-specific */ | ||||
if_ctx_t ctx = ifp->if_softc; | if_ctx_t ctx = ifp->if_softc; | ||||
iflib_txq_t txq = &ctx->ifc_txqs[kring->ring_id]; | iflib_txq_t txq = &ctx->ifc_txqs[kring->ring_id]; | ||||
bus_dmamap_sync(txq->ift_desc_tag, txq->ift_ifdi->idi_map, | bus_dmamap_sync(txq->ift_buf_tag, txq->ift_ifdi->idi_map, | ||||
BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); | BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); | ||||
/* | /* | ||||
* First part: process new packets to send. | * First part: process new packets to send. | ||||
* nm_i is the current index in the netmap kring, | * nm_i is the current index in the netmap kring, | ||||
* nic_i is the corresponding index in the NIC ring. | * nic_i is the corresponding index in the NIC ring. | ||||
* | * | ||||
* If we have packets to send (nm_i != head) | * If we have packets to send (nm_i != head) | ||||
* iterate over the netmap ring, fetch length and update | * iterate over the netmap ring, fetch length and update | ||||
* the corresponding slot in the NIC ring. Some drivers also | * the corresponding slot in the NIC ring. Some drivers also | ||||
▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Lines | for (n = 0; nm_i != head; n++) { | ||||
__builtin_prefetch(&txq->ift_sds.ifsd_m[nic_i + 1]); | __builtin_prefetch(&txq->ift_sds.ifsd_m[nic_i + 1]); | ||||
if (txq->ift_sds.ifsd_map) { | if (txq->ift_sds.ifsd_map) { | ||||
__builtin_prefetch(&txq->ift_sds.ifsd_map[nic_i + 1]); | __builtin_prefetch(&txq->ift_sds.ifsd_map[nic_i + 1]); | ||||
NM_CHECK_ADDR_LEN(na, addr, len); | NM_CHECK_ADDR_LEN(na, addr, len); | ||||
if (slot->flags & NS_BUF_CHANGED) { | if (slot->flags & NS_BUF_CHANGED) { | ||||
/* buffer has changed, reload map */ | /* buffer has changed, reload map */ | ||||
netmap_reload_map(na, txq->ift_desc_tag, txq->ift_sds.ifsd_map[nic_i], addr); | netmap_reload_map(na, txq->ift_buf_tag, | ||||
txq->ift_sds.ifsd_map[nic_i], addr); | |||||
} | } | ||||
/* make sure changes to the buffer are synced */ | /* make sure changes to the buffer are synced */ | ||||
bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_sds.ifsd_map[nic_i], | bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_sds.ifsd_map[nic_i], | ||||
BUS_DMASYNC_PREWRITE); | BUS_DMASYNC_PREWRITE); | ||||
} | } | ||||
slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED); | slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED); | ||||
nm_i = nm_next(nm_i, lim); | nm_i = nm_next(nm_i, lim); | ||||
nic_i = nm_next(nic_i, lim); | nic_i = nm_next(nic_i, lim); | ||||
} | } | ||||
kring->nr_hwcur = nm_i; | kring->nr_hwcur = nm_i; | ||||
/* synchronize the NIC ring */ | /* synchronize the NIC ring */ | ||||
bus_dmamap_sync(txq->ift_desc_tag, txq->ift_ifdi->idi_map, | bus_dmamap_sync(txq->ift_buf_tag, txq->ift_ifdi->idi_map, | ||||
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | ||||
/* (re)start the tx unit up to slot nic_i (excluded) */ | /* (re)start the tx unit up to slot nic_i (excluded) */ | ||||
ctx->isc_txd_flush(ctx->ifc_softc, txq->ift_id, nic_i); | ctx->isc_txd_flush(ctx->ifc_softc, txq->ift_id, nic_i); | ||||
} | } | ||||
/* | /* | ||||
* Second part: reclaim buffers for completed transmissions. | * Second part: reclaim buffers for completed transmissions. | ||||
▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Lines | iflib_netmap_rxsync(struct netmap_kring *kring, int flags) | ||||
iflib_fl_t fl = rxq->ifr_fl; | iflib_fl_t fl = rxq->ifr_fl; | ||||
if (head > lim) | if (head > lim) | ||||
return netmap_ring_reinit(kring); | return netmap_ring_reinit(kring); | ||||
/* XXX check sync modes */ | /* XXX check sync modes */ | ||||
for (i = 0, fl = rxq->ifr_fl; i < rxq->ifr_nfl; i++, fl++) { | for (i = 0, fl = rxq->ifr_fl; i < rxq->ifr_nfl; i++, fl++) { | ||||
if (fl->ifl_sds.ifsd_map == NULL) | if (fl->ifl_sds.ifsd_map == NULL) | ||||
continue; | continue; | ||||
bus_dmamap_sync(rxq->ifr_fl[i].ifl_desc_tag, fl->ifl_ifdi->idi_map, | bus_dmamap_sync(rxq->ifr_fl[i].ifl_buf_tag, | ||||
fl->ifl_ifdi->idi_map, | |||||
BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); | BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); | ||||
} | } | ||||
/* | /* | ||||
* First part: import newly received packets. | * First part: import newly received packets. | ||||
* | * | ||||
* nm_i is the index of the next free slot in the netmap ring, | * nm_i is the index of the next free slot in the netmap ring, | ||||
* nic_i is the index of the next received packet in the NIC ring, | * nic_i is the index of the next received packet in the NIC ring, | ||||
* and they may differ in case if_init() has been called while | * and they may differ in case if_init() has been called while | ||||
* in netmap mode. For the receive ring we have | * in netmap mode. For the receive ring we have | ||||
▲ Show 20 Lines • Show All 109 Lines • ▼ Show 20 Lines | for (int i = 0; i < ctx->ifc_softc_ctx.isc_ntxd[0]; i++) { | ||||
/* | /* | ||||
* In netmap mode, set the map for the packet buffer. | * In netmap mode, set the map for the packet buffer. | ||||
* NOTE: Some drivers (not this one) also need to set | * NOTE: Some drivers (not this one) also need to set | ||||
* the physical buffer address in the NIC ring. | * the physical buffer address in the NIC ring. | ||||
* netmap_idx_n2k() maps a nic index, i, into the corresponding | * netmap_idx_n2k() maps a nic index, i, into the corresponding | ||||
* netmap slot index, si | * netmap slot index, si | ||||
*/ | */ | ||||
int si = netmap_idx_n2k(na->tx_rings[txq->ift_id], i); | int si = netmap_idx_n2k(na->tx_rings[txq->ift_id], i); | ||||
netmap_load_map(na, txq->ift_desc_tag, txq->ift_sds.ifsd_map[i], NMB(na, slot + si)); | netmap_load_map(na, txq->ift_buf_tag, txq->ift_sds.ifsd_map[i], | ||||
NMB(na, slot + si)); | |||||
} | } | ||||
} | } | ||||
static void | static void | ||||
iflib_netmap_rxq_init(if_ctx_t ctx, iflib_rxq_t rxq) | iflib_netmap_rxq_init(if_ctx_t ctx, iflib_rxq_t rxq) | ||||
{ | { | ||||
struct netmap_adapter *na = NA(ctx->ifc_ifp); | struct netmap_adapter *na = NA(ctx->ifc_ifp); | ||||
struct netmap_kring *kring = na->rx_rings[rxq->ifr_id]; | struct netmap_kring *kring = na->rx_rings[rxq->ifr_id]; | ||||
▲ Show 20 Lines • Show All 360 Lines • ▼ Show 20 Lines | _iflib_irq_alloc(if_ctx_t ctx, if_irq_t irq, int rid, | ||||
irq->ii_tag = tag; | irq->ii_tag = tag; | ||||
return (0); | return (0); | ||||
} | } | ||||
/********************************************************************* | /********************************************************************* | ||||
* | * | ||||
* Allocate memory for tx_buffer structures. The tx_buffer stores all | * Allocate DMA resources for TX buffers as well as memory for the TX | ||||
* the information needed to transmit a packet on the wire. This is | * mbuf map. TX DMA maps (non-TSO/TSO) and TX mbuf map are kept in a | ||||
* called only once at attach, setup is done every reset. | * iflib_sw_tx_desc_array structure, storing all the information that | ||||
* is needed to transmit a packet on the wire. This is called only | |||||
* once at attach, setup is done every reset. | |||||
* | * | ||||
**********************************************************************/ | **********************************************************************/ | ||||
static int | static int | ||||
iflib_txsd_alloc(iflib_txq_t txq) | iflib_txsd_alloc(iflib_txq_t txq) | ||||
{ | { | ||||
if_ctx_t ctx = txq->ift_ctx; | if_ctx_t ctx = txq->ift_ctx; | ||||
if_shared_ctx_t sctx = ctx->ifc_sctx; | if_shared_ctx_t sctx = ctx->ifc_sctx; | ||||
if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; | if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; | ||||
device_t dev = ctx->ifc_dev; | device_t dev = ctx->ifc_dev; | ||||
bus_size_t tsomaxsize; | bus_size_t tsomaxsize; | ||||
Show All 9 Lines | iflib_txsd_alloc(iflib_txq_t txq) | ||||
MPASS(scctx->isc_ntxd[txq->ift_br_offset] > 0); | MPASS(scctx->isc_ntxd[txq->ift_br_offset] > 0); | ||||
MPASS(nsegments > 0); | MPASS(nsegments > 0); | ||||
if (if_getcapabilities(ctx->ifc_ifp) & IFCAP_TSO) { | if (if_getcapabilities(ctx->ifc_ifp) & IFCAP_TSO) { | ||||
MPASS(ntsosegments > 0); | MPASS(ntsosegments > 0); | ||||
MPASS(sctx->isc_tso_maxsize >= tsomaxsize); | MPASS(sctx->isc_tso_maxsize >= tsomaxsize); | ||||
} | } | ||||
/* | /* | ||||
* Setup DMA descriptor areas. | * Set up DMA tags for TX buffers. | ||||
*/ | */ | ||||
if ((err = bus_dma_tag_create(bus_get_dma_tag(dev), | if ((err = bus_dma_tag_create(bus_get_dma_tag(dev), | ||||
1, 0, /* alignment, bounds */ | 1, 0, /* alignment, bounds */ | ||||
BUS_SPACE_MAXADDR, /* lowaddr */ | BUS_SPACE_MAXADDR, /* lowaddr */ | ||||
BUS_SPACE_MAXADDR, /* highaddr */ | BUS_SPACE_MAXADDR, /* highaddr */ | ||||
NULL, NULL, /* filter, filterarg */ | NULL, NULL, /* filter, filterarg */ | ||||
sctx->isc_tx_maxsize, /* maxsize */ | sctx->isc_tx_maxsize, /* maxsize */ | ||||
nsegments, /* nsegments */ | nsegments, /* nsegments */ | ||||
sctx->isc_tx_maxsegsize, /* maxsegsize */ | sctx->isc_tx_maxsegsize, /* maxsegsize */ | ||||
0, /* flags */ | 0, /* flags */ | ||||
NULL, /* lockfunc */ | NULL, /* lockfunc */ | ||||
NULL, /* lockfuncarg */ | NULL, /* lockfuncarg */ | ||||
&txq->ift_desc_tag))) { | &txq->ift_buf_tag))) { | ||||
device_printf(dev,"Unable to allocate TX DMA tag: %d\n", err); | device_printf(dev,"Unable to allocate TX DMA tag: %d\n", err); | ||||
device_printf(dev,"maxsize: %ju nsegments: %d maxsegsize: %ju\n", | device_printf(dev,"maxsize: %ju nsegments: %d maxsegsize: %ju\n", | ||||
(uintmax_t)sctx->isc_tx_maxsize, nsegments, (uintmax_t)sctx->isc_tx_maxsegsize); | (uintmax_t)sctx->isc_tx_maxsize, nsegments, (uintmax_t)sctx->isc_tx_maxsegsize); | ||||
goto fail; | goto fail; | ||||
} | } | ||||
tso = (if_getcapabilities(ctx->ifc_ifp) & IFCAP_TSO) != 0; | tso = (if_getcapabilities(ctx->ifc_ifp) & IFCAP_TSO) != 0; | ||||
if (tso && (err = bus_dma_tag_create(bus_get_dma_tag(dev), | if (tso && (err = bus_dma_tag_create(bus_get_dma_tag(dev), | ||||
1, 0, /* alignment, bounds */ | 1, 0, /* alignment, bounds */ | ||||
BUS_SPACE_MAXADDR, /* lowaddr */ | BUS_SPACE_MAXADDR, /* lowaddr */ | ||||
BUS_SPACE_MAXADDR, /* highaddr */ | BUS_SPACE_MAXADDR, /* highaddr */ | ||||
NULL, NULL, /* filter, filterarg */ | NULL, NULL, /* filter, filterarg */ | ||||
tsomaxsize, /* maxsize */ | tsomaxsize, /* maxsize */ | ||||
ntsosegments, /* nsegments */ | ntsosegments, /* nsegments */ | ||||
sctx->isc_tso_maxsegsize,/* maxsegsize */ | sctx->isc_tso_maxsegsize,/* maxsegsize */ | ||||
0, /* flags */ | 0, /* flags */ | ||||
NULL, /* lockfunc */ | NULL, /* lockfunc */ | ||||
NULL, /* lockfuncarg */ | NULL, /* lockfuncarg */ | ||||
&txq->ift_tso_desc_tag))) { | &txq->ift_tso_buf_tag))) { | ||||
device_printf(dev,"Unable to allocate TX TSO DMA tag: %d\n", err); | device_printf(dev, "Unable to allocate TSO TX DMA tag: %d\n", | ||||
err); | |||||
goto fail; | goto fail; | ||||
} | } | ||||
/* Allocate memory for the TX mbuf map. */ | |||||
if (!(txq->ift_sds.ifsd_m = | if (!(txq->ift_sds.ifsd_m = | ||||
(struct mbuf **) malloc(sizeof(struct mbuf *) * | (struct mbuf **) malloc(sizeof(struct mbuf *) * | ||||
scctx->isc_ntxd[txq->ift_br_offset], M_IFLIB, M_NOWAIT | M_ZERO))) { | scctx->isc_ntxd[txq->ift_br_offset], M_IFLIB, M_NOWAIT | M_ZERO))) { | ||||
device_printf(dev, "Unable to allocate tx_buffer memory\n"); | device_printf(dev, "Unable to allocate TX mbuf map memory\n"); | ||||
err = ENOMEM; | err = ENOMEM; | ||||
goto fail; | goto fail; | ||||
} | } | ||||
/* Create the descriptor buffer dma maps */ | /* | ||||
* Create the DMA maps for TX buffers. | |||||
*/ | |||||
if ((txq->ift_sds.ifsd_map = (bus_dmamap_t *)malloc( | if ((txq->ift_sds.ifsd_map = (bus_dmamap_t *)malloc( | ||||
sizeof(bus_dmamap_t) * scctx->isc_ntxd[txq->ift_br_offset], | sizeof(bus_dmamap_t) * scctx->isc_ntxd[txq->ift_br_offset], | ||||
M_IFLIB, M_NOWAIT | M_ZERO)) == NULL) { | M_IFLIB, M_NOWAIT | M_ZERO)) == NULL) { | ||||
device_printf(dev, "Unable to allocate tx_buffer map memory\n"); | device_printf(dev, | ||||
"Unable to allocate TX buffer DMA map memory\n"); | |||||
err = ENOMEM; | err = ENOMEM; | ||||
goto fail; | goto fail; | ||||
} | } | ||||
if (tso && (txq->ift_sds.ifsd_tso_map = (bus_dmamap_t *)malloc( | if (tso && (txq->ift_sds.ifsd_tso_map = (bus_dmamap_t *)malloc( | ||||
sizeof(bus_dmamap_t) * scctx->isc_ntxd[txq->ift_br_offset], | sizeof(bus_dmamap_t) * scctx->isc_ntxd[txq->ift_br_offset], | ||||
M_IFLIB, M_NOWAIT | M_ZERO)) == NULL) { | M_IFLIB, M_NOWAIT | M_ZERO)) == NULL) { | ||||
device_printf(dev, "Unable to allocate TSO tx_buffer " | device_printf(dev, | ||||
"map memory\n"); | "Unable to allocate TSO TX buffer map memory\n"); | ||||
err = ENOMEM; | err = ENOMEM; | ||||
goto fail; | goto fail; | ||||
} | } | ||||
for (int i = 0; i < scctx->isc_ntxd[txq->ift_br_offset]; i++) { | for (int i = 0; i < scctx->isc_ntxd[txq->ift_br_offset]; i++) { | ||||
err = bus_dmamap_create(txq->ift_desc_tag, 0, | err = bus_dmamap_create(txq->ift_buf_tag, 0, | ||||
&txq->ift_sds.ifsd_map[i]); | &txq->ift_sds.ifsd_map[i]); | ||||
if (err != 0) { | if (err != 0) { | ||||
device_printf(dev, "Unable to create TX DMA map\n"); | device_printf(dev, "Unable to create TX DMA map\n"); | ||||
goto fail; | goto fail; | ||||
} | } | ||||
if (!tso) | if (!tso) | ||||
continue; | continue; | ||||
err = bus_dmamap_create(txq->ift_tso_desc_tag, 0, | err = bus_dmamap_create(txq->ift_tso_buf_tag, 0, | ||||
&txq->ift_sds.ifsd_tso_map[i]); | &txq->ift_sds.ifsd_tso_map[i]); | ||||
if (err != 0) { | if (err != 0) { | ||||
device_printf(dev, "Unable to create TSO TX DMA map\n"); | device_printf(dev, "Unable to create TSO TX DMA map\n"); | ||||
goto fail; | goto fail; | ||||
} | } | ||||
} | } | ||||
return (0); | return (0); | ||||
fail: | fail: | ||||
/* We free all, it handles case where we are in the middle */ | /* We free all, it handles case where we are in the middle */ | ||||
iflib_tx_structures_free(ctx); | iflib_tx_structures_free(ctx); | ||||
return (err); | return (err); | ||||
} | } | ||||
static void | static void | ||||
iflib_txsd_destroy(if_ctx_t ctx, iflib_txq_t txq, int i) | iflib_txsd_destroy(if_ctx_t ctx, iflib_txq_t txq, int i) | ||||
{ | { | ||||
bus_dmamap_t map; | bus_dmamap_t map; | ||||
map = NULL; | map = NULL; | ||||
if (txq->ift_sds.ifsd_map != NULL) | if (txq->ift_sds.ifsd_map != NULL) | ||||
map = txq->ift_sds.ifsd_map[i]; | map = txq->ift_sds.ifsd_map[i]; | ||||
if (map != NULL) { | if (map != NULL) { | ||||
bus_dmamap_sync(txq->ift_desc_tag, map, BUS_DMASYNC_POSTWRITE); | bus_dmamap_sync(txq->ift_buf_tag, map, BUS_DMASYNC_POSTWRITE); | ||||
bus_dmamap_unload(txq->ift_desc_tag, map); | bus_dmamap_unload(txq->ift_buf_tag, map); | ||||
bus_dmamap_destroy(txq->ift_desc_tag, map); | bus_dmamap_destroy(txq->ift_buf_tag, map); | ||||
txq->ift_sds.ifsd_map[i] = NULL; | txq->ift_sds.ifsd_map[i] = NULL; | ||||
} | } | ||||
map = NULL; | map = NULL; | ||||
if (txq->ift_sds.ifsd_tso_map != NULL) | if (txq->ift_sds.ifsd_tso_map != NULL) | ||||
map = txq->ift_sds.ifsd_tso_map[i]; | map = txq->ift_sds.ifsd_tso_map[i]; | ||||
if (map != NULL) { | if (map != NULL) { | ||||
bus_dmamap_sync(txq->ift_tso_desc_tag, map, | bus_dmamap_sync(txq->ift_tso_buf_tag, map, | ||||
BUS_DMASYNC_POSTWRITE); | BUS_DMASYNC_POSTWRITE); | ||||
bus_dmamap_unload(txq->ift_tso_desc_tag, map); | bus_dmamap_unload(txq->ift_tso_buf_tag, map); | ||||
bus_dmamap_destroy(txq->ift_tso_desc_tag, map); | bus_dmamap_destroy(txq->ift_tso_buf_tag, map); | ||||
txq->ift_sds.ifsd_tso_map[i] = NULL; | txq->ift_sds.ifsd_tso_map[i] = NULL; | ||||
} | } | ||||
} | } | ||||
static void | static void | ||||
iflib_txq_destroy(iflib_txq_t txq) | iflib_txq_destroy(iflib_txq_t txq) | ||||
{ | { | ||||
if_ctx_t ctx = txq->ift_ctx; | if_ctx_t ctx = txq->ift_ctx; | ||||
for (int i = 0; i < txq->ift_size; i++) | for (int i = 0; i < txq->ift_size; i++) | ||||
iflib_txsd_destroy(ctx, txq, i); | iflib_txsd_destroy(ctx, txq, i); | ||||
if (txq->ift_sds.ifsd_map != NULL) { | if (txq->ift_sds.ifsd_map != NULL) { | ||||
free(txq->ift_sds.ifsd_map, M_IFLIB); | free(txq->ift_sds.ifsd_map, M_IFLIB); | ||||
txq->ift_sds.ifsd_map = NULL; | txq->ift_sds.ifsd_map = NULL; | ||||
} | } | ||||
if (txq->ift_sds.ifsd_tso_map != NULL) { | if (txq->ift_sds.ifsd_tso_map != NULL) { | ||||
free(txq->ift_sds.ifsd_tso_map, M_IFLIB); | free(txq->ift_sds.ifsd_tso_map, M_IFLIB); | ||||
txq->ift_sds.ifsd_tso_map = NULL; | txq->ift_sds.ifsd_tso_map = NULL; | ||||
} | } | ||||
if (txq->ift_sds.ifsd_m != NULL) { | if (txq->ift_sds.ifsd_m != NULL) { | ||||
free(txq->ift_sds.ifsd_m, M_IFLIB); | free(txq->ift_sds.ifsd_m, M_IFLIB); | ||||
txq->ift_sds.ifsd_m = NULL; | txq->ift_sds.ifsd_m = NULL; | ||||
} | } | ||||
if (txq->ift_desc_tag != NULL) { | if (txq->ift_buf_tag != NULL) { | ||||
bus_dma_tag_destroy(txq->ift_desc_tag); | bus_dma_tag_destroy(txq->ift_buf_tag); | ||||
txq->ift_desc_tag = NULL; | txq->ift_buf_tag = NULL; | ||||
} | } | ||||
if (txq->ift_tso_desc_tag != NULL) { | if (txq->ift_tso_buf_tag != NULL) { | ||||
bus_dma_tag_destroy(txq->ift_tso_desc_tag); | bus_dma_tag_destroy(txq->ift_tso_buf_tag); | ||||
txq->ift_tso_desc_tag = NULL; | txq->ift_tso_buf_tag = NULL; | ||||
} | } | ||||
} | } | ||||
static void | static void | ||||
iflib_txsd_free(if_ctx_t ctx, iflib_txq_t txq, int i) | iflib_txsd_free(if_ctx_t ctx, iflib_txq_t txq, int i) | ||||
{ | { | ||||
struct mbuf **mp; | struct mbuf **mp; | ||||
mp = &txq->ift_sds.ifsd_m[i]; | mp = &txq->ift_sds.ifsd_m[i]; | ||||
if (*mp == NULL) | if (*mp == NULL) | ||||
return; | return; | ||||
if (txq->ift_sds.ifsd_map != NULL) { | if (txq->ift_sds.ifsd_map != NULL) { | ||||
bus_dmamap_sync(txq->ift_desc_tag, | bus_dmamap_sync(txq->ift_buf_tag, | ||||
txq->ift_sds.ifsd_map[i], BUS_DMASYNC_POSTWRITE); | txq->ift_sds.ifsd_map[i], BUS_DMASYNC_POSTWRITE); | ||||
bus_dmamap_unload(txq->ift_desc_tag, txq->ift_sds.ifsd_map[i]); | bus_dmamap_unload(txq->ift_buf_tag, txq->ift_sds.ifsd_map[i]); | ||||
} | } | ||||
if (txq->ift_sds.ifsd_tso_map != NULL) { | if (txq->ift_sds.ifsd_tso_map != NULL) { | ||||
bus_dmamap_sync(txq->ift_tso_desc_tag, | bus_dmamap_sync(txq->ift_tso_buf_tag, | ||||
txq->ift_sds.ifsd_tso_map[i], BUS_DMASYNC_POSTWRITE); | txq->ift_sds.ifsd_tso_map[i], BUS_DMASYNC_POSTWRITE); | ||||
bus_dmamap_unload(txq->ift_tso_desc_tag, | bus_dmamap_unload(txq->ift_tso_buf_tag, | ||||
txq->ift_sds.ifsd_tso_map[i]); | txq->ift_sds.ifsd_tso_map[i]); | ||||
} | } | ||||
m_free(*mp); | m_free(*mp); | ||||
DBG_COUNTER_INC(tx_frees); | DBG_COUNTER_INC(tx_frees); | ||||
*mp = NULL; | *mp = NULL; | ||||
} | } | ||||
static int | static int | ||||
Show All 22 Lines | iflib_txq_setup(iflib_txq_t txq) | ||||
for (i = 0, di = txq->ift_ifdi; i < sctx->isc_ntxqs; i++, di++) | for (i = 0, di = txq->ift_ifdi; i < sctx->isc_ntxqs; i++, di++) | ||||
bus_dmamap_sync(di->idi_tag, di->idi_map, | bus_dmamap_sync(di->idi_tag, di->idi_map, | ||||
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | ||||
return (0); | return (0); | ||||
} | } | ||||
/********************************************************************* | /********************************************************************* | ||||
* | * | ||||
* Allocate memory for rx_buffer structures. Since we use one | * Allocate DMA resources for RX buffers as well as memory for the RX | ||||
* rx_buffer per received packet, the maximum number of rx_buffer's | * mbuf map, direct RX cluster pointer map and RX cluster bus address | ||||
* that we'll need is equal to the number of receive descriptors | * map. RX DMA map, RX mbuf map, direct RX cluster pointer map and | ||||
* that we've allocated. | * RX cluster map are kept in a iflib_sw_rx_desc_array structure. | ||||
* Since we use use one entry in iflib_sw_rx_desc_array per received | |||||
* packet, the maximum number of entries we'll need is equal to the | |||||
* number of hardware receive descriptors that we've allocated. | |||||
* | * | ||||
**********************************************************************/ | **********************************************************************/ | ||||
static int | static int | ||||
iflib_rxsd_alloc(iflib_rxq_t rxq) | iflib_rxsd_alloc(iflib_rxq_t rxq) | ||||
{ | { | ||||
if_ctx_t ctx = rxq->ifr_ctx; | if_ctx_t ctx = rxq->ifr_ctx; | ||||
if_shared_ctx_t sctx = ctx->ifc_sctx; | if_shared_ctx_t sctx = ctx->ifc_sctx; | ||||
if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; | if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; | ||||
device_t dev = ctx->ifc_dev; | device_t dev = ctx->ifc_dev; | ||||
iflib_fl_t fl; | iflib_fl_t fl; | ||||
int err; | int err; | ||||
MPASS(scctx->isc_nrxd[0] > 0); | MPASS(scctx->isc_nrxd[0] > 0); | ||||
MPASS(scctx->isc_nrxd[rxq->ifr_fl_offset] > 0); | MPASS(scctx->isc_nrxd[rxq->ifr_fl_offset] > 0); | ||||
fl = rxq->ifr_fl; | fl = rxq->ifr_fl; | ||||
for (int i = 0; i < rxq->ifr_nfl; i++, fl++) { | for (int i = 0; i < rxq->ifr_nfl; i++, fl++) { | ||||
fl->ifl_size = scctx->isc_nrxd[rxq->ifr_fl_offset]; /* this isn't necessarily the same */ | fl->ifl_size = scctx->isc_nrxd[rxq->ifr_fl_offset]; /* this isn't necessarily the same */ | ||||
/* Set up DMA tag for RX buffers. */ | |||||
err = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ | err = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ | ||||
1, 0, /* alignment, bounds */ | 1, 0, /* alignment, bounds */ | ||||
BUS_SPACE_MAXADDR, /* lowaddr */ | BUS_SPACE_MAXADDR, /* lowaddr */ | ||||
BUS_SPACE_MAXADDR, /* highaddr */ | BUS_SPACE_MAXADDR, /* highaddr */ | ||||
NULL, NULL, /* filter, filterarg */ | NULL, NULL, /* filter, filterarg */ | ||||
sctx->isc_rx_maxsize, /* maxsize */ | sctx->isc_rx_maxsize, /* maxsize */ | ||||
sctx->isc_rx_nsegments, /* nsegments */ | sctx->isc_rx_nsegments, /* nsegments */ | ||||
sctx->isc_rx_maxsegsize, /* maxsegsize */ | sctx->isc_rx_maxsegsize, /* maxsegsize */ | ||||
0, /* flags */ | 0, /* flags */ | ||||
NULL, /* lockfunc */ | NULL, /* lockfunc */ | ||||
NULL, /* lockarg */ | NULL, /* lockarg */ | ||||
&fl->ifl_desc_tag); | &fl->ifl_buf_tag); | ||||
if (err) { | if (err) { | ||||
device_printf(dev, "%s: bus_dma_tag_create failed %d\n", | device_printf(dev, | ||||
__func__, err); | "Unable to allocate RX DMA tag: %d\n", err); | ||||
goto fail; | goto fail; | ||||
} | } | ||||
/* Allocate memory for the RX mbuf map. */ | |||||
if (!(fl->ifl_sds.ifsd_m = | if (!(fl->ifl_sds.ifsd_m = | ||||
(struct mbuf **) malloc(sizeof(struct mbuf *) * | (struct mbuf **) malloc(sizeof(struct mbuf *) * | ||||
scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT | M_ZERO))) { | scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT | M_ZERO))) { | ||||
device_printf(dev, "Unable to allocate tx_buffer memory\n"); | device_printf(dev, | ||||
"Unable to allocate RX mbuf map memory\n"); | |||||
err = ENOMEM; | err = ENOMEM; | ||||
goto fail; | goto fail; | ||||
} | } | ||||
/* Allocate memory for the direct RX cluster pointer map. */ | |||||
if (!(fl->ifl_sds.ifsd_cl = | if (!(fl->ifl_sds.ifsd_cl = | ||||
(caddr_t *) malloc(sizeof(caddr_t) * | (caddr_t *) malloc(sizeof(caddr_t) * | ||||
scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT | M_ZERO))) { | scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT | M_ZERO))) { | ||||
device_printf(dev, "Unable to allocate tx_buffer memory\n"); | device_printf(dev, | ||||
"Unable to allocate RX cluster map memory\n"); | |||||
err = ENOMEM; | err = ENOMEM; | ||||
goto fail; | goto fail; | ||||
} | } | ||||
/* Allocate memory for the RX cluster bus address map. */ | |||||
if (!(fl->ifl_sds.ifsd_ba = | if (!(fl->ifl_sds.ifsd_ba = | ||||
(bus_addr_t *) malloc(sizeof(bus_addr_t) * | (bus_addr_t *) malloc(sizeof(bus_addr_t) * | ||||
scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT | M_ZERO))) { | scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT | M_ZERO))) { | ||||
device_printf(dev, "Unable to allocate rx bus addr memory\n"); | device_printf(dev, | ||||
"Unable to allocate RX bus address map memory\n"); | |||||
err = ENOMEM; | err = ENOMEM; | ||||
goto fail; | goto fail; | ||||
} | } | ||||
/* Create the descriptor buffer dma maps */ | /* | ||||
* Create the DMA maps for RX buffers. | |||||
*/ | |||||
if (!(fl->ifl_sds.ifsd_map = | if (!(fl->ifl_sds.ifsd_map = | ||||
(bus_dmamap_t *) malloc(sizeof(bus_dmamap_t) * scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT | M_ZERO))) { | (bus_dmamap_t *) malloc(sizeof(bus_dmamap_t) * scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT | M_ZERO))) { | ||||
device_printf(dev, "Unable to allocate tx_buffer map memory\n"); | device_printf(dev, | ||||
"Unable to allocate RX buffer DMA map memory\n"); | |||||
err = ENOMEM; | err = ENOMEM; | ||||
goto fail; | goto fail; | ||||
} | } | ||||
for (int i = 0; i < scctx->isc_nrxd[rxq->ifr_fl_offset]; i++) { | for (int i = 0; i < scctx->isc_nrxd[rxq->ifr_fl_offset]; i++) { | ||||
err = bus_dmamap_create(fl->ifl_desc_tag, 0, &fl->ifl_sds.ifsd_map[i]); | err = bus_dmamap_create(fl->ifl_buf_tag, 0, | ||||
&fl->ifl_sds.ifsd_map[i]); | |||||
if (err != 0) { | if (err != 0) { | ||||
device_printf(dev, "Unable to create RX buffer DMA map\n"); | device_printf(dev, "Unable to create RX buffer DMA map\n"); | ||||
goto fail; | goto fail; | ||||
} | } | ||||
} | } | ||||
} | } | ||||
return (0); | return (0); | ||||
▲ Show 20 Lines • Show All 83 Lines • ▼ Show 20 Lines | if (frag_idx < 0) | ||||
bit_ffc(fl->ifl_rx_bitmap, fl->ifl_size, &frag_idx); | bit_ffc(fl->ifl_rx_bitmap, fl->ifl_size, &frag_idx); | ||||
MPASS(frag_idx >= 0); | MPASS(frag_idx >= 0); | ||||
if ((cl = sd_cl[frag_idx]) == NULL) { | if ((cl = sd_cl[frag_idx]) == NULL) { | ||||
if ((cl = m_cljget(NULL, M_NOWAIT, fl->ifl_buf_size)) == NULL) | if ((cl = m_cljget(NULL, M_NOWAIT, fl->ifl_buf_size)) == NULL) | ||||
break; | break; | ||||
cb_arg.error = 0; | cb_arg.error = 0; | ||||
MPASS(sd_map != NULL); | MPASS(sd_map != NULL); | ||||
err = bus_dmamap_load(fl->ifl_desc_tag, sd_map[frag_idx], | err = bus_dmamap_load(fl->ifl_buf_tag, sd_map[frag_idx], | ||||
cl, fl->ifl_buf_size, _rxq_refill_cb, &cb_arg, | cl, fl->ifl_buf_size, _rxq_refill_cb, &cb_arg, | ||||
BUS_DMA_NOWAIT); | BUS_DMA_NOWAIT); | ||||
if (err != 0 || cb_arg.error) { | if (err != 0 || cb_arg.error) { | ||||
/* | /* | ||||
* !zone_pack ? | * !zone_pack ? | ||||
*/ | */ | ||||
if (fl->ifl_zone == zone_pack) | if (fl->ifl_zone == zone_pack) | ||||
uma_zfree(fl->ifl_zone, cl); | uma_zfree(fl->ifl_zone, cl); | ||||
break; | break; | ||||
} | } | ||||
bus_dmamap_sync(fl->ifl_desc_tag, sd_map[frag_idx], | bus_dmamap_sync(fl->ifl_buf_tag, sd_map[frag_idx], | ||||
BUS_DMASYNC_PREREAD); | BUS_DMASYNC_PREREAD); | ||||
sd_ba[frag_idx] = bus_addr = cb_arg.seg.ds_addr; | sd_ba[frag_idx] = bus_addr = cb_arg.seg.ds_addr; | ||||
sd_cl[frag_idx] = cl; | sd_cl[frag_idx] = cl; | ||||
#if MEMORY_LOGGING | #if MEMORY_LOGGING | ||||
fl->ifl_cl_enqueued++; | fl->ifl_cl_enqueued++; | ||||
#endif | #endif | ||||
} else { | } else { | ||||
bus_addr = sd_ba[frag_idx]; | bus_addr = sd_ba[frag_idx]; | ||||
▲ Show 20 Lines • Show All 84 Lines • ▼ Show 20 Lines | iflib_fl_bufs_free(iflib_fl_t fl) | ||||
uint32_t i; | uint32_t i; | ||||
for (i = 0; i < fl->ifl_size; i++) { | for (i = 0; i < fl->ifl_size; i++) { | ||||
struct mbuf **sd_m = &fl->ifl_sds.ifsd_m[i]; | struct mbuf **sd_m = &fl->ifl_sds.ifsd_m[i]; | ||||
caddr_t *sd_cl = &fl->ifl_sds.ifsd_cl[i]; | caddr_t *sd_cl = &fl->ifl_sds.ifsd_cl[i]; | ||||
if (*sd_cl != NULL) { | if (*sd_cl != NULL) { | ||||
sd_map = fl->ifl_sds.ifsd_map[i]; | sd_map = fl->ifl_sds.ifsd_map[i]; | ||||
bus_dmamap_sync(fl->ifl_desc_tag, sd_map, | bus_dmamap_sync(fl->ifl_buf_tag, sd_map, | ||||
BUS_DMASYNC_POSTREAD); | BUS_DMASYNC_POSTREAD); | ||||
bus_dmamap_unload(fl->ifl_desc_tag, sd_map); | bus_dmamap_unload(fl->ifl_buf_tag, sd_map); | ||||
if (*sd_cl != NULL) | if (*sd_cl != NULL) | ||||
uma_zfree(fl->ifl_zone, *sd_cl); | uma_zfree(fl->ifl_zone, *sd_cl); | ||||
// XXX: Should this get moved out? | // XXX: Should this get moved out? | ||||
if (iflib_in_detach(fl->ifl_rxq->ifr_ctx)) | if (iflib_in_detach(fl->ifl_rxq->ifr_ctx)) | ||||
bus_dmamap_destroy(fl->ifl_desc_tag, sd_map); | bus_dmamap_destroy(fl->ifl_buf_tag, sd_map); | ||||
if (*sd_m != NULL) { | if (*sd_m != NULL) { | ||||
m_init(*sd_m, M_NOWAIT, MT_DATA, 0); | m_init(*sd_m, M_NOWAIT, MT_DATA, 0); | ||||
uma_zfree(zone_mbuf, *sd_m); | uma_zfree(zone_mbuf, *sd_m); | ||||
} | } | ||||
} else { | } else { | ||||
MPASS(*sd_cl == NULL); | MPASS(*sd_cl == NULL); | ||||
MPASS(*sd_m == NULL); | MPASS(*sd_m == NULL); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 85 Lines • ▼ Show 20 Lines | |||||
iflib_rx_sds_free(iflib_rxq_t rxq) | iflib_rx_sds_free(iflib_rxq_t rxq) | ||||
{ | { | ||||
iflib_fl_t fl; | iflib_fl_t fl; | ||||
int i, j; | int i, j; | ||||
if (rxq->ifr_fl != NULL) { | if (rxq->ifr_fl != NULL) { | ||||
for (i = 0; i < rxq->ifr_nfl; i++) { | for (i = 0; i < rxq->ifr_nfl; i++) { | ||||
fl = &rxq->ifr_fl[i]; | fl = &rxq->ifr_fl[i]; | ||||
if (fl->ifl_desc_tag != NULL) { | if (fl->ifl_buf_tag != NULL) { | ||||
if (fl->ifl_sds.ifsd_map != NULL) { | if (fl->ifl_sds.ifsd_map != NULL) { | ||||
for (j = 0; j < fl->ifl_size; j++) { | for (j = 0; j < fl->ifl_size; j++) { | ||||
if (fl->ifl_sds.ifsd_map[j] == | if (fl->ifl_sds.ifsd_map[j] == | ||||
NULL) | NULL) | ||||
continue; | continue; | ||||
bus_dmamap_sync( | bus_dmamap_sync( | ||||
fl->ifl_desc_tag, | fl->ifl_buf_tag, | ||||
fl->ifl_sds.ifsd_map[j], | fl->ifl_sds.ifsd_map[j], | ||||
BUS_DMASYNC_POSTREAD); | BUS_DMASYNC_POSTREAD); | ||||
bus_dmamap_unload( | bus_dmamap_unload( | ||||
fl->ifl_desc_tag, | fl->ifl_buf_tag, | ||||
fl->ifl_sds.ifsd_map[j]); | fl->ifl_sds.ifsd_map[j]); | ||||
} | } | ||||
} | } | ||||
bus_dma_tag_destroy(fl->ifl_desc_tag); | bus_dma_tag_destroy(fl->ifl_buf_tag); | ||||
fl->ifl_desc_tag = NULL; | fl->ifl_buf_tag = NULL; | ||||
} | } | ||||
free(fl->ifl_sds.ifsd_m, M_IFLIB); | free(fl->ifl_sds.ifsd_m, M_IFLIB); | ||||
free(fl->ifl_sds.ifsd_cl, M_IFLIB); | free(fl->ifl_sds.ifsd_cl, M_IFLIB); | ||||
free(fl->ifl_sds.ifsd_ba, M_IFLIB); | free(fl->ifl_sds.ifsd_ba, M_IFLIB); | ||||
free(fl->ifl_sds.ifsd_map, M_IFLIB); | free(fl->ifl_sds.ifsd_map, M_IFLIB); | ||||
fl->ifl_sds.ifsd_m = NULL; | fl->ifl_sds.ifsd_m = NULL; | ||||
fl->ifl_sds.ifsd_cl = NULL; | fl->ifl_sds.ifsd_cl = NULL; | ||||
fl->ifl_sds.ifsd_ba = NULL; | fl->ifl_sds.ifsd_ba = NULL; | ||||
▲ Show 20 Lines • Show All 268 Lines • ▼ Show 20 Lines | #endif | ||||
next = (cidx + CACHE_PTR_INCREMENT) & (fl->ifl_size-1); | next = (cidx + CACHE_PTR_INCREMENT) & (fl->ifl_size-1); | ||||
prefetch(&fl->ifl_sds.ifsd_map[next]); | prefetch(&fl->ifl_sds.ifsd_map[next]); | ||||
map = fl->ifl_sds.ifsd_map[cidx]; | map = fl->ifl_sds.ifsd_map[cidx]; | ||||
di = fl->ifl_ifdi; | di = fl->ifl_ifdi; | ||||
next = (cidx + CACHE_LINE_SIZE) & (fl->ifl_size-1); | next = (cidx + CACHE_LINE_SIZE) & (fl->ifl_size-1); | ||||
/* not valid assert if bxe really does SGE from non-contiguous elements */ | /* not valid assert if bxe really does SGE from non-contiguous elements */ | ||||
MPASS(fl->ifl_cidx == cidx); | MPASS(fl->ifl_cidx == cidx); | ||||
bus_dmamap_sync(fl->ifl_desc_tag, map, BUS_DMASYNC_POSTREAD); | bus_dmamap_sync(fl->ifl_buf_tag, map, BUS_DMASYNC_POSTREAD); | ||||
if (unload) | if (unload) | ||||
bus_dmamap_unload(fl->ifl_desc_tag, map); | bus_dmamap_unload(fl->ifl_buf_tag, map); | ||||
fl->ifl_cidx = (fl->ifl_cidx + 1) & (fl->ifl_size-1); | fl->ifl_cidx = (fl->ifl_cidx + 1) & (fl->ifl_size-1); | ||||
if (__predict_false(fl->ifl_cidx == 0)) | if (__predict_false(fl->ifl_cidx == 0)) | ||||
fl->ifl_gen = 0; | fl->ifl_gen = 0; | ||||
bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, | bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, | ||||
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | ||||
bit_clear(fl->ifl_rx_bitmap, cidx); | bit_clear(fl->ifl_rx_bitmap, cidx); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 66 Lines • ▼ Show 20 Lines | if (ri->iri_nfrags == 1 && | ||||
m = *sd.ifsd_m; | m = *sd.ifsd_m; | ||||
*sd.ifsd_m = NULL; | *sd.ifsd_m = NULL; | ||||
m_init(m, M_NOWAIT, MT_DATA, M_PKTHDR); | m_init(m, M_NOWAIT, MT_DATA, M_PKTHDR); | ||||
#ifndef __NO_STRICT_ALIGNMENT | #ifndef __NO_STRICT_ALIGNMENT | ||||
if (!IP_ALIGNED(m)) | if (!IP_ALIGNED(m)) | ||||
m->m_data += 2; | m->m_data += 2; | ||||
#endif | #endif | ||||
memcpy(m->m_data, *sd.ifsd_cl, ri->iri_len); | memcpy(m->m_data, *sd.ifsd_cl, ri->iri_len); | ||||
bus_dmamap_sync(rxq->ifr_fl->ifl_desc_tag, | bus_dmamap_sync(rxq->ifr_fl->ifl_buf_tag, | ||||
rxq->ifr_fl->ifl_sds.ifsd_map[ri->iri_frags[0].irf_idx], | rxq->ifr_fl->ifl_sds.ifsd_map[ri->iri_frags[0].irf_idx], | ||||
BUS_DMASYNC_PREREAD); | BUS_DMASYNC_PREREAD); | ||||
m->m_len = ri->iri_frags[0].irf_len; | m->m_len = ri->iri_frags[0].irf_len; | ||||
} else { | } else { | ||||
m = assemble_segments(rxq, ri, &sd); | m = assemble_segments(rxq, ri, &sd); | ||||
} | } | ||||
m->m_pkthdr.len = ri->iri_len; | m->m_pkthdr.len = ri->iri_len; | ||||
m->m_pkthdr.rcvif = ri->iri_ifp; | m->m_pkthdr.rcvif = ri->iri_ifp; | ||||
▲ Show 20 Lines • Show All 484 Lines • ▼ Show 20 Lines | iflib_remove_mbuf(iflib_txq_t txq) | ||||
struct mbuf *m, **ifsd_m; | struct mbuf *m, **ifsd_m; | ||||
ifsd_m = txq->ift_sds.ifsd_m; | ifsd_m = txq->ift_sds.ifsd_m; | ||||
ntxd = txq->ift_size; | ntxd = txq->ift_size; | ||||
pidx = txq->ift_pidx & (ntxd - 1); | pidx = txq->ift_pidx & (ntxd - 1); | ||||
ifsd_m = txq->ift_sds.ifsd_m; | ifsd_m = txq->ift_sds.ifsd_m; | ||||
m = ifsd_m[pidx]; | m = ifsd_m[pidx]; | ||||
ifsd_m[pidx] = NULL; | ifsd_m[pidx] = NULL; | ||||
bus_dmamap_unload(txq->ift_desc_tag, txq->ift_sds.ifsd_map[pidx]); | bus_dmamap_unload(txq->ift_buf_tag, txq->ift_sds.ifsd_map[pidx]); | ||||
if (txq->ift_sds.ifsd_tso_map != NULL) | if (txq->ift_sds.ifsd_tso_map != NULL) | ||||
bus_dmamap_unload(txq->ift_tso_desc_tag, | bus_dmamap_unload(txq->ift_tso_buf_tag, | ||||
txq->ift_sds.ifsd_tso_map[pidx]); | txq->ift_sds.ifsd_tso_map[pidx]); | ||||
#if MEMORY_LOGGING | #if MEMORY_LOGGING | ||||
txq->ift_dequeued++; | txq->ift_dequeued++; | ||||
#endif | #endif | ||||
return (m); | return (m); | ||||
} | } | ||||
static inline caddr_t | static inline caddr_t | ||||
▲ Show 20 Lines • Show All 60 Lines • ▼ Show 20 Lines | |||||
} | } | ||||
static int | static int | ||||
iflib_encap(iflib_txq_t txq, struct mbuf **m_headp) | iflib_encap(iflib_txq_t txq, struct mbuf **m_headp) | ||||
{ | { | ||||
if_ctx_t ctx; | if_ctx_t ctx; | ||||
if_shared_ctx_t sctx; | if_shared_ctx_t sctx; | ||||
if_softc_ctx_t scctx; | if_softc_ctx_t scctx; | ||||
bus_dma_tag_t buf_tag; | |||||
bus_dma_segment_t *segs; | bus_dma_segment_t *segs; | ||||
struct mbuf *m_head, **ifsd_m; | struct mbuf *m_head, **ifsd_m; | ||||
void *next_txd; | void *next_txd; | ||||
bus_dmamap_t map; | bus_dmamap_t map; | ||||
struct if_pkt_info pi; | struct if_pkt_info pi; | ||||
int remap = 0; | int remap = 0; | ||||
int err, nsegs, ndesc, max_segs, pidx, cidx, next, ntxd; | int err, nsegs, ndesc, max_segs, pidx, cidx, next, ntxd; | ||||
bus_dma_tag_t desc_tag; | |||||
ctx = txq->ift_ctx; | ctx = txq->ift_ctx; | ||||
sctx = ctx->ifc_sctx; | sctx = ctx->ifc_sctx; | ||||
scctx = &ctx->ifc_softc_ctx; | scctx = &ctx->ifc_softc_ctx; | ||||
segs = txq->ift_segs; | segs = txq->ift_segs; | ||||
ntxd = txq->ift_size; | ntxd = txq->ift_size; | ||||
m_head = *m_headp; | m_head = *m_headp; | ||||
map = NULL; | map = NULL; | ||||
Show All 14 Lines | if (ctx->ifc_flags & IFC_PREFETCH) { | ||||
prefetch(&txq->ift_sds.ifsd_m[next]); | prefetch(&txq->ift_sds.ifsd_m[next]); | ||||
prefetch(&txq->ift_sds.ifsd_map[next]); | prefetch(&txq->ift_sds.ifsd_map[next]); | ||||
next = (cidx + CACHE_LINE_SIZE) & (ntxd-1); | next = (cidx + CACHE_LINE_SIZE) & (ntxd-1); | ||||
} | } | ||||
map = txq->ift_sds.ifsd_map[pidx]; | map = txq->ift_sds.ifsd_map[pidx]; | ||||
ifsd_m = txq->ift_sds.ifsd_m; | ifsd_m = txq->ift_sds.ifsd_m; | ||||
if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { | if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { | ||||
desc_tag = txq->ift_tso_desc_tag; | buf_tag = txq->ift_tso_buf_tag; | ||||
max_segs = scctx->isc_tx_tso_segments_max; | max_segs = scctx->isc_tx_tso_segments_max; | ||||
map = txq->ift_sds.ifsd_tso_map[pidx]; | map = txq->ift_sds.ifsd_tso_map[pidx]; | ||||
MPASS(desc_tag != NULL); | MPASS(buf_tag != NULL); | ||||
MPASS(max_segs > 0); | MPASS(max_segs > 0); | ||||
} else { | } else { | ||||
desc_tag = txq->ift_desc_tag; | buf_tag = txq->ift_buf_tag; | ||||
max_segs = scctx->isc_tx_nsegments; | max_segs = scctx->isc_tx_nsegments; | ||||
map = txq->ift_sds.ifsd_map[pidx]; | map = txq->ift_sds.ifsd_map[pidx]; | ||||
} | } | ||||
if ((sctx->isc_flags & IFLIB_NEED_ETHER_PAD) && | if ((sctx->isc_flags & IFLIB_NEED_ETHER_PAD) && | ||||
__predict_false(m_head->m_pkthdr.len < scctx->isc_min_frame_size)) { | __predict_false(m_head->m_pkthdr.len < scctx->isc_min_frame_size)) { | ||||
err = iflib_ether_pad(ctx->ifc_dev, m_headp, scctx->isc_min_frame_size); | err = iflib_ether_pad(ctx->ifc_dev, m_headp, scctx->isc_min_frame_size); | ||||
if (err) { | if (err) { | ||||
DBG_COUNTER_INC(encap_txd_encap_fail); | DBG_COUNTER_INC(encap_txd_encap_fail); | ||||
Show All 15 Lines | if (__predict_true((pi.ipi_csum_flags | pi.ipi_vtag))) { | ||||
if (__predict_false((err = iflib_parse_header(txq, &pi, m_headp)) != 0)) { | if (__predict_false((err = iflib_parse_header(txq, &pi, m_headp)) != 0)) { | ||||
DBG_COUNTER_INC(encap_txd_encap_fail); | DBG_COUNTER_INC(encap_txd_encap_fail); | ||||
return (err); | return (err); | ||||
} | } | ||||
m_head = *m_headp; | m_head = *m_headp; | ||||
} | } | ||||
retry: | retry: | ||||
err = bus_dmamap_load_mbuf_sg(desc_tag, map, m_head, segs, &nsegs, | err = bus_dmamap_load_mbuf_sg(buf_tag, map, m_head, segs, &nsegs, | ||||
BUS_DMA_NOWAIT); | BUS_DMA_NOWAIT); | ||||
defrag: | defrag: | ||||
if (__predict_false(err)) { | if (__predict_false(err)) { | ||||
switch (err) { | switch (err) { | ||||
case EFBIG: | case EFBIG: | ||||
/* try collapse once and defrag once */ | /* try collapse once and defrag once */ | ||||
if (remap == 0) { | if (remap == 0) { | ||||
m_head = m_collapse(*m_headp, M_NOWAIT, max_segs); | m_head = m_collapse(*m_headp, M_NOWAIT, max_segs); | ||||
Show All 29 Lines | defrag: | ||||
ifsd_m[pidx] = m_head; | ifsd_m[pidx] = m_head; | ||||
/* | /* | ||||
* XXX assumes a 1 to 1 relationship between segments and | * XXX assumes a 1 to 1 relationship between segments and | ||||
* descriptors - this does not hold true on all drivers, e.g. | * descriptors - this does not hold true on all drivers, e.g. | ||||
* cxgb | * cxgb | ||||
*/ | */ | ||||
if (__predict_false(nsegs + 2 > TXQ_AVAIL(txq))) { | if (__predict_false(nsegs + 2 > TXQ_AVAIL(txq))) { | ||||
txq->ift_no_desc_avail++; | txq->ift_no_desc_avail++; | ||||
bus_dmamap_unload(desc_tag, map); | bus_dmamap_unload(buf_tag, map); | ||||
DBG_COUNTER_INC(encap_txq_avail_fail); | DBG_COUNTER_INC(encap_txq_avail_fail); | ||||
DBG_COUNTER_INC(encap_txd_encap_fail); | DBG_COUNTER_INC(encap_txd_encap_fail); | ||||
if ((txq->ift_task.gt_task.ta_flags & TASK_ENQUEUED) == 0) | if ((txq->ift_task.gt_task.ta_flags & TASK_ENQUEUED) == 0) | ||||
GROUPTASK_ENQUEUE(&txq->ift_task); | GROUPTASK_ENQUEUE(&txq->ift_task); | ||||
return (ENOBUFS); | return (ENOBUFS); | ||||
} | } | ||||
/* | /* | ||||
* On Intel cards we can greatly reduce the number of TX interrupts | * On Intel cards we can greatly reduce the number of TX interrupts | ||||
Show All 10 Lines | defrag: | ||||
pi.ipi_segs = segs; | pi.ipi_segs = segs; | ||||
pi.ipi_nsegs = nsegs; | pi.ipi_nsegs = nsegs; | ||||
MPASS(pidx >= 0 && pidx < txq->ift_size); | MPASS(pidx >= 0 && pidx < txq->ift_size); | ||||
#ifdef PKT_DEBUG | #ifdef PKT_DEBUG | ||||
print_pkt(&pi); | print_pkt(&pi); | ||||
#endif | #endif | ||||
bus_dmamap_sync(desc_tag, map, BUS_DMASYNC_PREWRITE); | bus_dmamap_sync(buf_tag, map, BUS_DMASYNC_PREWRITE); | ||||
if ((err = ctx->isc_txd_encap(ctx->ifc_softc, &pi)) == 0) { | if ((err = ctx->isc_txd_encap(ctx->ifc_softc, &pi)) == 0) { | ||||
bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map, | bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map, | ||||
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | ||||
DBG_COUNTER_INC(tx_encap); | DBG_COUNTER_INC(tx_encap); | ||||
MPASS(pi.ipi_new_pidx < txq->ift_size); | MPASS(pi.ipi_new_pidx < txq->ift_size); | ||||
ndesc = pi.ipi_new_pidx - pi.ipi_pidx; | ndesc = pi.ipi_new_pidx - pi.ipi_pidx; | ||||
if (pi.ipi_new_pidx < pi.ipi_pidx) { | if (pi.ipi_new_pidx < pi.ipi_pidx) { | ||||
▲ Show 20 Lines • Show All 59 Lines • ▼ Show 20 Lines | iflib_tx_desc_free(iflib_txq_t txq, int n) | ||||
while (n-- > 0) { | while (n-- > 0) { | ||||
if (do_prefetch) { | if (do_prefetch) { | ||||
prefetch(ifsd_m[(cidx + 3) & mask]); | prefetch(ifsd_m[(cidx + 3) & mask]); | ||||
prefetch(ifsd_m[(cidx + 4) & mask]); | prefetch(ifsd_m[(cidx + 4) & mask]); | ||||
} | } | ||||
if ((m = ifsd_m[cidx]) != NULL) { | if ((m = ifsd_m[cidx]) != NULL) { | ||||
prefetch(&ifsd_m[(cidx + CACHE_PTR_INCREMENT) & mask]); | prefetch(&ifsd_m[(cidx + CACHE_PTR_INCREMENT) & mask]); | ||||
if (m->m_pkthdr.csum_flags & CSUM_TSO) { | if (m->m_pkthdr.csum_flags & CSUM_TSO) { | ||||
bus_dmamap_sync(txq->ift_tso_desc_tag, | bus_dmamap_sync(txq->ift_tso_buf_tag, | ||||
txq->ift_sds.ifsd_tso_map[cidx], | txq->ift_sds.ifsd_tso_map[cidx], | ||||
BUS_DMASYNC_POSTWRITE); | BUS_DMASYNC_POSTWRITE); | ||||
bus_dmamap_unload(txq->ift_tso_desc_tag, | bus_dmamap_unload(txq->ift_tso_buf_tag, | ||||
txq->ift_sds.ifsd_tso_map[cidx]); | txq->ift_sds.ifsd_tso_map[cidx]); | ||||
} else { | } else { | ||||
bus_dmamap_sync(txq->ift_desc_tag, | bus_dmamap_sync(txq->ift_buf_tag, | ||||
txq->ift_sds.ifsd_map[cidx], | txq->ift_sds.ifsd_map[cidx], | ||||
BUS_DMASYNC_POSTWRITE); | BUS_DMASYNC_POSTWRITE); | ||||
bus_dmamap_unload(txq->ift_desc_tag, | bus_dmamap_unload(txq->ift_buf_tag, | ||||
txq->ift_sds.ifsd_map[cidx]); | txq->ift_sds.ifsd_map[cidx]); | ||||
} | } | ||||
/* XXX we don't support any drivers that batch packets yet */ | /* XXX we don't support any drivers that batch packets yet */ | ||||
MPASS(m->m_nextpkt == NULL); | MPASS(m->m_nextpkt == NULL); | ||||
m_freem(m); | m_freem(m); | ||||
ifsd_m[cidx] = NULL; | ifsd_m[cidx] = NULL; | ||||
#if MEMORY_LOGGING | #if MEMORY_LOGGING | ||||
txq->ift_dequeued++; | txq->ift_dequeued++; | ||||
▲ Show 20 Lines • Show All 1,790 Lines • ▼ Show 20 Lines | iflib_queues_alloc(if_ctx_t ctx) | ||||
rxq = ctx->ifc_rxqs; | rxq = ctx->ifc_rxqs; | ||||
/* | /* | ||||
* XXX handle allocation failure | * XXX handle allocation failure | ||||
*/ | */ | ||||
for (txconf = i = 0, cpu = CPU_FIRST(); i < ntxqsets; i++, txconf++, txq++, cpu = CPU_NEXT(cpu)) { | for (txconf = i = 0, cpu = CPU_FIRST(); i < ntxqsets; i++, txconf++, txq++, cpu = CPU_NEXT(cpu)) { | ||||
/* Set up some basics */ | /* Set up some basics */ | ||||
if ((ifdip = malloc(sizeof(struct iflib_dma_info) * ntxqs, M_IFLIB, M_WAITOK|M_ZERO)) == NULL) { | if ((ifdip = malloc(sizeof(struct iflib_dma_info) * ntxqs, | ||||
device_printf(dev, "failed to allocate iflib_dma_info\n"); | M_IFLIB, M_NOWAIT | M_ZERO)) == NULL) { | ||||
device_printf(dev, | |||||
"Unable to allocate TX DMA info memory\n"); | |||||
err = ENOMEM; | err = ENOMEM; | ||||
goto err_tx_desc; | goto err_tx_desc; | ||||
} | } | ||||
txq->ift_ifdi = ifdip; | txq->ift_ifdi = ifdip; | ||||
for (j = 0; j < ntxqs; j++, ifdip++) { | for (j = 0; j < ntxqs; j++, ifdip++) { | ||||
if (iflib_dma_alloc(ctx, txqsizes[j], ifdip, BUS_DMA_NOWAIT)) { | if (iflib_dma_alloc(ctx, txqsizes[j], ifdip, 0)) { | ||||
device_printf(dev, "Unable to allocate Descriptor memory\n"); | device_printf(dev, | ||||
"Unable to allocate TX descriptors\n"); | |||||
err = ENOMEM; | err = ENOMEM; | ||||
goto err_tx_desc; | goto err_tx_desc; | ||||
} | } | ||||
txq->ift_txd_size[j] = scctx->isc_txd_size[j]; | txq->ift_txd_size[j] = scctx->isc_txd_size[j]; | ||||
bzero((void *)ifdip->idi_vaddr, txqsizes[j]); | bzero((void *)ifdip->idi_vaddr, txqsizes[j]); | ||||
} | } | ||||
txq->ift_ctx = ctx; | txq->ift_ctx = ctx; | ||||
txq->ift_id = i; | txq->ift_id = i; | ||||
Show All 27 Lines | if (err) { | ||||
device_printf(dev, "Unable to allocate buf_ring\n"); | device_printf(dev, "Unable to allocate buf_ring\n"); | ||||
goto err_tx_desc; | goto err_tx_desc; | ||||
} | } | ||||
} | } | ||||
for (rxconf = i = 0; i < nrxqsets; i++, rxconf++, rxq++) { | for (rxconf = i = 0; i < nrxqsets; i++, rxconf++, rxq++) { | ||||
/* Set up some basics */ | /* Set up some basics */ | ||||
if ((ifdip = malloc(sizeof(struct iflib_dma_info) * nrxqs, M_IFLIB, M_WAITOK|M_ZERO)) == NULL) { | if ((ifdip = malloc(sizeof(struct iflib_dma_info) * nrxqs, | ||||
device_printf(dev, "failed to allocate iflib_dma_info\n"); | M_IFLIB, M_NOWAIT | M_ZERO)) == NULL) { | ||||
device_printf(dev, | |||||
"Unable to allocate RX DMA info memory\n"); | |||||
err = ENOMEM; | err = ENOMEM; | ||||
goto err_tx_desc; | goto err_tx_desc; | ||||
} | } | ||||
rxq->ifr_ifdi = ifdip; | rxq->ifr_ifdi = ifdip; | ||||
/* XXX this needs to be changed if #rx queues != #tx queues */ | /* XXX this needs to be changed if #rx queues != #tx queues */ | ||||
rxq->ifr_ntxqirq = 1; | rxq->ifr_ntxqirq = 1; | ||||
rxq->ifr_txqid[0] = i; | rxq->ifr_txqid[0] = i; | ||||
for (j = 0; j < nrxqs; j++, ifdip++) { | for (j = 0; j < nrxqs; j++, ifdip++) { | ||||
if (iflib_dma_alloc(ctx, rxqsizes[j], ifdip, BUS_DMA_NOWAIT)) { | if (iflib_dma_alloc(ctx, rxqsizes[j], ifdip, 0)) { | ||||
device_printf(dev, "Unable to allocate Descriptor memory\n"); | device_printf(dev, | ||||
"Unable to allocate RX descriptors\n"); | |||||
err = ENOMEM; | err = ENOMEM; | ||||
goto err_tx_desc; | goto err_tx_desc; | ||||
} | } | ||||
bzero((void *)ifdip->idi_vaddr, rxqsizes[j]); | bzero((void *)ifdip->idi_vaddr, rxqsizes[j]); | ||||
} | } | ||||
rxq->ifr_ctx = ctx; | rxq->ifr_ctx = ctx; | ||||
rxq->ifr_id = i; | rxq->ifr_id = i; | ||||
if (sctx->isc_flags & IFLIB_HAS_RXCQ) { | if (sctx->isc_flags & IFLIB_HAS_RXCQ) { | ||||
Show All 35 Lines | for (i = 0; i < ntxqsets; i++) { | ||||
iflib_dma_info_t di = ctx->ifc_txqs[i].ift_ifdi; | iflib_dma_info_t di = ctx->ifc_txqs[i].ift_ifdi; | ||||
for (j = 0; j < ntxqs; j++, di++) { | for (j = 0; j < ntxqs; j++, di++) { | ||||
vaddrs[i*ntxqs + j] = di->idi_vaddr; | vaddrs[i*ntxqs + j] = di->idi_vaddr; | ||||
paddrs[i*ntxqs + j] = di->idi_paddr; | paddrs[i*ntxqs + j] = di->idi_paddr; | ||||
} | } | ||||
} | } | ||||
if ((err = IFDI_TX_QUEUES_ALLOC(ctx, vaddrs, paddrs, ntxqs, ntxqsets)) != 0) { | if ((err = IFDI_TX_QUEUES_ALLOC(ctx, vaddrs, paddrs, ntxqs, ntxqsets)) != 0) { | ||||
device_printf(ctx->ifc_dev, "device queue allocation failed\n"); | device_printf(ctx->ifc_dev, | ||||
"Unable to allocate device TX queue\n"); | |||||
iflib_tx_structures_free(ctx); | iflib_tx_structures_free(ctx); | ||||
free(vaddrs, M_IFLIB); | free(vaddrs, M_IFLIB); | ||||
free(paddrs, M_IFLIB); | free(paddrs, M_IFLIB); | ||||
goto err_rx_desc; | goto err_rx_desc; | ||||
} | } | ||||
free(vaddrs, M_IFLIB); | free(vaddrs, M_IFLIB); | ||||
free(paddrs, M_IFLIB); | free(paddrs, M_IFLIB); | ||||
/* RXQs */ | /* RXQs */ | ||||
vaddrs = malloc(sizeof(caddr_t)*nrxqsets*nrxqs, M_IFLIB, M_WAITOK); | vaddrs = malloc(sizeof(caddr_t)*nrxqsets*nrxqs, M_IFLIB, M_WAITOK); | ||||
paddrs = malloc(sizeof(uint64_t)*nrxqsets*nrxqs, M_IFLIB, M_WAITOK); | paddrs = malloc(sizeof(uint64_t)*nrxqsets*nrxqs, M_IFLIB, M_WAITOK); | ||||
for (i = 0; i < nrxqsets; i++) { | for (i = 0; i < nrxqsets; i++) { | ||||
iflib_dma_info_t di = ctx->ifc_rxqs[i].ifr_ifdi; | iflib_dma_info_t di = ctx->ifc_rxqs[i].ifr_ifdi; | ||||
for (j = 0; j < nrxqs; j++, di++) { | for (j = 0; j < nrxqs; j++, di++) { | ||||
vaddrs[i*nrxqs + j] = di->idi_vaddr; | vaddrs[i*nrxqs + j] = di->idi_vaddr; | ||||
paddrs[i*nrxqs + j] = di->idi_paddr; | paddrs[i*nrxqs + j] = di->idi_paddr; | ||||
} | } | ||||
} | } | ||||
if ((err = IFDI_RX_QUEUES_ALLOC(ctx, vaddrs, paddrs, nrxqs, nrxqsets)) != 0) { | if ((err = IFDI_RX_QUEUES_ALLOC(ctx, vaddrs, paddrs, nrxqs, nrxqsets)) != 0) { | ||||
device_printf(ctx->ifc_dev, "device queue allocation failed\n"); | device_printf(ctx->ifc_dev, | ||||
"Unable to allocate device RX queue\n"); | |||||
iflib_tx_structures_free(ctx); | iflib_tx_structures_free(ctx); | ||||
free(vaddrs, M_IFLIB); | free(vaddrs, M_IFLIB); | ||||
free(paddrs, M_IFLIB); | free(paddrs, M_IFLIB); | ||||
goto err_rx_desc; | goto err_rx_desc; | ||||
} | } | ||||
free(vaddrs, M_IFLIB); | free(vaddrs, M_IFLIB); | ||||
free(paddrs, M_IFLIB); | free(paddrs, M_IFLIB); | ||||
▲ Show 20 Lines • Show All 1,142 Lines • Show Last 20 Lines |