Changeset View
Changeset View
Standalone View
Standalone View
head/sys/net/iflib.c
Show First 20 Lines • Show All 1,455 Lines • ▼ Show 20 Lines | iflib_fast_intr_rxtx(void *arg) | ||||
iflib_filter_info_t info = arg; | iflib_filter_info_t info = arg; | ||||
struct grouptask *gtask = info->ifi_task; | struct grouptask *gtask = info->ifi_task; | ||||
if_ctx_t ctx; | if_ctx_t ctx; | ||||
iflib_rxq_t rxq = (iflib_rxq_t)info->ifi_ctx; | iflib_rxq_t rxq = (iflib_rxq_t)info->ifi_ctx; | ||||
iflib_txq_t txq; | iflib_txq_t txq; | ||||
void *sc; | void *sc; | ||||
int i, cidx, result; | int i, cidx, result; | ||||
qidx_t txqid; | qidx_t txqid; | ||||
bool intr_enable, intr_legacy; | |||||
if (!iflib_started) | if (!iflib_started) | ||||
return (FILTER_STRAY); | return (FILTER_STRAY); | ||||
DBG_COUNTER_INC(fast_intrs); | DBG_COUNTER_INC(fast_intrs); | ||||
if (info->ifi_filter != NULL) { | if (info->ifi_filter != NULL) { | ||||
result = info->ifi_filter(info->ifi_filter_arg); | result = info->ifi_filter(info->ifi_filter_arg); | ||||
if ((result & FILTER_SCHEDULE_THREAD) == 0) | if ((result & FILTER_SCHEDULE_THREAD) == 0) | ||||
return (result); | return (result); | ||||
} | } | ||||
ctx = rxq->ifr_ctx; | ctx = rxq->ifr_ctx; | ||||
sc = ctx->ifc_softc; | sc = ctx->ifc_softc; | ||||
intr_enable = false; | |||||
intr_legacy = !!(ctx->ifc_flags & IFC_LEGACY); | |||||
MPASS(rxq->ifr_ntxqirq); | MPASS(rxq->ifr_ntxqirq); | ||||
for (i = 0; i < rxq->ifr_ntxqirq; i++) { | for (i = 0; i < rxq->ifr_ntxqirq; i++) { | ||||
txqid = rxq->ifr_txqid[i]; | txqid = rxq->ifr_txqid[i]; | ||||
txq = &ctx->ifc_txqs[txqid]; | txq = &ctx->ifc_txqs[txqid]; | ||||
bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map, | bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map, | ||||
BUS_DMASYNC_POSTREAD); | BUS_DMASYNC_POSTREAD); | ||||
if (!ctx->isc_txd_credits_update(sc, txqid, false)) { | if (!ctx->isc_txd_credits_update(sc, txqid, false)) { | ||||
if (intr_legacy) | |||||
intr_enable = true; | |||||
else | |||||
IFDI_TX_QUEUE_INTR_ENABLE(ctx, txqid); | IFDI_TX_QUEUE_INTR_ENABLE(ctx, txqid); | ||||
continue; | continue; | ||||
} | } | ||||
GROUPTASK_ENQUEUE(&txq->ift_task); | GROUPTASK_ENQUEUE(&txq->ift_task); | ||||
} | } | ||||
if (ctx->ifc_sctx->isc_flags & IFLIB_HAS_RXCQ) | if (ctx->ifc_sctx->isc_flags & IFLIB_HAS_RXCQ) | ||||
cidx = rxq->ifr_cq_cidx; | cidx = rxq->ifr_cq_cidx; | ||||
else | else | ||||
cidx = rxq->ifr_fl[0].ifl_cidx; | cidx = rxq->ifr_fl[0].ifl_cidx; | ||||
if (iflib_rxd_avail(ctx, rxq, cidx, 1)) | if (iflib_rxd_avail(ctx, rxq, cidx, 1)) | ||||
GROUPTASK_ENQUEUE(gtask); | GROUPTASK_ENQUEUE(gtask); | ||||
else { | else { | ||||
if (intr_legacy) | |||||
intr_enable = true; | |||||
else | |||||
IFDI_RX_QUEUE_INTR_ENABLE(ctx, rxq->ifr_id); | IFDI_RX_QUEUE_INTR_ENABLE(ctx, rxq->ifr_id); | ||||
DBG_COUNTER_INC(rx_intr_enables); | DBG_COUNTER_INC(rx_intr_enables); | ||||
} | } | ||||
if (intr_enable) | |||||
IFDI_INTR_ENABLE(ctx); | |||||
return (FILTER_HANDLED); | return (FILTER_HANDLED); | ||||
} | } | ||||
static int | static int | ||||
iflib_fast_intr_ctx(void *arg) | iflib_fast_intr_ctx(void *arg) | ||||
{ | { | ||||
iflib_filter_info_t info = arg; | iflib_filter_info_t info = arg; | ||||
▲ Show 20 Lines • Show All 840 Lines • ▼ Show 20 Lines | for (i = 0, rxq = ctx->ifc_rxqs; i < sctx->isc_nrxqsets; i++, rxq++) { | ||||
/* XXX this should really be done on a per-queue basis */ | /* XXX this should really be done on a per-queue basis */ | ||||
if (if_getcapenable(ifp) & IFCAP_NETMAP) { | if (if_getcapenable(ifp) & IFCAP_NETMAP) { | ||||
MPASS(rxq->ifr_id == i); | MPASS(rxq->ifr_id == i); | ||||
iflib_netmap_rxq_init(ctx, rxq); | iflib_netmap_rxq_init(ctx, rxq); | ||||
continue; | continue; | ||||
} | } | ||||
for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) { | for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) { | ||||
if (iflib_fl_setup(fl)) { | if (iflib_fl_setup(fl)) { | ||||
device_printf(ctx->ifc_dev, "freelist setup failed - check cluster settings\n"); | device_printf(ctx->ifc_dev, | ||||
"setting up free list %d failed - " | |||||
"check cluster settings\n", j); | |||||
goto done; | goto done; | ||||
} | } | ||||
} | } | ||||
} | } | ||||
done: | done: | ||||
if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); | if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); | ||||
IFDI_INTR_ENABLE(ctx); | IFDI_INTR_ENABLE(ctx); | ||||
txq = ctx->ifc_txqs; | txq = ctx->ifc_txqs; | ||||
▲ Show 20 Lines • Show All 1,364 Lines • ▼ Show 20 Lines | #endif | ||||
if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)) | if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)) | ||||
return; | return; | ||||
#ifdef DEV_NETMAP | #ifdef DEV_NETMAP | ||||
if (if_getcapenable(ifp) & IFCAP_NETMAP) { | if (if_getcapenable(ifp) & IFCAP_NETMAP) { | ||||
bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map, | bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map, | ||||
BUS_DMASYNC_POSTREAD); | BUS_DMASYNC_POSTREAD); | ||||
if (ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id, false)) | if (ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id, false)) | ||||
netmap_tx_irq(ifp, txq->ift_id); | netmap_tx_irq(ifp, txq->ift_id); | ||||
if (ctx->ifc_flags & IFC_LEGACY) | |||||
IFDI_INTR_ENABLE(ctx); | |||||
else | |||||
IFDI_TX_QUEUE_INTR_ENABLE(ctx, txq->ift_id); | IFDI_TX_QUEUE_INTR_ENABLE(ctx, txq->ift_id); | ||||
return; | return; | ||||
} | } | ||||
#endif | #endif | ||||
#ifdef ALTQ | #ifdef ALTQ | ||||
if (ALTQ_IS_ENABLED(&ifp->if_snd)) | if (ALTQ_IS_ENABLED(&ifp->if_snd)) | ||||
iflib_altq_if_start(ifp); | iflib_altq_if_start(ifp); | ||||
#endif | #endif | ||||
if (txq->ift_db_pending) | if (txq->ift_db_pending) | ||||
ifmp_ring_enqueue(txq->ift_br, (void **)&txq, 1, TX_BATCH_SIZE, abdicate); | ifmp_ring_enqueue(txq->ift_br, (void **)&txq, 1, TX_BATCH_SIZE, abdicate); | ||||
else if (!abdicate) | else if (!abdicate) | ||||
ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE); | ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE); | ||||
/* | /* | ||||
* When abdicating, we always need to check drainage, not just when we don't enqueue | * When abdicating, we always need to check drainage, not just when we don't enqueue | ||||
*/ | */ | ||||
if (abdicate) | if (abdicate) | ||||
ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE); | ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE); | ||||
if (ctx->ifc_flags & IFC_LEGACY) | if (ctx->ifc_flags & IFC_LEGACY) | ||||
IFDI_INTR_ENABLE(ctx); | IFDI_INTR_ENABLE(ctx); | ||||
else { | else | ||||
#ifdef INVARIANTS | |||||
int rc = | |||||
#endif | |||||
IFDI_TX_QUEUE_INTR_ENABLE(ctx, txq->ift_id); | IFDI_TX_QUEUE_INTR_ENABLE(ctx, txq->ift_id); | ||||
KASSERT(rc != ENOTSUP, ("MSI-X support requires queue_intr_enable, but not implemented in driver")); | |||||
} | } | ||||
} | |||||
static void | static void | ||||
_task_fn_rx(void *context) | _task_fn_rx(void *context) | ||||
{ | { | ||||
iflib_rxq_t rxq = context; | iflib_rxq_t rxq = context; | ||||
if_ctx_t ctx = rxq->ifr_ctx; | if_ctx_t ctx = rxq->ifr_ctx; | ||||
bool more; | bool more; | ||||
uint16_t budget; | uint16_t budget; | ||||
Show All 14 Lines | #ifdef DEV_NETMAP | ||||
} | } | ||||
#endif | #endif | ||||
budget = ctx->ifc_sysctl_rx_budget; | budget = ctx->ifc_sysctl_rx_budget; | ||||
if (budget == 0) | if (budget == 0) | ||||
budget = 16; /* XXX */ | budget = 16; /* XXX */ | ||||
if (more == false || (more = iflib_rxeof(rxq, budget)) == false) { | if (more == false || (more = iflib_rxeof(rxq, budget)) == false) { | ||||
if (ctx->ifc_flags & IFC_LEGACY) | if (ctx->ifc_flags & IFC_LEGACY) | ||||
IFDI_INTR_ENABLE(ctx); | IFDI_INTR_ENABLE(ctx); | ||||
else { | else | ||||
#ifdef INVARIANTS | |||||
int rc = | |||||
#endif | |||||
IFDI_RX_QUEUE_INTR_ENABLE(ctx, rxq->ifr_id); | IFDI_RX_QUEUE_INTR_ENABLE(ctx, rxq->ifr_id); | ||||
KASSERT(rc != ENOTSUP, ("MSI-X support requires queue_intr_enable, but not implemented in driver")); | |||||
DBG_COUNTER_INC(rx_intr_enables); | DBG_COUNTER_INC(rx_intr_enables); | ||||
} | } | ||||
} | |||||
if (__predict_false(!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING))) | if (__predict_false(!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING))) | ||||
return; | return; | ||||
if (more) | if (more) | ||||
GROUPTASK_ENQUEUE(&rxq->ifr_task); | GROUPTASK_ENQUEUE(&rxq->ifr_task); | ||||
} | } | ||||
static void | static void | ||||
_task_fn_admin(void *context) | _task_fn_admin(void *context) | ||||
▲ Show 20 Lines • Show All 717 Lines • ▼ Show 20 Lines | SLIST_FOREACH_SAFE(op, &cpu_offsets, entries, top) { | ||||
} | } | ||||
} | } | ||||
mtx_unlock(&cpu_offset_mtx); | mtx_unlock(&cpu_offset_mtx); | ||||
} | } | ||||
int | int | ||||
iflib_device_register(device_t dev, void *sc, if_shared_ctx_t sctx, if_ctx_t *ctxp) | iflib_device_register(device_t dev, void *sc, if_shared_ctx_t sctx, if_ctx_t *ctxp) | ||||
{ | { | ||||
int err, rid, msix; | |||||
if_ctx_t ctx; | if_ctx_t ctx; | ||||
if_t ifp; | if_t ifp; | ||||
if_softc_ctx_t scctx; | if_softc_ctx_t scctx; | ||||
int i; | kobjop_desc_t kobj_desc; | ||||
uint16_t main_txq; | kobj_method_t *kobj_method; | ||||
uint16_t main_rxq; | int err, i, msix, rid; | ||||
uint16_t main_rxq, main_txq; | |||||
ctx = malloc(sizeof(* ctx), M_IFLIB, M_WAITOK|M_ZERO); | ctx = malloc(sizeof(* ctx), M_IFLIB, M_WAITOK|M_ZERO); | ||||
if (sc == NULL) { | if (sc == NULL) { | ||||
sc = malloc(sctx->isc_driver->size, M_IFLIB, M_WAITOK|M_ZERO); | sc = malloc(sctx->isc_driver->size, M_IFLIB, M_WAITOK|M_ZERO); | ||||
device_set_softc(dev, ctx); | device_set_softc(dev, ctx); | ||||
ctx->ifc_flags |= IFC_SC_ALLOCATED; | ctx->ifc_flags |= IFC_SC_ALLOCATED; | ||||
} | } | ||||
▲ Show 20 Lines • Show All 142 Lines • ▼ Show 20 Lines | #endif | ||||
* SI_SUB_SMP. | * SI_SUB_SMP. | ||||
* | * | ||||
* XXX: disabling interrupts doesn't actually work, at least for | * XXX: disabling interrupts doesn't actually work, at least for | ||||
* the non-MSI case. When they occur before SI_SUB_SMP completes, | * the non-MSI case. When they occur before SI_SUB_SMP completes, | ||||
* we do null handling and depend on this not causing too large an | * we do null handling and depend on this not causing too large an | ||||
* interrupt storm. | * interrupt storm. | ||||
*/ | */ | ||||
IFDI_INTR_DISABLE(ctx); | IFDI_INTR_DISABLE(ctx); | ||||
if (msix > 1 && (err = IFDI_MSIX_INTR_ASSIGN(ctx, msix)) != 0) { | |||||
device_printf(dev, "IFDI_MSIX_INTR_ASSIGN failed %d\n", err); | if (msix > 1) { | ||||
/* | |||||
* When using MSI-X, ensure that ifdi_{r,t}x_queue_intr_enable | |||||
* aren't the default NULL implementation. | |||||
*/ | |||||
kobj_desc = &ifdi_rx_queue_intr_enable_desc; | |||||
kobj_method = kobj_lookup_method(((kobj_t)ctx)->ops->cls, NULL, | |||||
kobj_desc); | |||||
if (kobj_method == &kobj_desc->deflt) { | |||||
device_printf(dev, | |||||
"MSI-X requires ifdi_rx_queue_intr_enable method"); | |||||
err = EOPNOTSUPP; | |||||
goto fail_queues; | goto fail_queues; | ||||
} | } | ||||
if (msix <= 1) { | kobj_desc = &ifdi_tx_queue_intr_enable_desc; | ||||
kobj_method = kobj_lookup_method(((kobj_t)ctx)->ops->cls, NULL, | |||||
kobj_desc); | |||||
if (kobj_method == &kobj_desc->deflt) { | |||||
device_printf(dev, | |||||
"MSI-X requires ifdi_tx_queue_intr_enable method"); | |||||
err = EOPNOTSUPP; | |||||
goto fail_queues; | |||||
} | |||||
/* | |||||
* Assign the MSI-X vectors. | |||||
* Note that the default NULL ifdi_msix_intr_assign method will | |||||
* fail here, too. | |||||
*/ | |||||
err = IFDI_MSIX_INTR_ASSIGN(ctx, msix); | |||||
if (err != 0) { | |||||
device_printf(dev, "IFDI_MSIX_INTR_ASSIGN failed %d\n", | |||||
err); | |||||
goto fail_queues; | |||||
} | |||||
} else { | |||||
rid = 0; | rid = 0; | ||||
if (scctx->isc_intr == IFLIB_INTR_MSI) { | if (scctx->isc_intr == IFLIB_INTR_MSI) { | ||||
MPASS(msix == 1); | MPASS(msix == 1); | ||||
rid = 1; | rid = 1; | ||||
} | } | ||||
if ((err = iflib_legacy_setup(ctx, ctx->isc_legacy_intr, ctx->ifc_softc, &rid, "irq0")) != 0) { | if ((err = iflib_legacy_setup(ctx, ctx->isc_legacy_intr, ctx->ifc_softc, &rid, "irq0")) != 0) { | ||||
device_printf(dev, "iflib_legacy_setup failed %d\n", err); | device_printf(dev, "iflib_legacy_setup failed %d\n", err); | ||||
goto fail_queues; | goto fail_queues; | ||||
Show All 23 Lines | #endif | ||||
NETDUMP_SET(ctx->ifc_ifp, iflib); | NETDUMP_SET(ctx->ifc_ifp, iflib); | ||||
if_setgetcounterfn(ctx->ifc_ifp, iflib_if_get_counter); | if_setgetcounterfn(ctx->ifc_ifp, iflib_if_get_counter); | ||||
iflib_add_device_sysctl_post(ctx); | iflib_add_device_sysctl_post(ctx); | ||||
iflib_add_pfil(ctx); | iflib_add_pfil(ctx); | ||||
ctx->ifc_flags |= IFC_INIT_DONE; | ctx->ifc_flags |= IFC_INIT_DONE; | ||||
CTX_UNLOCK(ctx); | CTX_UNLOCK(ctx); | ||||
return (0); | return (0); | ||||
fail_detach: | fail_detach: | ||||
ether_ifdetach(ctx->ifc_ifp); | ether_ifdetach(ctx->ifc_ifp); | ||||
fail_intr_free: | fail_intr_free: | ||||
iflib_free_intr_mem(ctx); | iflib_free_intr_mem(ctx); | ||||
fail_queues: | fail_queues: | ||||
iflib_tx_structures_free(ctx); | iflib_tx_structures_free(ctx); | ||||
▲ Show 20 Lines • Show All 198 Lines • ▼ Show 20 Lines | for (i = 0; i < scctx->isc_nrxqsets; i++) | ||||
IFDI_RX_CLSET(ctx, 0, i, ctx->ifc_rxqs[i].ifr_fl[0].ifl_sds.ifsd_cl); | IFDI_RX_CLSET(ctx, 0, i, ctx->ifc_rxqs[i].ifr_fl[0].ifl_sds.ifsd_cl); | ||||
*ctxp = ctx; | *ctxp = ctx; | ||||
if_setgetcounterfn(ctx->ifc_ifp, iflib_if_get_counter); | if_setgetcounterfn(ctx->ifc_ifp, iflib_if_get_counter); | ||||
iflib_add_device_sysctl_post(ctx); | iflib_add_device_sysctl_post(ctx); | ||||
ctx->ifc_flags |= IFC_INIT_DONE; | ctx->ifc_flags |= IFC_INIT_DONE; | ||||
CTX_UNLOCK(ctx); | CTX_UNLOCK(ctx); | ||||
return (0); | return (0); | ||||
fail_detach: | fail_detach: | ||||
ether_ifdetach(ctx->ifc_ifp); | ether_ifdetach(ctx->ifc_ifp); | ||||
fail_queues: | fail_queues: | ||||
iflib_tx_structures_free(ctx); | iflib_tx_structures_free(ctx); | ||||
iflib_rx_structures_free(ctx); | iflib_rx_structures_free(ctx); | ||||
fail_iflib_detach: | fail_iflib_detach: | ||||
IFDI_DETACH(ctx); | IFDI_DETACH(ctx); | ||||
▲ Show 20 Lines • Show All 376 Lines • ▼ Show 20 Lines | #endif | ||||
if ((sctx->isc_flags & IFLIB_DRIVER_MEDIA) == 0) { | if ((sctx->isc_flags & IFLIB_DRIVER_MEDIA) == 0) { | ||||
ctx->ifc_mediap = &ctx->ifc_media; | ctx->ifc_mediap = &ctx->ifc_media; | ||||
ifmedia_init(ctx->ifc_mediap, IFM_IMASK, | ifmedia_init(ctx->ifc_mediap, IFM_IMASK, | ||||
iflib_media_change, iflib_media_status); | iflib_media_change, iflib_media_status); | ||||
} | } | ||||
return (0); | return (0); | ||||
} | } | ||||
static int | static int | ||||
iflib_queues_alloc(if_ctx_t ctx) | iflib_queues_alloc(if_ctx_t ctx) | ||||
{ | { | ||||
if_shared_ctx_t sctx = ctx->ifc_sctx; | if_shared_ctx_t sctx = ctx->ifc_sctx; | ||||
if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; | if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; | ||||
device_t dev = ctx->ifc_dev; | device_t dev = ctx->ifc_dev; | ||||
int nrxqsets = scctx->isc_nrxqsets; | int nrxqsets = scctx->isc_nrxqsets; | ||||
int ntxqsets = scctx->isc_ntxqsets; | int ntxqsets = scctx->isc_ntxqsets; | ||||
▲ Show 20 Lines • Show All 242 Lines • ▼ Show 20 Lines | |||||
* | * | ||||
**********************************************************************/ | **********************************************************************/ | ||||
static int | static int | ||||
iflib_rx_structures_setup(if_ctx_t ctx) | iflib_rx_structures_setup(if_ctx_t ctx) | ||||
{ | { | ||||
iflib_rxq_t rxq = ctx->ifc_rxqs; | iflib_rxq_t rxq = ctx->ifc_rxqs; | ||||
int q; | int q; | ||||
#if defined(INET6) || defined(INET) | #if defined(INET6) || defined(INET) | ||||
int i, err; | int err, i; | ||||
#endif | #endif | ||||
for (q = 0; q < ctx->ifc_softc_ctx.isc_nrxqsets; q++, rxq++) { | for (q = 0; q < ctx->ifc_softc_ctx.isc_nrxqsets; q++, rxq++) { | ||||
#if defined(INET6) || defined(INET) | #if defined(INET6) || defined(INET) | ||||
tcp_lro_free(&rxq->ifr_lc); | if (if_getcapabilities(ctx->ifc_ifp) & IFCAP_LRO) { | ||||
if ((err = tcp_lro_init_args(&rxq->ifr_lc, ctx->ifc_ifp, | err = tcp_lro_init_args(&rxq->ifr_lc, ctx->ifc_ifp, | ||||
TCP_LRO_ENTRIES, min(1024, | TCP_LRO_ENTRIES, min(1024, | ||||
ctx->ifc_softc_ctx.isc_nrxd[rxq->ifr_fl_offset]))) != 0) { | ctx->ifc_softc_ctx.isc_nrxd[rxq->ifr_fl_offset])); | ||||
device_printf(ctx->ifc_dev, "LRO Initialization failed!\n"); | if (err != 0) { | ||||
device_printf(ctx->ifc_dev, | |||||
"LRO Initialization failed!\n"); | |||||
goto fail; | goto fail; | ||||
} | } | ||||
} | |||||
#endif | #endif | ||||
IFDI_RXQ_SETUP(ctx, rxq->ifr_id); | IFDI_RXQ_SETUP(ctx, rxq->ifr_id); | ||||
} | } | ||||
return (0); | return (0); | ||||
#if defined(INET6) || defined(INET) | #if defined(INET6) || defined(INET) | ||||
fail: | fail: | ||||
/* | /* | ||||
* Free RX software descriptors allocated so far, we will only handle | * Free LRO resources allocated so far, we will only handle | ||||
* the rings that completed, the failing case will have | * the rings that completed, the failing case will have | ||||
* cleaned up for itself. 'q' failed, so its the terminus. | * cleaned up for itself. 'q' failed, so its the terminus. | ||||
*/ | */ | ||||
rxq = ctx->ifc_rxqs; | rxq = ctx->ifc_rxqs; | ||||
for (i = 0; i < q; ++i, rxq++) { | for (i = 0; i < q; ++i, rxq++) { | ||||
iflib_rx_sds_free(rxq); | if (if_getcapabilities(ctx->ifc_ifp) & IFCAP_LRO) | ||||
rxq->ifr_cq_cidx = 0; | tcp_lro_free(&rxq->ifr_lc); | ||||
} | } | ||||
return (err); | return (err); | ||||
#endif | #endif | ||||
} | } | ||||
/********************************************************************* | /********************************************************************* | ||||
* | * | ||||
* Free all receive rings. | * Free all receive rings. | ||||
* | * | ||||
**********************************************************************/ | **********************************************************************/ | ||||
static void | static void | ||||
iflib_rx_structures_free(if_ctx_t ctx) | iflib_rx_structures_free(if_ctx_t ctx) | ||||
{ | { | ||||
iflib_rxq_t rxq = ctx->ifc_rxqs; | iflib_rxq_t rxq = ctx->ifc_rxqs; | ||||
int i; | |||||
for (int i = 0; i < ctx->ifc_softc_ctx.isc_nrxqsets; i++, rxq++) { | for (i = 0; i < ctx->ifc_softc_ctx.isc_nrxqsets; i++, rxq++) { | ||||
iflib_rx_sds_free(rxq); | iflib_rx_sds_free(rxq); | ||||
if (if_getcapabilities(ctx->ifc_ifp) & IFCAP_LRO) | |||||
tcp_lro_free(&rxq->ifr_lc); | |||||
} | } | ||||
free(ctx->ifc_rxqs, M_IFLIB); | free(ctx->ifc_rxqs, M_IFLIB); | ||||
ctx->ifc_rxqs = NULL; | ctx->ifc_rxqs = NULL; | ||||
} | } | ||||
static int | static int | ||||
iflib_qset_structures_setup(if_ctx_t ctx) | iflib_qset_structures_setup(if_ctx_t ctx) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 154 Lines • ▼ Show 20 Lines | iflib_irq_set_affinity(if_ctx_t ctx, if_irq_t irq, iflib_intr_type_t type, | ||||
int co, cpuid, err, tid; | int co, cpuid, err, tid; | ||||
dev = ctx->ifc_dev; | dev = ctx->ifc_dev; | ||||
co = ctx->ifc_sysctl_core_offset; | co = ctx->ifc_sysctl_core_offset; | ||||
if (ctx->ifc_sysctl_separate_txrx && type == IFLIB_INTR_TX) | if (ctx->ifc_sysctl_separate_txrx && type == IFLIB_INTR_TX) | ||||
co += ctx->ifc_softc_ctx.isc_nrxqsets; | co += ctx->ifc_softc_ctx.isc_nrxqsets; | ||||
cpuid = find_nth(ctx, qid + co); | cpuid = find_nth(ctx, qid + co); | ||||
tid = get_core_offset(ctx, type, qid); | tid = get_core_offset(ctx, type, qid); | ||||
MPASS(tid >= 0); | if (tid < 0) { | ||||
device_printf(dev, "get_core_offset failed\n"); | |||||
return (EOPNOTSUPP); | |||||
} | |||||
cpuid = find_close_core(cpuid, tid); | cpuid = find_close_core(cpuid, tid); | ||||
err = taskqgroup_attach_cpu(tqg, gtask, uniq, cpuid, dev, irq->ii_res, | err = taskqgroup_attach_cpu(tqg, gtask, uniq, cpuid, dev, irq->ii_res, | ||||
name); | name); | ||||
if (err) { | if (err) { | ||||
device_printf(dev, "taskqgroup_attach_cpu failed %d\n", err); | device_printf(dev, "taskqgroup_attach_cpu failed %d\n", err); | ||||
return (err); | return (err); | ||||
} | } | ||||
#ifdef notyet | #ifdef notyet | ||||
if (cpuid > ctx->ifc_cpuid_highest) | if (cpuid > ctx->ifc_cpuid_highest) | ||||
ctx->ifc_cpuid_highest = cpuid; | ctx->ifc_cpuid_highest = cpuid; | ||||
#endif | #endif | ||||
return 0; | return (0); | ||||
} | } | ||||
int | int | ||||
iflib_irq_alloc_generic(if_ctx_t ctx, if_irq_t irq, int rid, | iflib_irq_alloc_generic(if_ctx_t ctx, if_irq_t irq, int rid, | ||||
iflib_intr_type_t type, driver_filter_t *filter, | iflib_intr_type_t type, driver_filter_t *filter, | ||||
void *filter_arg, int qid, const char *name) | void *filter_arg, int qid, const char *name) | ||||
{ | { | ||||
device_t dev; | device_t dev; | ||||
▲ Show 20 Lines • Show All 43 Lines • ▼ Show 20 Lines | case IFLIB_INTR_ADMIN: | ||||
tqrid = -1; | tqrid = -1; | ||||
info = &ctx->ifc_filter_info; | info = &ctx->ifc_filter_info; | ||||
gtask = &ctx->ifc_admin_task; | gtask = &ctx->ifc_admin_task; | ||||
tqg = qgroup_if_config_tqg; | tqg = qgroup_if_config_tqg; | ||||
fn = _task_fn_admin; | fn = _task_fn_admin; | ||||
intr_fast = iflib_fast_intr_ctx; | intr_fast = iflib_fast_intr_ctx; | ||||
break; | break; | ||||
default: | default: | ||||
panic("unknown net intr type"); | device_printf(ctx->ifc_dev, "%s: unknown net intr type\n", | ||||
__func__); | |||||
return (EINVAL); | |||||
} | } | ||||
info->ifi_filter = filter; | info->ifi_filter = filter; | ||||
info->ifi_filter_arg = filter_arg; | info->ifi_filter_arg = filter_arg; | ||||
info->ifi_task = gtask; | info->ifi_task = gtask; | ||||
info->ifi_ctx = q; | info->ifi_ctx = q; | ||||
dev = ctx->ifc_dev; | dev = ctx->ifc_dev; | ||||
▲ Show 20 Lines • Show All 94 Lines • ▼ Show 20 Lines | iflib_legacy_setup(if_ctx_t ctx, driver_filter_t filter, void *filter_arg, int *rid, const char *name) | ||||
tqg = qgroup_if_io_tqg; | tqg = qgroup_if_io_tqg; | ||||
tqrid = irq->ii_rid = *rid; | tqrid = irq->ii_rid = *rid; | ||||
fn = _task_fn_rx; | fn = _task_fn_rx; | ||||
ctx->ifc_flags |= IFC_LEGACY; | ctx->ifc_flags |= IFC_LEGACY; | ||||
info->ifi_filter = filter; | info->ifi_filter = filter; | ||||
info->ifi_filter_arg = filter_arg; | info->ifi_filter_arg = filter_arg; | ||||
info->ifi_task = gtask; | info->ifi_task = gtask; | ||||
info->ifi_ctx = ctx; | info->ifi_ctx = q; | ||||
dev = ctx->ifc_dev; | dev = ctx->ifc_dev; | ||||
/* We allocate a single interrupt resource */ | /* We allocate a single interrupt resource */ | ||||
if ((err = _iflib_irq_alloc(ctx, irq, tqrid, iflib_fast_intr_ctx, NULL, info, name)) != 0) | if ((err = _iflib_irq_alloc(ctx, irq, tqrid, iflib_fast_intr_rxtx, | ||||
NULL, info, name)) != 0) | |||||
return (err); | return (err); | ||||
GROUPTASK_INIT(gtask, 0, fn, q); | GROUPTASK_INIT(gtask, 0, fn, q); | ||||
res = irq->ii_res; | res = irq->ii_res; | ||||
taskqgroup_attach(tqg, gtask, q, dev, res, name); | taskqgroup_attach(tqg, gtask, q, dev, res, name); | ||||
GROUPTASK_INIT(&txq->ift_task, 0, _task_fn_tx, txq); | GROUPTASK_INIT(&txq->ift_task, 0, _task_fn_tx, txq); | ||||
taskqgroup_attach(qgroup_if_io_tqg, &txq->ift_task, txq, dev, res, | taskqgroup_attach(qgroup_if_io_tqg, &txq->ift_task, txq, dev, res, | ||||
"tx"); | "tx"); | ||||
▲ Show 20 Lines • Show All 148 Lines • ▼ Show 20 Lines | |||||
} | } | ||||
static int | static int | ||||
iflib_msix_init(if_ctx_t ctx) | iflib_msix_init(if_ctx_t ctx) | ||||
{ | { | ||||
device_t dev = ctx->ifc_dev; | device_t dev = ctx->ifc_dev; | ||||
if_shared_ctx_t sctx = ctx->ifc_sctx; | if_shared_ctx_t sctx = ctx->ifc_sctx; | ||||
if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; | if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; | ||||
int vectors, queues, rx_queues, tx_queues, queuemsgs, msgs; | int admincnt, bar, err, iflib_num_rx_queues, iflib_num_tx_queues; | ||||
int iflib_num_tx_queues, iflib_num_rx_queues; | int msgs, queuemsgs, queues, rx_queues, tx_queues, vectors; | ||||
int err, admincnt, bar; | |||||
iflib_num_tx_queues = ctx->ifc_sysctl_ntxqs; | iflib_num_tx_queues = ctx->ifc_sysctl_ntxqs; | ||||
iflib_num_rx_queues = ctx->ifc_sysctl_nrxqs; | iflib_num_rx_queues = ctx->ifc_sysctl_nrxqs; | ||||
if (bootverbose) | if (bootverbose) | ||||
device_printf(dev, "msix_init qsets capped at %d\n", | device_printf(dev, "msix_init qsets capped at %d\n", | ||||
imax(scctx->isc_ntxqsets, scctx->isc_nrxqsets)); | imax(scctx->isc_ntxqsets, scctx->isc_nrxqsets)); | ||||
bar = ctx->ifc_softc_ctx.isc_msix_bar; | |||||
admincnt = sctx->isc_admin_intrcnt; | |||||
/* Override by tuneable */ | /* Override by tuneable */ | ||||
if (scctx->isc_disable_msix) | if (scctx->isc_disable_msix) | ||||
goto msi; | goto msi; | ||||
/* First try MSI-X */ | /* First try MSI-X */ | ||||
if ((msgs = pci_msix_count(dev)) == 0) { | if ((msgs = pci_msix_count(dev)) == 0) { | ||||
if (bootverbose) | if (bootverbose) | ||||
device_printf(dev, "MSI-X not supported or disabled\n"); | device_printf(dev, "MSI-X not supported or disabled\n"); | ||||
goto msi; | goto msi; | ||||
} | } | ||||
bar = ctx->ifc_softc_ctx.isc_msix_bar; | |||||
/* | /* | ||||
* bar == -1 => "trust me I know what I'm doing" | * bar == -1 => "trust me I know what I'm doing" | ||||
* Some drivers are for hardware that is so shoddily | * Some drivers are for hardware that is so shoddily | ||||
* documented that no one knows which bars are which | * documented that no one knows which bars are which | ||||
* so the developer has to map all bars. This hack | * so the developer has to map all bars. This hack | ||||
* allows shoddy garbage to use MSI-X in this framework. | * allows shoddy garbage to use MSI-X in this framework. | ||||
*/ | */ | ||||
if (bar != -1) { | if (bar != -1) { | ||||
ctx->ifc_msix_mem = bus_alloc_resource_any(dev, | ctx->ifc_msix_mem = bus_alloc_resource_any(dev, | ||||
SYS_RES_MEMORY, &bar, RF_ACTIVE); | SYS_RES_MEMORY, &bar, RF_ACTIVE); | ||||
if (ctx->ifc_msix_mem == NULL) { | if (ctx->ifc_msix_mem == NULL) { | ||||
device_printf(dev, "Unable to map MSI-X table\n"); | device_printf(dev, "Unable to map MSI-X table\n"); | ||||
goto msi; | goto msi; | ||||
} | } | ||||
} | } | ||||
admincnt = sctx->isc_admin_intrcnt; | |||||
#if IFLIB_DEBUG | #if IFLIB_DEBUG | ||||
/* use only 1 qset in debug mode */ | /* use only 1 qset in debug mode */ | ||||
queuemsgs = min(msgs - admincnt, 1); | queuemsgs = min(msgs - admincnt, 1); | ||||
#else | #else | ||||
queuemsgs = msgs - admincnt; | queuemsgs = msgs - admincnt; | ||||
#endif | #endif | ||||
#ifdef RSS | #ifdef RSS | ||||
queues = imin(queuemsgs, rss_getnumbuckets()); | queues = imin(queuemsgs, rss_getnumbuckets()); | ||||
Show All 35 Lines | if (tx_queues != rx_queues) | ||||
device_printf(dev, | device_printf(dev, | ||||
"queue equality override not set, capping rx_queues at %d and tx_queues at %d\n", | "queue equality override not set, capping rx_queues at %d and tx_queues at %d\n", | ||||
min(rx_queues, tx_queues), min(rx_queues, tx_queues)); | min(rx_queues, tx_queues), min(rx_queues, tx_queues)); | ||||
#endif | #endif | ||||
tx_queues = min(rx_queues, tx_queues); | tx_queues = min(rx_queues, tx_queues); | ||||
rx_queues = min(rx_queues, tx_queues); | rx_queues = min(rx_queues, tx_queues); | ||||
} | } | ||||
vectors = rx_queues + admincnt; | |||||
if (msgs < vectors) { | |||||
device_printf(dev, | |||||
"insufficient number of MSI-X vectors " | |||||
"(supported %d, need %d)\n", msgs, vectors); | |||||
goto msi; | |||||
} | |||||
device_printf(dev, "Using %d RX queues %d TX queues\n", rx_queues, | device_printf(dev, "Using %d RX queues %d TX queues\n", rx_queues, | ||||
tx_queues); | tx_queues); | ||||
msgs = vectors; | |||||
vectors = rx_queues + admincnt; | |||||
if ((err = pci_alloc_msix(dev, &vectors)) == 0) { | if ((err = pci_alloc_msix(dev, &vectors)) == 0) { | ||||
if (vectors != msgs) { | |||||
device_printf(dev, | |||||
"Unable to allocate sufficient MSI-X vectors " | |||||
"(got %d, need %d)\n", vectors, msgs); | |||||
pci_release_msi(dev); | |||||
if (bar != -1) { | |||||
bus_release_resource(dev, SYS_RES_MEMORY, bar, | |||||
ctx->ifc_msix_mem); | |||||
ctx->ifc_msix_mem = NULL; | |||||
} | |||||
goto msi; | |||||
} | |||||
device_printf(dev, "Using MSI-X interrupts with %d vectors\n", | device_printf(dev, "Using MSI-X interrupts with %d vectors\n", | ||||
vectors); | vectors); | ||||
scctx->isc_vectors = vectors; | scctx->isc_vectors = vectors; | ||||
scctx->isc_nrxqsets = rx_queues; | scctx->isc_nrxqsets = rx_queues; | ||||
scctx->isc_ntxqsets = tx_queues; | scctx->isc_ntxqsets = tx_queues; | ||||
scctx->isc_intr = IFLIB_INTR_MSIX; | scctx->isc_intr = IFLIB_INTR_MSIX; | ||||
return (vectors); | return (vectors); | ||||
} else { | } else { | ||||
device_printf(dev, | device_printf(dev, | ||||
"failed to allocate %d MSI-X vectors, err: %d - using MSI\n", | "failed to allocate %d MSI-X vectors, err: %d\n", vectors, | ||||
vectors, err); | err); | ||||
if (bar != -1) { | |||||
bus_release_resource(dev, SYS_RES_MEMORY, bar, | bus_release_resource(dev, SYS_RES_MEMORY, bar, | ||||
ctx->ifc_msix_mem); | ctx->ifc_msix_mem); | ||||
ctx->ifc_msix_mem = NULL; | ctx->ifc_msix_mem = NULL; | ||||
} | } | ||||
} | |||||
msi: | msi: | ||||
vectors = pci_msi_count(dev); | vectors = pci_msi_count(dev); | ||||
scctx->isc_nrxqsets = 1; | scctx->isc_nrxqsets = 1; | ||||
scctx->isc_ntxqsets = 1; | scctx->isc_ntxqsets = 1; | ||||
scctx->isc_vectors = vectors; | scctx->isc_vectors = vectors; | ||||
if (vectors == 1 && pci_alloc_msi(dev, &vectors) == 0) { | if (vectors == 1 && pci_alloc_msi(dev, &vectors) == 0) { | ||||
device_printf(dev,"Using an MSI interrupt\n"); | device_printf(dev,"Using an MSI interrupt\n"); | ||||
scctx->isc_intr = IFLIB_INTR_MSI; | scctx->isc_intr = IFLIB_INTR_MSI; | ||||
▲ Show 20 Lines • Show All 45 Lines • ▼ Show 20 Lines | |||||
{ | { | ||||
if_ctx_t ctx = (void *)arg1; | if_ctx_t ctx = (void *)arg1; | ||||
enum iflib_ndesc_handler type = arg2; | enum iflib_ndesc_handler type = arg2; | ||||
char buf[256] = {0}; | char buf[256] = {0}; | ||||
qidx_t *ndesc; | qidx_t *ndesc; | ||||
char *p, *next; | char *p, *next; | ||||
int nqs, rc, i; | int nqs, rc, i; | ||||
MPASS(type == IFLIB_NTXD_HANDLER || type == IFLIB_NRXD_HANDLER); | |||||
nqs = 8; | nqs = 8; | ||||
switch(type) { | switch(type) { | ||||
case IFLIB_NTXD_HANDLER: | case IFLIB_NTXD_HANDLER: | ||||
ndesc = ctx->ifc_sysctl_ntxds; | ndesc = ctx->ifc_sysctl_ntxds; | ||||
if (ctx->ifc_sctx) | if (ctx->ifc_sctx) | ||||
nqs = ctx->ifc_sctx->isc_ntxqs; | nqs = ctx->ifc_sctx->isc_ntxqs; | ||||
break; | break; | ||||
case IFLIB_NRXD_HANDLER: | case IFLIB_NRXD_HANDLER: | ||||
ndesc = ctx->ifc_sysctl_nrxds; | ndesc = ctx->ifc_sysctl_nrxds; | ||||
if (ctx->ifc_sctx) | if (ctx->ifc_sctx) | ||||
nqs = ctx->ifc_sctx->isc_nrxqs; | nqs = ctx->ifc_sctx->isc_nrxqs; | ||||
break; | break; | ||||
default: | default: | ||||
panic("unhandled type"); | printf("%s: unhandled type\n", __func__); | ||||
return (EINVAL); | |||||
} | } | ||||
if (nqs == 0) | if (nqs == 0) | ||||
nqs = 8; | nqs = 8; | ||||
for (i=0; i<8; i++) { | for (i=0; i<8; i++) { | ||||
if (i >= nqs) | if (i >= nqs) | ||||
break; | break; | ||||
if (i) | if (i) | ||||
▲ Show 20 Lines • Show All 345 Lines • Show Last 20 Lines |