Changeset View
Changeset View
Standalone View
Standalone View
head/sys/net/iflib.c
Show First 20 Lines • Show All 4,403 Lines • ▼ Show 20 Lines | if (scctx->isc_ntxqsets == 0 || (scctx->isc_ntxqsets_max && scctx->isc_ntxqsets_max < scctx->isc_ntxqsets)) | ||||
scctx->isc_ntxqsets = scctx->isc_ntxqsets_max; | scctx->isc_ntxqsets = scctx->isc_ntxqsets_max; | ||||
if (scctx->isc_nrxqsets == 0 || (scctx->isc_nrxqsets_max && scctx->isc_nrxqsets_max < scctx->isc_nrxqsets)) | if (scctx->isc_nrxqsets == 0 || (scctx->isc_nrxqsets_max && scctx->isc_nrxqsets_max < scctx->isc_nrxqsets)) | ||||
scctx->isc_nrxqsets = scctx->isc_nrxqsets_max; | scctx->isc_nrxqsets = scctx->isc_nrxqsets_max; | ||||
main_txq = (sctx->isc_flags & IFLIB_HAS_TXCQ) ? 1 : 0; | main_txq = (sctx->isc_flags & IFLIB_HAS_TXCQ) ? 1 : 0; | ||||
main_rxq = (sctx->isc_flags & IFLIB_HAS_RXCQ) ? 1 : 0; | main_rxq = (sctx->isc_flags & IFLIB_HAS_RXCQ) ? 1 : 0; | ||||
/* XXX change for per-queue sizes */ | /* XXX change for per-queue sizes */ | ||||
device_printf(dev, "using %d tx descriptors and %d rx descriptors\n", | device_printf(dev, "Using %d tx descriptors and %d rx descriptors\n", | ||||
scctx->isc_ntxd[main_txq], scctx->isc_nrxd[main_rxq]); | scctx->isc_ntxd[main_txq], scctx->isc_nrxd[main_rxq]); | ||||
for (i = 0; i < sctx->isc_nrxqs; i++) { | for (i = 0; i < sctx->isc_nrxqs; i++) { | ||||
if (!powerof2(scctx->isc_nrxd[i])) { | if (!powerof2(scctx->isc_nrxd[i])) { | ||||
/* round down instead? */ | /* round down instead? */ | ||||
device_printf(dev, "# rx descriptors must be a power of 2\n"); | device_printf(dev, "# rx descriptors must be a power of 2\n"); | ||||
err = EINVAL; | err = EINVAL; | ||||
goto fail_iflib_detach; | goto fail_iflib_detach; | ||||
} | } | ||||
} | } | ||||
▲ Show 20 Lines • Show All 45 Lines • ▼ Show 20 Lines | #endif | ||||
/* Set up cpu set. If it fails, use the set of all CPUs. */ | /* Set up cpu set. If it fails, use the set of all CPUs. */ | ||||
if (bus_get_cpus(dev, INTR_CPUS, sizeof(ctx->ifc_cpus), &ctx->ifc_cpus) != 0) { | if (bus_get_cpus(dev, INTR_CPUS, sizeof(ctx->ifc_cpus), &ctx->ifc_cpus) != 0) { | ||||
device_printf(dev, "Unable to fetch CPU list\n"); | device_printf(dev, "Unable to fetch CPU list\n"); | ||||
CPU_COPY(&all_cpus, &ctx->ifc_cpus); | CPU_COPY(&all_cpus, &ctx->ifc_cpus); | ||||
} | } | ||||
MPASS(CPU_COUNT(&ctx->ifc_cpus) > 0); | MPASS(CPU_COUNT(&ctx->ifc_cpus) > 0); | ||||
/* | /* | ||||
** Now setup MSI or MSI/X, should | ** Now set up MSI or MSI-X, should return us the number of supported | ||||
** return us the number of supported | ** vectors (will be 1 for a legacy interrupt and MSI). | ||||
** vectors. (Will be 1 for MSI) | |||||
*/ | */ | ||||
if (sctx->isc_flags & IFLIB_SKIP_MSIX) { | if (sctx->isc_flags & IFLIB_SKIP_MSIX) { | ||||
msix = scctx->isc_vectors; | msix = scctx->isc_vectors; | ||||
} else if (scctx->isc_msix_bar != 0) | } else if (scctx->isc_msix_bar != 0) | ||||
/* | /* | ||||
* The simple fact that isc_msix_bar is not 0 does not mean we | * The simple fact that isc_msix_bar is not 0 does not mean we | ||||
* we have a good value there that is known to work. | * we have a good value there that is known to work. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 180 Lines • ▼ Show 20 Lines | if (scctx->isc_ntxqsets == 0 || (scctx->isc_ntxqsets_max && scctx->isc_ntxqsets_max < scctx->isc_ntxqsets)) | ||||
scctx->isc_ntxqsets = scctx->isc_ntxqsets_max; | scctx->isc_ntxqsets = scctx->isc_ntxqsets_max; | ||||
if (scctx->isc_nrxqsets == 0 || (scctx->isc_nrxqsets_max && scctx->isc_nrxqsets_max < scctx->isc_nrxqsets)) | if (scctx->isc_nrxqsets == 0 || (scctx->isc_nrxqsets_max && scctx->isc_nrxqsets_max < scctx->isc_nrxqsets)) | ||||
scctx->isc_nrxqsets = scctx->isc_nrxqsets_max; | scctx->isc_nrxqsets = scctx->isc_nrxqsets_max; | ||||
main_txq = (sctx->isc_flags & IFLIB_HAS_TXCQ) ? 1 : 0; | main_txq = (sctx->isc_flags & IFLIB_HAS_TXCQ) ? 1 : 0; | ||||
main_rxq = (sctx->isc_flags & IFLIB_HAS_RXCQ) ? 1 : 0; | main_rxq = (sctx->isc_flags & IFLIB_HAS_RXCQ) ? 1 : 0; | ||||
/* XXX change for per-queue sizes */ | /* XXX change for per-queue sizes */ | ||||
device_printf(dev, "using %d tx descriptors and %d rx descriptors\n", | device_printf(dev, "Using %d tx descriptors and %d rx descriptors\n", | ||||
scctx->isc_ntxd[main_txq], scctx->isc_nrxd[main_rxq]); | scctx->isc_ntxd[main_txq], scctx->isc_nrxd[main_rxq]); | ||||
for (i = 0; i < sctx->isc_nrxqs; i++) { | for (i = 0; i < sctx->isc_nrxqs; i++) { | ||||
if (!powerof2(scctx->isc_nrxd[i])) { | if (!powerof2(scctx->isc_nrxd[i])) { | ||||
/* round down instead? */ | /* round down instead? */ | ||||
device_printf(dev, "# rx descriptors must be a power of 2\n"); | device_printf(dev, "# rx descriptors must be a power of 2\n"); | ||||
err = EINVAL; | err = EINVAL; | ||||
goto fail_iflib_detach; | goto fail_iflib_detach; | ||||
} | } | ||||
} | } | ||||
▲ Show 20 Lines • Show All 239 Lines • ▼ Show 20 Lines | #endif | ||||
free(ctx, M_IFLIB); | free(ctx, M_IFLIB); | ||||
return (0); | return (0); | ||||
} | } | ||||
static void | static void | ||||
iflib_free_intr_mem(if_ctx_t ctx) | iflib_free_intr_mem(if_ctx_t ctx) | ||||
{ | { | ||||
if (ctx->ifc_softc_ctx.isc_intr != IFLIB_INTR_LEGACY) { | |||||
pci_release_msi(ctx->ifc_dev); | |||||
} | |||||
if (ctx->ifc_softc_ctx.isc_intr != IFLIB_INTR_MSIX) { | if (ctx->ifc_softc_ctx.isc_intr != IFLIB_INTR_MSIX) { | ||||
iflib_irq_free(ctx, &ctx->ifc_legacy_irq); | iflib_irq_free(ctx, &ctx->ifc_legacy_irq); | ||||
} | } | ||||
if (ctx->ifc_softc_ctx.isc_intr != IFLIB_INTR_LEGACY) { | |||||
pci_release_msi(ctx->ifc_dev); | |||||
} | |||||
if (ctx->ifc_msix_mem != NULL) { | if (ctx->ifc_msix_mem != NULL) { | ||||
bus_release_resource(ctx->ifc_dev, SYS_RES_MEMORY, | bus_release_resource(ctx->ifc_dev, SYS_RES_MEMORY, | ||||
ctx->ifc_softc_ctx.isc_msix_bar, ctx->ifc_msix_mem); | rman_get_rid(ctx->ifc_msix_mem), ctx->ifc_msix_mem); | ||||
ctx->ifc_msix_mem = NULL; | ctx->ifc_msix_mem = NULL; | ||||
} | } | ||||
} | } | ||||
int | int | ||||
iflib_device_detach(device_t dev) | iflib_device_detach(device_t dev) | ||||
{ | { | ||||
if_ctx_t ctx = device_get_softc(dev); | if_ctx_t ctx = device_get_softc(dev); | ||||
▲ Show 20 Lines • Show All 810 Lines • ▼ Show 20 Lines | iflib_softirq_alloc_generic(if_ctx_t ctx, if_irq_t irq, iflib_intr_type_t type, void *arg, int qid, const char *name) | ||||
else { | else { | ||||
taskqgroup_attach(tqg, gtask, q, irq_num, name); | taskqgroup_attach(tqg, gtask, q, irq_num, name); | ||||
} | } | ||||
} | } | ||||
void | void | ||||
iflib_irq_free(if_ctx_t ctx, if_irq_t irq) | iflib_irq_free(if_ctx_t ctx, if_irq_t irq) | ||||
{ | { | ||||
if (irq->ii_tag) | if (irq->ii_tag) | ||||
bus_teardown_intr(ctx->ifc_dev, irq->ii_res, irq->ii_tag); | bus_teardown_intr(ctx->ifc_dev, irq->ii_res, irq->ii_tag); | ||||
if (irq->ii_res) | if (irq->ii_res) | ||||
bus_release_resource(ctx->ifc_dev, SYS_RES_IRQ, irq->ii_rid, irq->ii_res); | bus_release_resource(ctx->ifc_dev, SYS_RES_IRQ, | ||||
rman_get_rid(irq->ii_res), irq->ii_res); | |||||
} | } | ||||
static int | static int | ||||
iflib_legacy_setup(if_ctx_t ctx, driver_filter_t filter, void *filter_arg, int *rid, const char *name) | iflib_legacy_setup(if_ctx_t ctx, driver_filter_t filter, void *filter_arg, int *rid, const char *name) | ||||
{ | { | ||||
iflib_txq_t txq = ctx->ifc_txqs; | iflib_txq_t txq = ctx->ifc_txqs; | ||||
iflib_rxq_t rxq = ctx->ifc_rxqs; | iflib_rxq_t rxq = ctx->ifc_rxqs; | ||||
if_irq_t irq = &ctx->ifc_legacy_irq; | if_irq_t irq = &ctx->ifc_legacy_irq; | ||||
▲ Show 20 Lines • Show All 177 Lines • ▼ Show 20 Lines | iflib_msix_init(if_ctx_t ctx) | ||||
if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; | if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; | ||||
int vectors, queues, rx_queues, tx_queues, queuemsgs, msgs; | int vectors, queues, rx_queues, tx_queues, queuemsgs, msgs; | ||||
int iflib_num_tx_queues, iflib_num_rx_queues; | int iflib_num_tx_queues, iflib_num_rx_queues; | ||||
int err, admincnt, bar; | int err, admincnt, bar; | ||||
iflib_num_tx_queues = ctx->ifc_sysctl_ntxqs; | iflib_num_tx_queues = ctx->ifc_sysctl_ntxqs; | ||||
iflib_num_rx_queues = ctx->ifc_sysctl_nrxqs; | iflib_num_rx_queues = ctx->ifc_sysctl_nrxqs; | ||||
device_printf(dev, "msix_init qsets capped at %d\n", imax(scctx->isc_ntxqsets, scctx->isc_nrxqsets)); | if (bootverbose) | ||||
device_printf(dev, "msix_init qsets capped at %d\n", | |||||
imax(scctx->isc_ntxqsets, scctx->isc_nrxqsets)); | |||||
bar = ctx->ifc_softc_ctx.isc_msix_bar; | bar = ctx->ifc_softc_ctx.isc_msix_bar; | ||||
admincnt = sctx->isc_admin_intrcnt; | admincnt = sctx->isc_admin_intrcnt; | ||||
/* Override by tuneable */ | /* Override by tuneable */ | ||||
if (scctx->isc_disable_msix) | if (scctx->isc_disable_msix) | ||||
goto msi; | goto msi; | ||||
/* First try MSI-X */ | |||||
if ((msgs = pci_msix_count(dev)) == 0) { | |||||
if (bootverbose) | |||||
device_printf(dev, "MSI-X not supported or disabled\n"); | |||||
goto msi; | |||||
} | |||||
/* | /* | ||||
* bar == -1 => "trust me I know what I'm doing" | * bar == -1 => "trust me I know what I'm doing" | ||||
* Some drivers are for hardware that is so shoddily | * Some drivers are for hardware that is so shoddily | ||||
* documented that no one knows which bars are which | * documented that no one knows which bars are which | ||||
* so the developer has to map all bars. This hack | * so the developer has to map all bars. This hack | ||||
* allows shoddy garbage to use msix in this framework. | * allows shoddy garbage to use MSI-X in this framework. | ||||
*/ | */ | ||||
if (bar != -1) { | if (bar != -1) { | ||||
ctx->ifc_msix_mem = bus_alloc_resource_any(dev, | ctx->ifc_msix_mem = bus_alloc_resource_any(dev, | ||||
SYS_RES_MEMORY, &bar, RF_ACTIVE); | SYS_RES_MEMORY, &bar, RF_ACTIVE); | ||||
if (ctx->ifc_msix_mem == NULL) { | if (ctx->ifc_msix_mem == NULL) { | ||||
/* May not be enabled */ | device_printf(dev, "Unable to map MSI-X table\n"); | ||||
device_printf(dev, "Unable to map MSIX table \n"); | |||||
goto msi; | goto msi; | ||||
} | } | ||||
} | } | ||||
/* First try MSI/X */ | |||||
if ((msgs = pci_msix_count(dev)) == 0) { /* system has msix disabled */ | |||||
device_printf(dev, "System has MSIX disabled \n"); | |||||
bus_release_resource(dev, SYS_RES_MEMORY, | |||||
bar, ctx->ifc_msix_mem); | |||||
ctx->ifc_msix_mem = NULL; | |||||
goto msi; | |||||
} | |||||
#if IFLIB_DEBUG | #if IFLIB_DEBUG | ||||
/* use only 1 qset in debug mode */ | /* use only 1 qset in debug mode */ | ||||
queuemsgs = min(msgs - admincnt, 1); | queuemsgs = min(msgs - admincnt, 1); | ||||
#else | #else | ||||
queuemsgs = msgs - admincnt; | queuemsgs = msgs - admincnt; | ||||
#endif | #endif | ||||
#ifdef RSS | #ifdef RSS | ||||
queues = imin(queuemsgs, rss_getnumbuckets()); | queues = imin(queuemsgs, rss_getnumbuckets()); | ||||
#else | #else | ||||
queues = queuemsgs; | queues = queuemsgs; | ||||
#endif | #endif | ||||
queues = imin(CPU_COUNT(&ctx->ifc_cpus), queues); | queues = imin(CPU_COUNT(&ctx->ifc_cpus), queues); | ||||
device_printf(dev, "pxm cpus: %d queue msgs: %d admincnt: %d\n", | if (bootverbose) | ||||
device_printf(dev, | |||||
"intr CPUs: %d queue msgs: %d admincnt: %d\n", | |||||
CPU_COUNT(&ctx->ifc_cpus), queuemsgs, admincnt); | CPU_COUNT(&ctx->ifc_cpus), queuemsgs, admincnt); | ||||
#ifdef RSS | #ifdef RSS | ||||
/* If we're doing RSS, clamp at the number of RSS buckets */ | /* If we're doing RSS, clamp at the number of RSS buckets */ | ||||
if (queues > rss_getnumbuckets()) | if (queues > rss_getnumbuckets()) | ||||
queues = rss_getnumbuckets(); | queues = rss_getnumbuckets(); | ||||
#endif | #endif | ||||
if (iflib_num_rx_queues > 0 && iflib_num_rx_queues < queuemsgs - admincnt) | if (iflib_num_rx_queues > 0 && iflib_num_rx_queues < queuemsgs - admincnt) | ||||
rx_queues = iflib_num_rx_queues; | rx_queues = iflib_num_rx_queues; | ||||
else | else | ||||
Show All 19 Lines | if (tx_queues != rx_queues) | ||||
device_printf(dev, | device_printf(dev, | ||||
"queue equality override not set, capping rx_queues at %d and tx_queues at %d\n", | "queue equality override not set, capping rx_queues at %d and tx_queues at %d\n", | ||||
min(rx_queues, tx_queues), min(rx_queues, tx_queues)); | min(rx_queues, tx_queues), min(rx_queues, tx_queues)); | ||||
#endif | #endif | ||||
tx_queues = min(rx_queues, tx_queues); | tx_queues = min(rx_queues, tx_queues); | ||||
rx_queues = min(rx_queues, tx_queues); | rx_queues = min(rx_queues, tx_queues); | ||||
} | } | ||||
device_printf(dev, "using %d rx queues %d tx queues \n", rx_queues, tx_queues); | device_printf(dev, "Using %d rx queues %d tx queues\n", | ||||
rx_queues, tx_queues); | |||||
vectors = rx_queues + admincnt; | vectors = rx_queues + admincnt; | ||||
if ((err = pci_alloc_msix(dev, &vectors)) == 0) { | if ((err = pci_alloc_msix(dev, &vectors)) == 0) { | ||||
device_printf(dev, "Using MSIX interrupts with %d vectors\n", vectors); | device_printf(dev, "Using MSI-X interrupts with %d vectors\n", | ||||
vectors); | |||||
scctx->isc_vectors = vectors; | scctx->isc_vectors = vectors; | ||||
scctx->isc_nrxqsets = rx_queues; | scctx->isc_nrxqsets = rx_queues; | ||||
scctx->isc_ntxqsets = tx_queues; | scctx->isc_ntxqsets = tx_queues; | ||||
scctx->isc_intr = IFLIB_INTR_MSIX; | scctx->isc_intr = IFLIB_INTR_MSIX; | ||||
return (vectors); | return (vectors); | ||||
} else { | } else { | ||||
device_printf(dev, | device_printf(dev, | ||||
"failed to allocate %d msix vectors, err: %d - using MSI\n", vectors, err); | "failed to allocate %d MSI-X vectors, err: %d - using MSI\n", | ||||
vectors, err); | |||||
bus_release_resource(dev, SYS_RES_MEMORY, bar, | bus_release_resource(dev, SYS_RES_MEMORY, bar, | ||||
ctx->ifc_msix_mem); | ctx->ifc_msix_mem); | ||||
ctx->ifc_msix_mem = NULL; | ctx->ifc_msix_mem = NULL; | ||||
} | } | ||||
msi: | msi: | ||||
vectors = pci_msi_count(dev); | vectors = pci_msi_count(dev); | ||||
scctx->isc_nrxqsets = 1; | scctx->isc_nrxqsets = 1; | ||||
scctx->isc_ntxqsets = 1; | scctx->isc_ntxqsets = 1; | ||||
▲ Show 20 Lines • Show All 119 Lines • ▼ Show 20 Lines | iflib_add_device_sysctl_pre(if_ctx_t ctx) | ||||
SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "override_nrxqs", | SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "override_nrxqs", | ||||
CTLFLAG_RWTUN, &ctx->ifc_sysctl_nrxqs, 0, | CTLFLAG_RWTUN, &ctx->ifc_sysctl_nrxqs, 0, | ||||
"# of rxqs to use, 0 => use default #"); | "# of rxqs to use, 0 => use default #"); | ||||
SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "override_qs_enable", | SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "override_qs_enable", | ||||
CTLFLAG_RWTUN, &ctx->ifc_sysctl_qs_eq_override, 0, | CTLFLAG_RWTUN, &ctx->ifc_sysctl_qs_eq_override, 0, | ||||
"permit #txq != #rxq"); | "permit #txq != #rxq"); | ||||
SYSCTL_ADD_INT(ctx_list, oid_list, OID_AUTO, "disable_msix", | SYSCTL_ADD_INT(ctx_list, oid_list, OID_AUTO, "disable_msix", | ||||
CTLFLAG_RWTUN, &ctx->ifc_softc_ctx.isc_disable_msix, 0, | CTLFLAG_RWTUN, &ctx->ifc_softc_ctx.isc_disable_msix, 0, | ||||
"disable MSIX (default 0)"); | "disable MSI-X (default 0)"); | ||||
SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "rx_budget", | SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "rx_budget", | ||||
CTLFLAG_RWTUN, &ctx->ifc_sysctl_rx_budget, 0, | CTLFLAG_RWTUN, &ctx->ifc_sysctl_rx_budget, 0, | ||||
"set the rx budget"); | "set the rx budget"); | ||||
SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "tx_abdicate", | SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "tx_abdicate", | ||||
CTLFLAG_RWTUN, &ctx->ifc_sysctl_tx_abdicate, 0, | CTLFLAG_RWTUN, &ctx->ifc_sysctl_tx_abdicate, 0, | ||||
"cause tx to abdicate instead of running to completion"); | "cause tx to abdicate instead of running to completion"); | ||||
/* XXX change for per-queue sizes */ | /* XXX change for per-queue sizes */ | ||||
▲ Show 20 Lines • Show All 286 Lines • Show Last 20 Lines |