Changeset View
Standalone View
sys/net/iflib.c
Show First 20 Lines • Show All 182 Lines • ▼ Show 20 Lines | struct iflib_ctx { | ||||
struct ifmedia ifc_media; | struct ifmedia ifc_media; | ||||
struct sysctl_oid *ifc_sysctl_node; | struct sysctl_oid *ifc_sysctl_node; | ||||
uint16_t ifc_sysctl_ntxqs; | uint16_t ifc_sysctl_ntxqs; | ||||
uint16_t ifc_sysctl_nrxqs; | uint16_t ifc_sysctl_nrxqs; | ||||
uint16_t ifc_sysctl_qs_eq_override; | uint16_t ifc_sysctl_qs_eq_override; | ||||
uint16_t ifc_sysctl_rx_budget; | uint16_t ifc_sysctl_rx_budget; | ||||
uint16_t ifc_sysctl_tx_abdicate; | uint16_t ifc_sysctl_tx_abdicate; | ||||
uint16_t ifc_sysctl_core_offset; | |||||
uint8_t ifc_sysctl_separate_txrx; | |||||
marius: #define<tab> according to style(9) | |||||
qidx_t ifc_sysctl_ntxds[8]; | qidx_t ifc_sysctl_ntxds[8]; | ||||
qidx_t ifc_sysctl_nrxds[8]; | qidx_t ifc_sysctl_nrxds[8]; | ||||
struct if_txrx ifc_txrx; | struct if_txrx ifc_txrx; | ||||
#define isc_txd_encap ifc_txrx.ift_txd_encap | #define isc_txd_encap ifc_txrx.ift_txd_encap | ||||
#define isc_txd_flush ifc_txrx.ift_txd_flush | #define isc_txd_flush ifc_txrx.ift_txd_flush | ||||
#define isc_txd_credits_update ifc_txrx.ift_txd_credits_update | #define isc_txd_credits_update ifc_txrx.ift_txd_credits_update | ||||
#define isc_rxd_available ifc_txrx.ift_rxd_available | #define isc_rxd_available ifc_txrx.ift_rxd_available | ||||
▲ Show 20 Lines • Show All 521 Lines • ▼ Show 20 Lines | |||||
static void iflib_ifmp_purge(iflib_txq_t txq); | static void iflib_ifmp_purge(iflib_txq_t txq); | ||||
static void _iflib_pre_assert(if_softc_ctx_t scctx); | static void _iflib_pre_assert(if_softc_ctx_t scctx); | ||||
static void iflib_if_init_locked(if_ctx_t ctx); | static void iflib_if_init_locked(if_ctx_t ctx); | ||||
static void iflib_free_intr_mem(if_ctx_t ctx); | static void iflib_free_intr_mem(if_ctx_t ctx); | ||||
#ifndef __NO_STRICT_ALIGNMENT | #ifndef __NO_STRICT_ALIGNMENT | ||||
static struct mbuf * iflib_fixup_rx(struct mbuf *m); | static struct mbuf * iflib_fixup_rx(struct mbuf *m); | ||||
#endif | #endif | ||||
static SLIST_HEAD(cpu_offset_list, cpu_offset) cpu_offsets = | |||||
SLIST_HEAD_INITIALIZER(cpu_offsets); | |||||
struct cpu_offset { | |||||
SLIST_ENTRY(cpu_offset) entries; | |||||
cpuset_t set; | |||||
uint16_t offset; | |||||
}; | |||||
static struct mtx cpu_offset_mtx; | |||||
MTX_SYSINIT(iflib_cpu_offset, &cpu_offset_mtx, "iflib_cpu_offset lock", MTX_DEF); | |||||
NETDUMP_DEFINE(iflib); | NETDUMP_DEFINE(iflib); | ||||
#ifdef DEV_NETMAP | #ifdef DEV_NETMAP | ||||
#include <sys/selinfo.h> | #include <sys/selinfo.h> | ||||
#include <net/netmap.h> | #include <net/netmap.h> | ||||
#include <dev/netmap/netmap_kern.h> | #include <dev/netmap/netmap_kern.h> | ||||
MODULE_DEPEND(iflib, netmap, 1, 1, 1); | MODULE_DEPEND(iflib, netmap, 1, 1, 1); | ||||
▲ Show 20 Lines • Show All 3,625 Lines • ▼ Show 20 Lines | for (i = 0; i < sctx->isc_ntxqs; i++) { | ||||
if (scctx->isc_ntxd[i] > sctx->isc_ntxd_max[i]) { | if (scctx->isc_ntxd[i] > sctx->isc_ntxd_max[i]) { | ||||
device_printf(dev, "ntxd%d: %d greater than ntxd_max %d - resetting to max\n", | device_printf(dev, "ntxd%d: %d greater than ntxd_max %d - resetting to max\n", | ||||
i, scctx->isc_ntxd[i], sctx->isc_ntxd_max[i]); | i, scctx->isc_ntxd[i], sctx->isc_ntxd_max[i]); | ||||
scctx->isc_ntxd[i] = sctx->isc_ntxd_max[i]; | scctx->isc_ntxd[i] = sctx->isc_ntxd_max[i]; | ||||
} | } | ||||
} | } | ||||
} | } | ||||
static uint16_t get_ctx_core_offset(if_ctx_t ctx) | |||||
mariusUnsubmitted Done Inline ActionsPlease place the function type on its own line as suggested by style(9) (so searching for the implementation via grep ^get_ctx_core_offset works). marius: Please place the function type on its own line as suggested by style(9) (so searching for the… | |||||
{ | |||||
if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; | |||||
uint16_t qc; | |||||
struct cpu_offset *op; | |||||
uint16_t ret = ctx->ifc_sysctl_core_offset; | |||||
char kename[128]; | |||||
mariusUnsubmitted Done Inline ActionsPlease sort variables according to style(9). marius: Please sort variables according to style(9). | |||||
/* XXX This is terrible... */ | |||||
shurdAuthorUnsubmitted Done Inline ActionsJust to highlight this... this is, in fact, terrible. I didn't want to use hints to ensure iflib is in the name, and didn't find any other immediately obvious method to have a non-zero default tunable value. Something less terrible would be great. shurd: Just to highlight this... this is, in fact, terrible. I didn't want to use hints to ensure… | |||||
mariusUnsubmitted Done Inline ActionsHum, what happens if you simply set ctx->ifc_sysctl_core_offset to 0xffff before calling SYSCTL_ADD_U16(9)? It doesn't look like sysctl(9) has special code to initialize dynamically added SYSCTLs to 0 and in the usual case, their default value of 0 is simply preserved. marius: Hum, what happens if you simply set ctx->ifc_sysctl_core_offset to 0xffff before calling… | |||||
sprintf(kename, "dev.%s.%d.iflib.core_offset", | |||||
device_get_name(ctx->ifc_dev), device_get_unit(ctx->ifc_dev)); | |||||
if (testenv(kename) == 0 && ret == 0) | |||||
ret = 0xffff; | |||||
if (ret != 0xffff) | |||||
return ret; | |||||
mariusUnsubmitted Done Inline ActionsPlease enclose the returned valued in parentheses a la style(9). marius: Please enclose the returned valued in parentheses a la style(9). | |||||
if (ctx->ifc_sysctl_separate_txrx) | |||||
qc = scctx->isc_ntxqsets + scctx->isc_nrxqsets; | |||||
else | |||||
qc = max(scctx->isc_ntxqsets, scctx->isc_nrxqsets); | |||||
mtx_lock(&cpu_offset_mtx); | |||||
SLIST_FOREACH(op, &cpu_offsets, entries) { | |||||
if (CPU_CMP(&ctx->ifc_cpus, &op->set) == 0) { | |||||
ret = op->offset; | |||||
op->offset += qc; | |||||
break; | |||||
} | |||||
} | |||||
if (ret == 0xffff) { | |||||
ret = 0; | |||||
op = malloc(sizeof(struct cpu_offset), M_IFLIB, | |||||
mariusUnsubmitted Done Inline ActionsLooks like this memory will be leaked when a device goes away. Similarly, the whole cpu_offset entry probably should be removed from the list again in iflib_device_deregister(). marius: Looks like this memory will be leaked when a device goes away. Similarly, the whole cpu_offset… | |||||
shurdAuthorUnsubmitted Done Inline ActionsThis allocation is shared by all iflib drivers. I've added a comment in iflib_module_event_handler() in case MOD_UNLOAD is ever change to return zero. shurd: This allocation is shared by all iflib drivers. I've added a comment in… | |||||
mariusUnsubmitted Done Inline ActionsHum, can't you just add a reference count to the cpu_offest entries and once the count for a particular CPU set drops to 0 on iflib_device_deregister(), remove that entry from the list and free its memory? marius: Hum, can't you just add a reference count to the cpu_offest entries and once the count for a… | |||||
shurdAuthorUnsubmitted Done Inline ActionsIt feels a bit weird to reference-count it since the driver doesn't actually have a reference, but at least this prevents it from leaking in the future. shurd: It feels a bit weird to reference-count it since the driver doesn't actually have a reference… | |||||
M_NOWAIT | M_ZERO); | |||||
Done Inline Actionsstyle(9): "Closing and opening braces go on the same line as the else." marius: style(9): "Closing and opening braces go on the same line as the else." | |||||
MPASS(op != NULL); | |||||
mariusUnsubmitted Done Inline ActionsIt doesn't look like a good idea to allocate memory using M_NOWAIT and then to immediately panic if malloc(9) failed, especially if the code below looks like it would be handling the allocation failure gracefully. marius: It doesn't look like a good idea to allocate memory using M_NOWAIT and then to immediately… | |||||
if (op != NULL) { | |||||
op->offset = qc; | |||||
CPU_COPY(&ctx->ifc_cpus, &op->set); | |||||
SLIST_INSERT_HEAD(&cpu_offsets, op, entries); | |||||
} | |||||
} | |||||
mtx_unlock(&cpu_offset_mtx); | |||||
return ret; | |||||
mariusUnsubmitted Done Inline Actionsreturn (ret); marius: return (ret); | |||||
} | |||||
int | int | ||||
iflib_device_register(device_t dev, void *sc, if_shared_ctx_t sctx, if_ctx_t *ctxp) | iflib_device_register(device_t dev, void *sc, if_shared_ctx_t sctx, if_ctx_t *ctxp) | ||||
{ | { | ||||
int err, rid, msix; | int err, rid, msix; | ||||
if_ctx_t ctx; | if_ctx_t ctx; | ||||
if_t ifp; | if_t ifp; | ||||
if_softc_ctx_t scctx; | if_softc_ctx_t scctx; | ||||
int i; | int i; | ||||
▲ Show 20 Lines • Show All 136 Lines • ▼ Show 20 Lines | if ((err = iflib_queues_alloc(ctx))) { | ||||
device_printf(dev, "Unable to allocate queue memory\n"); | device_printf(dev, "Unable to allocate queue memory\n"); | ||||
goto fail_intr_free; | goto fail_intr_free; | ||||
} | } | ||||
if ((err = iflib_qset_structures_setup(ctx))) | if ((err = iflib_qset_structures_setup(ctx))) | ||||
goto fail_queues; | goto fail_queues; | ||||
/* | /* | ||||
* Now that we know how many queues there are, get the core offset. | |||||
*/ | |||||
ctx->ifc_sysctl_core_offset = get_ctx_core_offset(ctx); | |||||
/* | |||||
* Group taskqueues aren't properly set up until SMP is started, | * Group taskqueues aren't properly set up until SMP is started, | ||||
* so we disable interrupts until we can handle them post | * so we disable interrupts until we can handle them post | ||||
* SI_SUB_SMP. | * SI_SUB_SMP. | ||||
* | * | ||||
* XXX: disabling interrupts doesn't actually work, at least for | * XXX: disabling interrupts doesn't actually work, at least for | ||||
* the non-MSI case. When they occur before SI_SUB_SMP completes, | * the non-MSI case. When they occur before SI_SUB_SMP completes, | ||||
* we do null handling and depend on this not causing too large an | * we do null handling and depend on this not causing too large an | ||||
* interrupt storm. | * interrupt storm. | ||||
▲ Show 20 Lines • Show All 1,024 Lines • ▼ Show 20 Lines | find_child_with_core(int cpu, struct cpu_group *grp) | ||||
return -1; | return -1; | ||||
} | } | ||||
/* | /* | ||||
* Find the nth "close" core to the specified core | * Find the nth "close" core to the specified core | ||||
* "close" is defined as the deepest level that shares | * "close" is defined as the deepest level that shares | ||||
* at least an L2 cache. With threads, this will be | * at least an L2 cache. With threads, this will be | ||||
* threads on the same core. If the sahred cache is L3 | * threads on the same core. If the shared cache is L3 | ||||
* or higher, simply returns the same core. | * or higher, simply returns the same core. | ||||
*/ | */ | ||||
static int | static int | ||||
find_close_core(int cpu, int core_offset) | find_close_core(int cpu, int core_offset) | ||||
{ | { | ||||
struct cpu_group *grp; | struct cpu_group *grp; | ||||
int i; | int i; | ||||
int fcpu; | int fcpu; | ||||
▲ Show 20 Lines • Show All 67 Lines • ▼ Show 20 Lines | |||||
/* Just to avoid copy/paste */ | /* Just to avoid copy/paste */ | ||||
static inline int | static inline int | ||||
iflib_irq_set_affinity(if_ctx_t ctx, if_irq_t irq, iflib_intr_type_t type, | iflib_irq_set_affinity(if_ctx_t ctx, if_irq_t irq, iflib_intr_type_t type, | ||||
int qid, struct grouptask *gtask, struct taskqgroup *tqg, void *uniq, | int qid, struct grouptask *gtask, struct taskqgroup *tqg, void *uniq, | ||||
const char *name) | const char *name) | ||||
{ | { | ||||
device_t dev; | device_t dev; | ||||
int err, cpuid, tid; | int co, cpuid, err, tid; | ||||
dev = ctx->ifc_dev; | dev = ctx->ifc_dev; | ||||
cpuid = find_nth(ctx, qid); | co = ctx->ifc_sysctl_core_offset; | ||||
if (ctx->ifc_sysctl_separate_txrx && type == IFLIB_INTR_TX) | |||||
co += ctx->ifc_softc_ctx.isc_nrxqsets; | |||||
cpuid = find_nth(ctx, qid + co); | |||||
tid = get_core_offset(ctx, type, qid); | tid = get_core_offset(ctx, type, qid); | ||||
MPASS(tid >= 0); | MPASS(tid >= 0); | ||||
cpuid = find_close_core(cpuid, tid); | cpuid = find_close_core(cpuid, tid); | ||||
err = taskqgroup_attach_cpu(tqg, gtask, uniq, cpuid, dev, irq->ii_res, | err = taskqgroup_attach_cpu(tqg, gtask, uniq, cpuid, dev, irq->ii_res, | ||||
name); | name); | ||||
if (err) { | if (err) { | ||||
device_printf(dev, "taskqgroup_attach_cpu failed %d\n", err); | device_printf(dev, "taskqgroup_attach_cpu failed %d\n", err); | ||||
return (err); | return (err); | ||||
▲ Show 20 Lines • Show All 585 Lines • ▼ Show 20 Lines | SYSCTL_ADD_INT(ctx_list, oid_list, OID_AUTO, "disable_msix", | ||||
CTLFLAG_RWTUN, &ctx->ifc_softc_ctx.isc_disable_msix, 0, | CTLFLAG_RWTUN, &ctx->ifc_softc_ctx.isc_disable_msix, 0, | ||||
"disable MSI-X (default 0)"); | "disable MSI-X (default 0)"); | ||||
SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "rx_budget", | SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "rx_budget", | ||||
CTLFLAG_RWTUN, &ctx->ifc_sysctl_rx_budget, 0, | CTLFLAG_RWTUN, &ctx->ifc_sysctl_rx_budget, 0, | ||||
"set the rx budget"); | "set the rx budget"); | ||||
SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "tx_abdicate", | SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "tx_abdicate", | ||||
CTLFLAG_RWTUN, &ctx->ifc_sysctl_tx_abdicate, 0, | CTLFLAG_RWTUN, &ctx->ifc_sysctl_tx_abdicate, 0, | ||||
"cause tx to abdicate instead of running to completion"); | "cause tx to abdicate instead of running to completion"); | ||||
SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "core_offset", | |||||
CTLFLAG_RDTUN, &ctx->ifc_sysctl_core_offset, 0, | |||||
"offset to start using cores at"); | |||||
SYSCTL_ADD_U8(ctx_list, oid_list, OID_AUTO, "separate_txrx", | |||||
CTLFLAG_RDTUN, &ctx->ifc_sysctl_separate_txrx, 0, | |||||
"use separate cores for TX and RX"); | |||||
/* XXX change for per-queue sizes */ | /* XXX change for per-queue sizes */ | ||||
SYSCTL_ADD_PROC(ctx_list, oid_list, OID_AUTO, "override_ntxds", | SYSCTL_ADD_PROC(ctx_list, oid_list, OID_AUTO, "override_ntxds", | ||||
CTLTYPE_STRING|CTLFLAG_RWTUN, ctx, IFLIB_NTXD_HANDLER, | CTLTYPE_STRING|CTLFLAG_RWTUN, ctx, IFLIB_NTXD_HANDLER, | ||||
mp_ndesc_handler, "A", | mp_ndesc_handler, "A", | ||||
"list of # of tx descriptors to use, 0 = use default #"); | "list of # of tx descriptors to use, 0 = use default #"); | ||||
SYSCTL_ADD_PROC(ctx_list, oid_list, OID_AUTO, "override_nrxds", | SYSCTL_ADD_PROC(ctx_list, oid_list, OID_AUTO, "override_nrxds", | ||||
CTLTYPE_STRING|CTLFLAG_RWTUN, ctx, IFLIB_NRXD_HANDLER, | CTLTYPE_STRING|CTLFLAG_RWTUN, ctx, IFLIB_NRXD_HANDLER, | ||||
▲ Show 20 Lines • Show All 280 Lines • Show Last 20 Lines |
#define<tab> according to style(9)