Changeset View
Changeset View
Standalone View
Standalone View
head/sys/net/iflib.c
Show First 20 Lines • Show All 183 Lines • ▼ Show 20 Lines | struct iflib_ctx { | ||||
struct ifmedia ifc_media; | struct ifmedia ifc_media; | ||||
struct sysctl_oid *ifc_sysctl_node; | struct sysctl_oid *ifc_sysctl_node; | ||||
uint16_t ifc_sysctl_ntxqs; | uint16_t ifc_sysctl_ntxqs; | ||||
uint16_t ifc_sysctl_nrxqs; | uint16_t ifc_sysctl_nrxqs; | ||||
uint16_t ifc_sysctl_qs_eq_override; | uint16_t ifc_sysctl_qs_eq_override; | ||||
uint16_t ifc_sysctl_rx_budget; | uint16_t ifc_sysctl_rx_budget; | ||||
uint16_t ifc_sysctl_tx_abdicate; | uint16_t ifc_sysctl_tx_abdicate; | ||||
uint16_t ifc_sysctl_core_offset; | |||||
#define CORE_OFFSET_UNSPECIFIED 0xffff | |||||
uint8_t ifc_sysctl_separate_txrx; | |||||
qidx_t ifc_sysctl_ntxds[8]; | qidx_t ifc_sysctl_ntxds[8]; | ||||
qidx_t ifc_sysctl_nrxds[8]; | qidx_t ifc_sysctl_nrxds[8]; | ||||
struct if_txrx ifc_txrx; | struct if_txrx ifc_txrx; | ||||
#define isc_txd_encap ifc_txrx.ift_txd_encap | #define isc_txd_encap ifc_txrx.ift_txd_encap | ||||
#define isc_txd_flush ifc_txrx.ift_txd_flush | #define isc_txd_flush ifc_txrx.ift_txd_flush | ||||
#define isc_txd_credits_update ifc_txrx.ift_txd_credits_update | #define isc_txd_credits_update ifc_txrx.ift_txd_credits_update | ||||
#define isc_rxd_available ifc_txrx.ift_rxd_available | #define isc_rxd_available ifc_txrx.ift_rxd_available | ||||
▲ Show 20 Lines • Show All 518 Lines • ▼ Show 20 Lines | |||||
static void iflib_ifmp_purge(iflib_txq_t txq); | static void iflib_ifmp_purge(iflib_txq_t txq); | ||||
static void _iflib_pre_assert(if_softc_ctx_t scctx); | static void _iflib_pre_assert(if_softc_ctx_t scctx); | ||||
static void iflib_if_init_locked(if_ctx_t ctx); | static void iflib_if_init_locked(if_ctx_t ctx); | ||||
static void iflib_free_intr_mem(if_ctx_t ctx); | static void iflib_free_intr_mem(if_ctx_t ctx); | ||||
#ifndef __NO_STRICT_ALIGNMENT | #ifndef __NO_STRICT_ALIGNMENT | ||||
static struct mbuf * iflib_fixup_rx(struct mbuf *m); | static struct mbuf * iflib_fixup_rx(struct mbuf *m); | ||||
#endif | #endif | ||||
static SLIST_HEAD(cpu_offset_list, cpu_offset) cpu_offsets = | |||||
SLIST_HEAD_INITIALIZER(cpu_offsets); | |||||
struct cpu_offset { | |||||
SLIST_ENTRY(cpu_offset) entries; | |||||
cpuset_t set; | |||||
unsigned int refcount; | |||||
uint16_t offset; | |||||
}; | |||||
static struct mtx cpu_offset_mtx; | |||||
MTX_SYSINIT(iflib_cpu_offset, &cpu_offset_mtx, "iflib_cpu_offset lock", | |||||
MTX_DEF); | |||||
NETDUMP_DEFINE(iflib); | NETDUMP_DEFINE(iflib); | ||||
#ifdef DEV_NETMAP | #ifdef DEV_NETMAP | ||||
#include <sys/selinfo.h> | #include <sys/selinfo.h> | ||||
#include <net/netmap.h> | #include <net/netmap.h> | ||||
#include <dev/netmap/netmap_kern.h> | #include <dev/netmap/netmap_kern.h> | ||||
MODULE_DEPEND(iflib, netmap, 1, 1, 1); | MODULE_DEPEND(iflib, netmap, 1, 1, 1); | ||||
▲ Show 20 Lines • Show All 3,722 Lines • ▼ Show 20 Lines | iflib_rem_pfil(if_ctx_t ctx) | ||||
rxq = ctx->ifc_rxqs; | rxq = ctx->ifc_rxqs; | ||||
pfil = rxq->pfil; | pfil = rxq->pfil; | ||||
for (i = 0; i < NRXQSETS(ctx); i++, rxq++) { | for (i = 0; i < NRXQSETS(ctx); i++, rxq++) { | ||||
rxq->pfil = NULL; | rxq->pfil = NULL; | ||||
} | } | ||||
pfil_head_unregister(pfil); | pfil_head_unregister(pfil); | ||||
} | } | ||||
static uint16_t | |||||
get_ctx_core_offset(if_ctx_t ctx) | |||||
{ | |||||
if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; | |||||
struct cpu_offset *op; | |||||
uint16_t qc; | |||||
uint16_t ret = ctx->ifc_sysctl_core_offset; | |||||
if (ret != CORE_OFFSET_UNSPECIFIED) | |||||
return (ret); | |||||
if (ctx->ifc_sysctl_separate_txrx) | |||||
qc = scctx->isc_ntxqsets + scctx->isc_nrxqsets; | |||||
else | |||||
qc = max(scctx->isc_ntxqsets, scctx->isc_nrxqsets); | |||||
mtx_lock(&cpu_offset_mtx); | |||||
SLIST_FOREACH(op, &cpu_offsets, entries) { | |||||
if (CPU_CMP(&ctx->ifc_cpus, &op->set) == 0) { | |||||
ret = op->offset; | |||||
op->offset += qc; | |||||
MPASS(op->refcount < UINT_MAX); | |||||
op->refcount++; | |||||
break; | |||||
} | |||||
} | |||||
if (ret == CORE_OFFSET_UNSPECIFIED) { | |||||
ret = 0; | |||||
op = malloc(sizeof(struct cpu_offset), M_IFLIB, | |||||
M_NOWAIT | M_ZERO); | |||||
if (op == NULL) { | |||||
device_printf(ctx->ifc_dev, | |||||
"allocation for cpu offset failed.\n"); | |||||
} else { | |||||
op->offset = qc; | |||||
op->refcount = 1; | |||||
CPU_COPY(&ctx->ifc_cpus, &op->set); | |||||
SLIST_INSERT_HEAD(&cpu_offsets, op, entries); | |||||
} | |||||
} | |||||
mtx_unlock(&cpu_offset_mtx); | |||||
return (ret); | |||||
} | |||||
static void | |||||
unref_ctx_core_offset(if_ctx_t ctx) | |||||
{ | |||||
struct cpu_offset *op, *top; | |||||
mtx_lock(&cpu_offset_mtx); | |||||
SLIST_FOREACH_SAFE(op, &cpu_offsets, entries, top) { | |||||
if (CPU_CMP(&ctx->ifc_cpus, &op->set) == 0) { | |||||
MPASS(op->refcount > 0); | |||||
op->refcount--; | |||||
if (op->refcount == 0) { | |||||
SLIST_REMOVE(&cpu_offsets, op, cpu_offset, entries); | |||||
free(op, M_IFLIB); | |||||
} | |||||
break; | |||||
} | |||||
} | |||||
mtx_unlock(&cpu_offset_mtx); | |||||
} | |||||
int | int | ||||
iflib_device_register(device_t dev, void *sc, if_shared_ctx_t sctx, if_ctx_t *ctxp) | iflib_device_register(device_t dev, void *sc, if_shared_ctx_t sctx, if_ctx_t *ctxp) | ||||
{ | { | ||||
int err, rid, msix; | int err, rid, msix; | ||||
if_ctx_t ctx; | if_ctx_t ctx; | ||||
if_t ifp; | if_t ifp; | ||||
if_softc_ctx_t scctx; | if_softc_ctx_t scctx; | ||||
int i; | int i; | ||||
▲ Show 20 Lines • Show All 136 Lines • ▼ Show 20 Lines | if ((err = iflib_queues_alloc(ctx))) { | ||||
device_printf(dev, "Unable to allocate queue memory\n"); | device_printf(dev, "Unable to allocate queue memory\n"); | ||||
goto fail_intr_free; | goto fail_intr_free; | ||||
} | } | ||||
if ((err = iflib_qset_structures_setup(ctx))) | if ((err = iflib_qset_structures_setup(ctx))) | ||||
goto fail_queues; | goto fail_queues; | ||||
/* | /* | ||||
* Now that we know how many queues there are, get the core offset. | |||||
*/ | |||||
ctx->ifc_sysctl_core_offset = get_ctx_core_offset(ctx); | |||||
/* | |||||
* Group taskqueues aren't properly set up until SMP is started, | * Group taskqueues aren't properly set up until SMP is started, | ||||
* so we disable interrupts until we can handle them post | * so we disable interrupts until we can handle them post | ||||
* SI_SUB_SMP. | * SI_SUB_SMP. | ||||
* | * | ||||
* XXX: disabling interrupts doesn't actually work, at least for | * XXX: disabling interrupts doesn't actually work, at least for | ||||
* the non-MSI case. When they occur before SI_SUB_SMP completes, | * the non-MSI case. When they occur before SI_SUB_SMP completes, | ||||
* we do null handling and depend on this not causing too large an | * we do null handling and depend on this not causing too large an | ||||
* interrupt storm. | * interrupt storm. | ||||
▲ Show 20 Lines • Show All 408 Lines • ▼ Show 20 Lines | #endif | ||||
bus_generic_detach(dev); | bus_generic_detach(dev); | ||||
if_free(ifp); | if_free(ifp); | ||||
iflib_tx_structures_free(ctx); | iflib_tx_structures_free(ctx); | ||||
iflib_rx_structures_free(ctx); | iflib_rx_structures_free(ctx); | ||||
if (ctx->ifc_flags & IFC_SC_ALLOCATED) | if (ctx->ifc_flags & IFC_SC_ALLOCATED) | ||||
free(ctx->ifc_softc, M_IFLIB); | free(ctx->ifc_softc, M_IFLIB); | ||||
unref_ctx_core_offset(ctx); | |||||
STATE_LOCK_DESTROY(ctx); | STATE_LOCK_DESTROY(ctx); | ||||
free(ctx, M_IFLIB); | free(ctx, M_IFLIB); | ||||
return (0); | return (0); | ||||
} | } | ||||
static void | static void | ||||
iflib_free_intr_mem(if_ctx_t ctx) | iflib_free_intr_mem(if_ctx_t ctx) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 602 Lines • ▼ Show 20 Lines | find_child_with_core(int cpu, struct cpu_group *grp) | ||||
return -1; | return -1; | ||||
} | } | ||||
/* | /* | ||||
* Find the nth "close" core to the specified core | * Find the nth "close" core to the specified core | ||||
* "close" is defined as the deepest level that shares | * "close" is defined as the deepest level that shares | ||||
* at least an L2 cache. With threads, this will be | * at least an L2 cache. With threads, this will be | ||||
* threads on the same core. If the sahred cache is L3 | * threads on the same core. If the shared cache is L3 | ||||
* or higher, simply returns the same core. | * or higher, simply returns the same core. | ||||
*/ | */ | ||||
static int | static int | ||||
find_close_core(int cpu, int core_offset) | find_close_core(int cpu, int core_offset) | ||||
{ | { | ||||
struct cpu_group *grp; | struct cpu_group *grp; | ||||
int i; | int i; | ||||
int fcpu; | int fcpu; | ||||
▲ Show 20 Lines • Show All 67 Lines • ▼ Show 20 Lines | |||||
/* Just to avoid copy/paste */ | /* Just to avoid copy/paste */ | ||||
static inline int | static inline int | ||||
iflib_irq_set_affinity(if_ctx_t ctx, if_irq_t irq, iflib_intr_type_t type, | iflib_irq_set_affinity(if_ctx_t ctx, if_irq_t irq, iflib_intr_type_t type, | ||||
int qid, struct grouptask *gtask, struct taskqgroup *tqg, void *uniq, | int qid, struct grouptask *gtask, struct taskqgroup *tqg, void *uniq, | ||||
const char *name) | const char *name) | ||||
{ | { | ||||
device_t dev; | device_t dev; | ||||
int err, cpuid, tid; | int co, cpuid, err, tid; | ||||
dev = ctx->ifc_dev; | dev = ctx->ifc_dev; | ||||
cpuid = find_nth(ctx, qid); | co = ctx->ifc_sysctl_core_offset; | ||||
if (ctx->ifc_sysctl_separate_txrx && type == IFLIB_INTR_TX) | |||||
co += ctx->ifc_softc_ctx.isc_nrxqsets; | |||||
cpuid = find_nth(ctx, qid + co); | |||||
tid = get_core_offset(ctx, type, qid); | tid = get_core_offset(ctx, type, qid); | ||||
MPASS(tid >= 0); | MPASS(tid >= 0); | ||||
cpuid = find_close_core(cpuid, tid); | cpuid = find_close_core(cpuid, tid); | ||||
err = taskqgroup_attach_cpu(tqg, gtask, uniq, cpuid, dev, irq->ii_res, | err = taskqgroup_attach_cpu(tqg, gtask, uniq, cpuid, dev, irq->ii_res, | ||||
name); | name); | ||||
if (err) { | if (err) { | ||||
device_printf(dev, "taskqgroup_attach_cpu failed %d\n", err); | device_printf(dev, "taskqgroup_attach_cpu failed %d\n", err); | ||||
return (err); | return (err); | ||||
▲ Show 20 Lines • Show All 585 Lines • ▼ Show 20 Lines | SYSCTL_ADD_INT(ctx_list, oid_list, OID_AUTO, "disable_msix", | ||||
CTLFLAG_RWTUN, &ctx->ifc_softc_ctx.isc_disable_msix, 0, | CTLFLAG_RWTUN, &ctx->ifc_softc_ctx.isc_disable_msix, 0, | ||||
"disable MSI-X (default 0)"); | "disable MSI-X (default 0)"); | ||||
SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "rx_budget", | SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "rx_budget", | ||||
CTLFLAG_RWTUN, &ctx->ifc_sysctl_rx_budget, 0, | CTLFLAG_RWTUN, &ctx->ifc_sysctl_rx_budget, 0, | ||||
"set the rx budget"); | "set the rx budget"); | ||||
SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "tx_abdicate", | SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "tx_abdicate", | ||||
CTLFLAG_RWTUN, &ctx->ifc_sysctl_tx_abdicate, 0, | CTLFLAG_RWTUN, &ctx->ifc_sysctl_tx_abdicate, 0, | ||||
"cause tx to abdicate instead of running to completion"); | "cause tx to abdicate instead of running to completion"); | ||||
ctx->ifc_sysctl_core_offset = CORE_OFFSET_UNSPECIFIED; | |||||
SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "core_offset", | |||||
CTLFLAG_RDTUN, &ctx->ifc_sysctl_core_offset, 0, | |||||
"offset to start using cores at"); | |||||
SYSCTL_ADD_U8(ctx_list, oid_list, OID_AUTO, "separate_txrx", | |||||
CTLFLAG_RDTUN, &ctx->ifc_sysctl_separate_txrx, 0, | |||||
"use separate cores for TX and RX"); | |||||
/* XXX change for per-queue sizes */ | /* XXX change for per-queue sizes */ | ||||
SYSCTL_ADD_PROC(ctx_list, oid_list, OID_AUTO, "override_ntxds", | SYSCTL_ADD_PROC(ctx_list, oid_list, OID_AUTO, "override_ntxds", | ||||
CTLTYPE_STRING|CTLFLAG_RWTUN, ctx, IFLIB_NTXD_HANDLER, | CTLTYPE_STRING|CTLFLAG_RWTUN, ctx, IFLIB_NTXD_HANDLER, | ||||
mp_ndesc_handler, "A", | mp_ndesc_handler, "A", | ||||
"list of # of tx descriptors to use, 0 = use default #"); | "list of # of tx descriptors to use, 0 = use default #"); | ||||
SYSCTL_ADD_PROC(ctx_list, oid_list, OID_AUTO, "override_nrxds", | SYSCTL_ADD_PROC(ctx_list, oid_list, OID_AUTO, "override_nrxds", | ||||
CTLTYPE_STRING|CTLFLAG_RWTUN, ctx, IFLIB_NRXD_HANDLER, | CTLTYPE_STRING|CTLFLAG_RWTUN, ctx, IFLIB_NRXD_HANDLER, | ||||
▲ Show 20 Lines • Show All 280 Lines • Show Last 20 Lines |