Changeset View
Changeset View
Standalone View
Standalone View
sys/net/iflib.c
Show First 20 Lines • Show All 179 Lines • ▼ Show 20 Lines | |||||
struct grouptask ifc_vflr_task; | struct grouptask ifc_vflr_task; | ||||
struct iflib_filter_info ifc_filter_info; | struct iflib_filter_info ifc_filter_info; | ||||
struct ifmedia ifc_media; | struct ifmedia ifc_media; | ||||
struct sysctl_oid *ifc_sysctl_node; | struct sysctl_oid *ifc_sysctl_node; | ||||
uint16_t ifc_sysctl_ntxqs; | uint16_t ifc_sysctl_ntxqs; | ||||
uint16_t ifc_sysctl_nrxqs; | uint16_t ifc_sysctl_nrxqs; | ||||
uint16_t ifc_sysctl_qs_eq_override; | uint16_t ifc_sysctl_qs_eq_override; | ||||
uint16_t ifc_cpuid_highest; | |||||
qidx_t ifc_sysctl_ntxds[8]; | qidx_t ifc_sysctl_ntxds[8]; | ||||
qidx_t ifc_sysctl_nrxds[8]; | qidx_t ifc_sysctl_nrxds[8]; | ||||
struct if_txrx ifc_txrx; | struct if_txrx ifc_txrx; | ||||
#define isc_txd_encap ifc_txrx.ift_txd_encap | #define isc_txd_encap ifc_txrx.ift_txd_encap | ||||
#define isc_txd_flush ifc_txrx.ift_txd_flush | #define isc_txd_flush ifc_txrx.ift_txd_flush | ||||
#define isc_txd_credits_update ifc_txrx.ift_txd_credits_update | #define isc_txd_credits_update ifc_txrx.ift_txd_credits_update | ||||
#define isc_rxd_available ifc_txrx.ift_rxd_available | #define isc_rxd_available ifc_txrx.ift_rxd_available | ||||
#define isc_rxd_pkt_get ifc_txrx.ift_rxd_pkt_get | #define isc_rxd_pkt_get ifc_txrx.ift_rxd_pkt_get | ||||
#define isc_rxd_refill ifc_txrx.ift_rxd_refill | #define isc_rxd_refill ifc_txrx.ift_rxd_refill | ||||
#define isc_rxd_flush ifc_txrx.ift_rxd_flush | #define isc_rxd_flush ifc_txrx.ift_rxd_flush | ||||
#define isc_rxd_refill ifc_txrx.ift_rxd_refill | #define isc_rxd_refill ifc_txrx.ift_rxd_refill | ||||
#define isc_rxd_refill ifc_txrx.ift_rxd_refill | #define isc_rxd_refill ifc_txrx.ift_rxd_refill | ||||
#define isc_legacy_intr ifc_txrx.ift_legacy_intr | #define isc_legacy_intr ifc_txrx.ift_legacy_intr | ||||
eventhandler_tag ifc_vlan_attach_event; | eventhandler_tag ifc_vlan_attach_event; | ||||
eventhandler_tag ifc_vlan_detach_event; | eventhandler_tag ifc_vlan_detach_event; | ||||
uint8_t ifc_mac[ETHER_ADDR_LEN]; | uint8_t ifc_mac[ETHER_ADDR_LEN]; | ||||
char ifc_mtx_name[16]; | char ifc_mtx_name[16]; | ||||
LIST_ENTRY(iflib_ctx) ifc_next; | |||||
}; | }; | ||||
static LIST_HEAD(ctx_head, iflib_ctx) ctx_list; | |||||
static struct mtx ctx_list_lock; | |||||
TASKQGROUP_DEFINE(if_io, mp_ncpus, 1, true, PI_NET); | |||||
TASKQGROUP_DEFINE(if_config, 1, 1, false, PI_SOFT); | |||||
static void | |||||
iflib_ctx_apply(void (*fn)(if_ctx_t ctx, void *arg), void *arg) | |||||
{ | |||||
if_ctx_t ctx; | |||||
mtx_lock(&ctx_list_lock); | |||||
LIST_FOREACH(ctx, &ctx_list, ifc_next) { | |||||
(fn)(ctx, arg); | |||||
} | |||||
mtx_unlock(&ctx_list_lock); | |||||
} | |||||
static void | |||||
_iflib_cpuid_highest(if_ctx_t ctx, void *arg) { | |||||
int *cpuid = arg; | |||||
if (*cpuid < ctx->ifc_cpuid_highest) | |||||
*cpuid = ctx->ifc_cpuid_highest; | |||||
} | |||||
static int | |||||
iflib_cpuid_highest(void) | |||||
{ | |||||
int cpuid = 0; | |||||
iflib_ctx_apply(_iflib_cpuid_highest, &cpuid); | |||||
return (cpuid); | |||||
} | |||||
static void | |||||
iflib_ctx_insert(if_ctx_t ctx) | |||||
{ | |||||
mtx_lock(&ctx_list_lock); | |||||
LIST_INSERT_HEAD(&ctx_list, ctx, ifc_next); | |||||
mtx_unlock(&ctx_list_lock); | |||||
} | |||||
static void | |||||
iflib_ctx_remove(if_ctx_t ctx) | |||||
{ | |||||
int max_cpuid_prev, max_cpuid_new; | |||||
max_cpuid_prev = iflib_cpuid_highest(); | |||||
mtx_lock(&ctx_list_lock); | |||||
LIST_REMOVE(ctx, ifc_next); | |||||
mtx_unlock(&ctx_list_lock); | |||||
max_cpuid_new = max(1, iflib_cpuid_highest()); | |||||
if (max_cpuid_new < max_cpuid_prev) { | |||||
taskqgroup_adjust(qgroup_if_io, max_cpuid_new, 1, true, PI_NET); | |||||
} | |||||
} | |||||
void * | void * | ||||
iflib_get_softc(if_ctx_t ctx) | iflib_get_softc(if_ctx_t ctx) | ||||
{ | { | ||||
return (ctx->ifc_softc); | return (ctx->ifc_softc); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 332 Lines • ▼ Show 20 Lines | |||||
}; | }; | ||||
DECLARE_MODULE(iflib, iflib_moduledata, SI_SUB_INIT_IF, SI_ORDER_ANY); | DECLARE_MODULE(iflib, iflib_moduledata, SI_SUB_INIT_IF, SI_ORDER_ANY); | ||||
MODULE_VERSION(iflib, 1); | MODULE_VERSION(iflib, 1); | ||||
MODULE_DEPEND(iflib, pci, 1, 1, 1); | MODULE_DEPEND(iflib, pci, 1, 1, 1); | ||||
MODULE_DEPEND(iflib, ether, 1, 1, 1); | MODULE_DEPEND(iflib, ether, 1, 1, 1); | ||||
TASKQGROUP_DEFINE(if_io, mp_ncpus, 1, true, PI_NET); | |||||
TASKQGROUP_DEFINE(if_config, 1, 1, false, PI_SOFT); | |||||
#ifndef IFLIB_DEBUG_COUNTERS | #ifndef IFLIB_DEBUG_COUNTERS | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
#define IFLIB_DEBUG_COUNTERS 1 | #define IFLIB_DEBUG_COUNTERS 1 | ||||
#else | #else | ||||
#define IFLIB_DEBUG_COUNTERS 0 | #define IFLIB_DEBUG_COUNTERS 0 | ||||
#endif /* !INVARIANTS */ | #endif /* !INVARIANTS */ | ||||
#endif | #endif | ||||
▲ Show 20 Lines • Show All 1,982 Lines • ▼ Show 20 Lines | |||||
if ((err = iflib_netmap_attach(ctx))) { | if ((err = iflib_netmap_attach(ctx))) { | ||||
device_printf(ctx->ifc_dev, "netmap attach failed: %d\n", err); | device_printf(ctx->ifc_dev, "netmap attach failed: %d\n", err); | ||||
goto fail_detach; | goto fail_detach; | ||||
} | } | ||||
*ctxp = ctx; | *ctxp = ctx; | ||||
if_setgetcounterfn(ctx->ifc_ifp, iflib_if_get_counter); | if_setgetcounterfn(ctx->ifc_ifp, iflib_if_get_counter); | ||||
iflib_add_device_sysctl_post(ctx); | iflib_add_device_sysctl_post(ctx); | ||||
iflib_ctx_insert(ctx); | |||||
ctx->ifc_flags |= IFC_INIT_DONE; | ctx->ifc_flags |= IFC_INIT_DONE; | ||||
return (0); | return (0); | ||||
fail_detach: | fail_detach: | ||||
ether_ifdetach(ctx->ifc_ifp); | ether_ifdetach(ctx->ifc_ifp); | ||||
fail_intr_free: | fail_intr_free: | ||||
if (scctx->isc_intr == IFLIB_INTR_MSIX || scctx->isc_intr == IFLIB_INTR_MSI) | if (scctx->isc_intr == IFLIB_INTR_MSIX || scctx->isc_intr == IFLIB_INTR_MSI) | ||||
pci_release_msi(ctx->ifc_dev); | pci_release_msi(ctx->ifc_dev); | ||||
fail_queues: | fail_queues: | ||||
▲ Show 20 Lines • Show All 88 Lines • ▼ Show 20 Lines | |||||
bus_generic_detach(dev); | bus_generic_detach(dev); | ||||
if_free(ifp); | if_free(ifp); | ||||
iflib_tx_structures_free(ctx); | iflib_tx_structures_free(ctx); | ||||
iflib_rx_structures_free(ctx); | iflib_rx_structures_free(ctx); | ||||
if (ctx->ifc_flags & IFC_SC_ALLOCATED) | if (ctx->ifc_flags & IFC_SC_ALLOCATED) | ||||
free(ctx->ifc_softc, M_IFLIB); | free(ctx->ifc_softc, M_IFLIB); | ||||
iflib_ctx_remove(ctx); | |||||
free(ctx, M_IFLIB); | free(ctx, M_IFLIB); | ||||
return (0); | return (0); | ||||
} | } | ||||
int | int | ||||
iflib_device_detach(device_t dev) | iflib_device_detach(device_t dev) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 79 Lines • ▼ Show 20 Lines | |||||
} | } | ||||
/********************************************************************* | /********************************************************************* | ||||
* | * | ||||
* MODULE FUNCTION DEFINITIONS | * MODULE FUNCTION DEFINITIONS | ||||
* | * | ||||
**********************************************************************/ | **********************************************************************/ | ||||
/* | |||||
* - Start a fast taskqueue thread for each core | |||||
* - Start a taskqueue for control operations | |||||
*/ | |||||
static int | static int | ||||
iflib_module_init(void) | iflib_module_init(void) | ||||
{ | { | ||||
LIST_INIT(&ctx_list); | |||||
mtx_init(&ctx_list_lock, "ctx list", NULL, MTX_DEF); | |||||
return (0); | return (0); | ||||
} | } | ||||
static int | static int | ||||
iflib_module_event_handler(module_t mod, int what, void *arg) | iflib_module_event_handler(module_t mod, int what, void *arg) | ||||
{ | { | ||||
int err; | int err; | ||||
▲ Show 20 Lines • Show All 520 Lines • ▼ Show 20 Lines | |||||
device_printf(ctx->ifc_dev, "_iflib_irq_alloc failed %d\n", err); | device_printf(ctx->ifc_dev, "_iflib_irq_alloc failed %d\n", err); | ||||
return (err); | return (err); | ||||
} | } | ||||
if (type == IFLIB_INTR_ADMIN) | if (type == IFLIB_INTR_ADMIN) | ||||
return (0); | return (0); | ||||
if (tqrid != -1) { | if (tqrid != -1) { | ||||
cpuid = find_nth(ctx, &cpus, qid); | cpuid = find_nth(ctx, &cpus, qid); | ||||
taskqgroup_attach_cpu(tqg, gtask, q, cpuid, irq->ii_rid, name); | err = taskqgroup_attach_cpu(tqg, gtask, q, cpuid, irq->ii_rid, name); | ||||
if (err) { | |||||
device_printf(ctx->ifc_dev, "taskqgroup_attach_cpu failed %d\n", err); | |||||
return (err); | |||||
} | |||||
if (cpuid > ctx->ifc_cpuid_highest) | |||||
ctx->ifc_cpuid_highest = cpuid; | |||||
MPASS(gtask->gt_taskqueue != NULL); | |||||
} else { | } else { | ||||
taskqgroup_attach(tqg, gtask, q, tqrid, name); | taskqgroup_attach(tqg, gtask, q, tqrid, name); | ||||
} | } | ||||
return (0); | return (0); | ||||
} | } | ||||
void | void | ||||
▲ Show 20 Lines • Show All 673 Lines • Show Last 20 Lines |