Changeset View
Changeset View
Standalone View
Standalone View
head/sys/net/iflib.c
Show First 20 Lines • Show All 95 Lines • ▼ Show 20 Lines | |||||
#include <sys/memdesc.h> | #include <sys/memdesc.h> | ||||
#include <machine/bus.h> | #include <machine/bus.h> | ||||
#include <machine/md_var.h> | #include <machine/md_var.h> | ||||
#include <machine/specialreg.h> | #include <machine/specialreg.h> | ||||
#include <x86/include/busdma_impl.h> | #include <x86/include/busdma_impl.h> | ||||
#include <x86/iommu/busdma_dmar.h> | #include <x86/iommu/busdma_dmar.h> | ||||
#endif | #endif | ||||
#ifdef PCI_IOV | |||||
#include <dev/pci/pci_iov.h> | |||||
#endif | |||||
#include <sys/bitstring.h> | #include <sys/bitstring.h> | ||||
/* | /* | ||||
* enable accounting of every mbuf as it comes in to and goes out of | * enable accounting of every mbuf as it comes in to and goes out of | ||||
* iflib's software descriptor references | * iflib's software descriptor references | ||||
*/ | */ | ||||
#define MEMORY_LOGGING 0 | #define MEMORY_LOGGING 0 | ||||
/* | /* | ||||
* Enable mbuf vectors for compressing long mbuf chains | * Enable mbuf vectors for compressing long mbuf chains | ||||
Show All 40 Lines | typedef struct iflib_filter_info { | ||||
driver_filter_t *ifi_filter; | driver_filter_t *ifi_filter; | ||||
void *ifi_filter_arg; | void *ifi_filter_arg; | ||||
struct grouptask *ifi_task; | struct grouptask *ifi_task; | ||||
void *ifi_ctx; | void *ifi_ctx; | ||||
} *iflib_filter_info_t; | } *iflib_filter_info_t; | ||||
struct iflib_ctx { | struct iflib_ctx { | ||||
KOBJ_FIELDS; | KOBJ_FIELDS; | ||||
/* | /* | ||||
* Pointer to hardware driver's softc | * Pointer to hardware driver's softc | ||||
*/ | */ | ||||
void *ifc_softc; | void *ifc_softc; | ||||
device_t ifc_dev; | device_t ifc_dev; | ||||
if_t ifc_ifp; | if_t ifc_ifp; | ||||
cpuset_t ifc_cpus; | cpuset_t ifc_cpus; | ||||
if_shared_ctx_t ifc_sctx; | if_shared_ctx_t ifc_sctx; | ||||
struct if_softc_ctx ifc_softc_ctx; | struct if_softc_ctx ifc_softc_ctx; | ||||
struct sx ifc_ctx_sx; | struct sx ifc_ctx_sx; | ||||
struct mtx ifc_state_mtx; | struct mtx ifc_state_mtx; | ||||
uint16_t ifc_nhwtxqs; | uint16_t ifc_nhwtxqs; | ||||
iflib_txq_t ifc_txqs; | iflib_txq_t ifc_txqs; | ||||
iflib_rxq_t ifc_rxqs; | iflib_rxq_t ifc_rxqs; | ||||
uint32_t ifc_if_flags; | uint32_t ifc_if_flags; | ||||
uint32_t ifc_flags; | uint32_t ifc_flags; | ||||
uint32_t ifc_max_fl_buf_size; | uint32_t ifc_max_fl_buf_size; | ||||
int ifc_in_detach; | |||||
int ifc_link_state; | int ifc_link_state; | ||||
int ifc_link_irq; | int ifc_link_irq; | ||||
int ifc_watchdog_events; | int ifc_watchdog_events; | ||||
struct cdev *ifc_led_dev; | struct cdev *ifc_led_dev; | ||||
struct resource *ifc_msix_mem; | struct resource *ifc_msix_mem; | ||||
struct if_irq ifc_legacy_irq; | struct if_irq ifc_legacy_irq; | ||||
▲ Show 20 Lines • Show All 59 Lines • ▼ Show 20 Lines | |||||
uint32_t | uint32_t | ||||
iflib_get_flags(if_ctx_t ctx) | iflib_get_flags(if_ctx_t ctx) | ||||
{ | { | ||||
return (ctx->ifc_flags); | return (ctx->ifc_flags); | ||||
} | } | ||||
void | void | ||||
iflib_set_detach(if_ctx_t ctx) | |||||
{ | |||||
ctx->ifc_in_detach = 1; | |||||
} | |||||
void | |||||
iflib_set_mac(if_ctx_t ctx, uint8_t mac[ETHER_ADDR_LEN]) | iflib_set_mac(if_ctx_t ctx, uint8_t mac[ETHER_ADDR_LEN]) | ||||
{ | { | ||||
bcopy(mac, ctx->ifc_mac, ETHER_ADDR_LEN); | bcopy(mac, ctx->ifc_mac, ETHER_ADDR_LEN); | ||||
} | } | ||||
if_softc_ctx_t | if_softc_ctx_t | ||||
iflib_get_softc_ctx(if_ctx_t ctx) | iflib_get_softc_ctx(if_ctx_t ctx) | ||||
▲ Show 20 Lines • Show All 295 Lines • ▼ Show 20 Lines | |||||
#define STATE_UNLOCK(ctx) mtx_unlock(&(ctx)->ifc_state_mtx) | #define STATE_UNLOCK(ctx) mtx_unlock(&(ctx)->ifc_state_mtx) | ||||
#define STATE_LOCK_DESTROY(ctx) mtx_destroy(&(ctx)->ifc_state_mtx) | #define STATE_LOCK_DESTROY(ctx) mtx_destroy(&(ctx)->ifc_state_mtx) | ||||
#define CALLOUT_LOCK(txq) mtx_lock(&txq->ift_mtx) | #define CALLOUT_LOCK(txq) mtx_lock(&txq->ift_mtx) | ||||
#define CALLOUT_UNLOCK(txq) mtx_unlock(&txq->ift_mtx) | #define CALLOUT_UNLOCK(txq) mtx_unlock(&txq->ift_mtx) | ||||
void | |||||
iflib_set_detach(if_ctx_t ctx) | |||||
{ | |||||
STATE_LOCK(ctx); | |||||
ctx->ifc_flags |= IFC_IN_DETACH; | |||||
STATE_UNLOCK(ctx); | |||||
} | |||||
/* Our boot-time initialization hook */ | /* Our boot-time initialization hook */ | ||||
static int iflib_module_event_handler(module_t, int, void *); | static int iflib_module_event_handler(module_t, int, void *); | ||||
static moduledata_t iflib_moduledata = { | static moduledata_t iflib_moduledata = { | ||||
"iflib", | "iflib", | ||||
iflib_module_event_handler, | iflib_module_event_handler, | ||||
NULL | NULL | ||||
▲ Show 20 Lines • Show All 151 Lines • ▼ Show 20 Lines | |||||
#endif | #endif | ||||
static int iflib_register(if_ctx_t); | static int iflib_register(if_ctx_t); | ||||
static void iflib_init_locked(if_ctx_t ctx); | static void iflib_init_locked(if_ctx_t ctx); | ||||
static void iflib_add_device_sysctl_pre(if_ctx_t ctx); | static void iflib_add_device_sysctl_pre(if_ctx_t ctx); | ||||
static void iflib_add_device_sysctl_post(if_ctx_t ctx); | static void iflib_add_device_sysctl_post(if_ctx_t ctx); | ||||
static void iflib_ifmp_purge(iflib_txq_t txq); | static void iflib_ifmp_purge(iflib_txq_t txq); | ||||
static void _iflib_pre_assert(if_softc_ctx_t scctx); | static void _iflib_pre_assert(if_softc_ctx_t scctx); | ||||
static void iflib_if_init_locked(if_ctx_t ctx); | static void iflib_if_init_locked(if_ctx_t ctx); | ||||
static void iflib_free_intr_mem(if_ctx_t ctx); | |||||
#ifndef __NO_STRICT_ALIGNMENT | #ifndef __NO_STRICT_ALIGNMENT | ||||
static struct mbuf * iflib_fixup_rx(struct mbuf *m); | static struct mbuf * iflib_fixup_rx(struct mbuf *m); | ||||
#endif | #endif | ||||
NETDUMP_DEFINE(iflib); | NETDUMP_DEFINE(iflib); | ||||
#ifdef DEV_NETMAP | #ifdef DEV_NETMAP | ||||
#include <sys/selinfo.h> | #include <sys/selinfo.h> | ||||
▲ Show 20 Lines • Show All 1,318 Lines • ▼ Show 20 Lines | #endif | ||||
MPASS(fl->ifl_credits <= fl->ifl_size); | MPASS(fl->ifl_credits <= fl->ifl_size); | ||||
MPASS(reclaimable == delta); | MPASS(reclaimable == delta); | ||||
if (reclaimable > 0) | if (reclaimable > 0) | ||||
_iflib_fl_refill(ctx, fl, min(max, reclaimable)); | _iflib_fl_refill(ctx, fl, min(max, reclaimable)); | ||||
} | } | ||||
uint8_t | |||||
iflib_in_detach(if_ctx_t ctx) | |||||
{ | |||||
bool in_detach; | |||||
STATE_LOCK(ctx); | |||||
in_detach = !!(ctx->ifc_flags & IFC_IN_DETACH); | |||||
STATE_UNLOCK(ctx); | |||||
return (in_detach); | |||||
} | |||||
static void | static void | ||||
iflib_fl_bufs_free(iflib_fl_t fl) | iflib_fl_bufs_free(iflib_fl_t fl) | ||||
{ | { | ||||
iflib_dma_info_t idi = fl->ifl_ifdi; | iflib_dma_info_t idi = fl->ifl_ifdi; | ||||
uint32_t i; | uint32_t i; | ||||
for (i = 0; i < fl->ifl_size; i++) { | for (i = 0; i < fl->ifl_size; i++) { | ||||
struct mbuf **sd_m = &fl->ifl_sds.ifsd_m[i]; | struct mbuf **sd_m = &fl->ifl_sds.ifsd_m[i]; | ||||
uint8_t *sd_flags = &fl->ifl_sds.ifsd_flags[i]; | uint8_t *sd_flags = &fl->ifl_sds.ifsd_flags[i]; | ||||
caddr_t *sd_cl = &fl->ifl_sds.ifsd_cl[i]; | caddr_t *sd_cl = &fl->ifl_sds.ifsd_cl[i]; | ||||
if (*sd_flags & RX_SW_DESC_INUSE) { | if (*sd_flags & RX_SW_DESC_INUSE) { | ||||
if (fl->ifl_sds.ifsd_map != NULL) { | if (fl->ifl_sds.ifsd_map != NULL) { | ||||
bus_dmamap_t sd_map = fl->ifl_sds.ifsd_map[i]; | bus_dmamap_t sd_map = fl->ifl_sds.ifsd_map[i]; | ||||
bus_dmamap_unload(fl->ifl_desc_tag, sd_map); | bus_dmamap_unload(fl->ifl_desc_tag, sd_map); | ||||
if (fl->ifl_rxq->ifr_ctx->ifc_in_detach) | // XXX: Should this get moved out? | ||||
if (iflib_in_detach(fl->ifl_rxq->ifr_ctx)) | |||||
bus_dmamap_destroy(fl->ifl_desc_tag, sd_map); | bus_dmamap_destroy(fl->ifl_desc_tag, sd_map); | ||||
} | } | ||||
if (*sd_m != NULL) { | if (*sd_m != NULL) { | ||||
m_init(*sd_m, M_NOWAIT, MT_DATA, 0); | m_init(*sd_m, M_NOWAIT, MT_DATA, 0); | ||||
uma_zfree(zone_mbuf, *sd_m); | uma_zfree(zone_mbuf, *sd_m); | ||||
} | } | ||||
if (*sd_cl != NULL) | if (*sd_cl != NULL) | ||||
uma_zfree(fl->ifl_zone, *sd_cl); | uma_zfree(fl->ifl_zone, *sd_cl); | ||||
▲ Show 20 Lines • Show All 1,738 Lines • ▼ Show 20 Lines | |||||
static void | static void | ||||
_task_fn_admin(void *context) | _task_fn_admin(void *context) | ||||
{ | { | ||||
if_ctx_t ctx = context; | if_ctx_t ctx = context; | ||||
if_softc_ctx_t sctx = &ctx->ifc_softc_ctx; | if_softc_ctx_t sctx = &ctx->ifc_softc_ctx; | ||||
iflib_txq_t txq; | iflib_txq_t txq; | ||||
int i; | int i; | ||||
bool oactive, running, do_reset, do_watchdog; | bool oactive, running, do_reset, do_watchdog, in_detach; | ||||
uint32_t reset_on = hz / 2; | uint32_t reset_on = hz / 2; | ||||
STATE_LOCK(ctx); | STATE_LOCK(ctx); | ||||
running = (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING); | running = (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING); | ||||
oactive = (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_OACTIVE); | oactive = (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_OACTIVE); | ||||
do_reset = (ctx->ifc_flags & IFC_DO_RESET); | do_reset = (ctx->ifc_flags & IFC_DO_RESET); | ||||
do_watchdog = (ctx->ifc_flags & IFC_DO_WATCHDOG); | do_watchdog = (ctx->ifc_flags & IFC_DO_WATCHDOG); | ||||
in_detach = (ctx->ifc_flags & IFC_IN_DETACH); | |||||
ctx->ifc_flags &= ~(IFC_DO_RESET|IFC_DO_WATCHDOG); | ctx->ifc_flags &= ~(IFC_DO_RESET|IFC_DO_WATCHDOG); | ||||
STATE_UNLOCK(ctx); | STATE_UNLOCK(ctx); | ||||
if ((!running & !oactive) && | if ((!running && !oactive) && !(ctx->ifc_sctx->isc_flags & IFLIB_ADMIN_ALWAYS_RUN)) | ||||
!(ctx->ifc_sctx->isc_flags & IFLIB_ADMIN_ALWAYS_RUN)) | |||||
return; | return; | ||||
if (in_detach) | |||||
return; | |||||
CTX_LOCK(ctx); | CTX_LOCK(ctx); | ||||
for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) { | for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) { | ||||
CALLOUT_LOCK(txq); | CALLOUT_LOCK(txq); | ||||
callout_stop(&txq->ift_timer); | callout_stop(&txq->ift_timer); | ||||
CALLOUT_UNLOCK(txq); | CALLOUT_UNLOCK(txq); | ||||
} | } | ||||
if (do_watchdog) { | if (do_watchdog) { | ||||
Show All 21 Lines | |||||
} | } | ||||
static void | static void | ||||
_task_fn_iov(void *context) | _task_fn_iov(void *context) | ||||
{ | { | ||||
if_ctx_t ctx = context; | if_ctx_t ctx = context; | ||||
if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)) | if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING) && | ||||
!(ctx->ifc_sctx->isc_flags & IFLIB_ADMIN_ALWAYS_RUN)) | |||||
return; | return; | ||||
CTX_LOCK(ctx); | CTX_LOCK(ctx); | ||||
IFDI_VFLR_HANDLE(ctx); | IFDI_VFLR_HANDLE(ctx); | ||||
CTX_UNLOCK(ctx); | CTX_UNLOCK(ctx); | ||||
} | } | ||||
static int | static int | ||||
▲ Show 20 Lines • Show All 770 Lines • ▼ Show 20 Lines | #endif | ||||
NETDUMP_SET(ctx->ifc_ifp, iflib); | NETDUMP_SET(ctx->ifc_ifp, iflib); | ||||
if_setgetcounterfn(ctx->ifc_ifp, iflib_if_get_counter); | if_setgetcounterfn(ctx->ifc_ifp, iflib_if_get_counter); | ||||
iflib_add_device_sysctl_post(ctx); | iflib_add_device_sysctl_post(ctx); | ||||
ctx->ifc_flags |= IFC_INIT_DONE; | ctx->ifc_flags |= IFC_INIT_DONE; | ||||
CTX_UNLOCK(ctx); | CTX_UNLOCK(ctx); | ||||
return (0); | return (0); | ||||
fail_detach: | fail_detach: | ||||
ether_ifdetach(ctx->ifc_ifp); | ether_ifdetach(ctx->ifc_ifp); | ||||
fail_intr_free: | fail_intr_free: | ||||
if (scctx->isc_intr == IFLIB_INTR_MSIX || scctx->isc_intr == IFLIB_INTR_MSI) | |||||
pci_release_msi(ctx->ifc_dev); | |||||
fail_queues: | fail_queues: | ||||
iflib_tx_structures_free(ctx); | iflib_tx_structures_free(ctx); | ||||
iflib_rx_structures_free(ctx); | iflib_rx_structures_free(ctx); | ||||
fail: | fail: | ||||
iflib_free_intr_mem(ctx); | |||||
IFDI_DETACH(ctx); | IFDI_DETACH(ctx); | ||||
CTX_UNLOCK(ctx); | CTX_UNLOCK(ctx); | ||||
return (err); | return (err); | ||||
} | } | ||||
int | int | ||||
iflib_pseudo_register(device_t dev, if_shared_ctx_t sctx, if_ctx_t *ctxp, | iflib_pseudo_register(device_t dev, if_shared_ctx_t sctx, if_ctx_t *ctxp, | ||||
struct iflib_cloneattach_ctx *clctx) | struct iflib_cloneattach_ctx *clctx) | ||||
{ | { | ||||
int err; | int err; | ||||
▲ Show 20 Lines • Show All 268 Lines • ▼ Show 20 Lines | iflib_device_deregister(if_ctx_t ctx) | ||||
iflib_rxq_t rxq; | iflib_rxq_t rxq; | ||||
device_t dev = ctx->ifc_dev; | device_t dev = ctx->ifc_dev; | ||||
int i, j; | int i, j; | ||||
struct taskqgroup *tqg; | struct taskqgroup *tqg; | ||||
iflib_fl_t fl; | iflib_fl_t fl; | ||||
/* Make sure VLANS are not using driver */ | /* Make sure VLANS are not using driver */ | ||||
if (if_vlantrunkinuse(ifp)) { | if (if_vlantrunkinuse(ifp)) { | ||||
device_printf(dev,"Vlan in use, detach first\n"); | device_printf(dev, "Vlan in use, detach first\n"); | ||||
return (EBUSY); | return (EBUSY); | ||||
} | } | ||||
#ifdef PCI_IOV | |||||
if (!CTX_IS_VF(ctx) && pci_iov_detach(dev) != 0) { | |||||
device_printf(dev, "SR-IOV in use; detach first.\n"); | |||||
return (EBUSY); | |||||
} | |||||
#endif | |||||
STATE_LOCK(ctx); | |||||
ctx->ifc_flags |= IFC_IN_DETACH; | |||||
STATE_UNLOCK(ctx); | |||||
CTX_LOCK(ctx); | CTX_LOCK(ctx); | ||||
ctx->ifc_in_detach = 1; | |||||
iflib_stop(ctx); | iflib_stop(ctx); | ||||
CTX_UNLOCK(ctx); | CTX_UNLOCK(ctx); | ||||
/* Unregister VLAN events */ | /* Unregister VLAN events */ | ||||
if (ctx->ifc_vlan_attach_event != NULL) | if (ctx->ifc_vlan_attach_event != NULL) | ||||
EVENTHANDLER_DEREGISTER(vlan_config, ctx->ifc_vlan_attach_event); | EVENTHANDLER_DEREGISTER(vlan_config, ctx->ifc_vlan_attach_event); | ||||
if (ctx->ifc_vlan_detach_event != NULL) | if (ctx->ifc_vlan_detach_event != NULL) | ||||
EVENTHANDLER_DEREGISTER(vlan_unconfig, ctx->ifc_vlan_detach_event); | EVENTHANDLER_DEREGISTER(vlan_unconfig, ctx->ifc_vlan_detach_event); | ||||
Show All 24 Lines | if (ctx->ifc_vflr_task.gt_uniq != NULL) | ||||
taskqgroup_detach(tqg, &ctx->ifc_vflr_task); | taskqgroup_detach(tqg, &ctx->ifc_vflr_task); | ||||
CTX_LOCK(ctx); | CTX_LOCK(ctx); | ||||
IFDI_DETACH(ctx); | IFDI_DETACH(ctx); | ||||
CTX_UNLOCK(ctx); | CTX_UNLOCK(ctx); | ||||
/* ether_ifdetach calls if_qflush - lock must be destroy afterwards*/ | /* ether_ifdetach calls if_qflush - lock must be destroy afterwards*/ | ||||
CTX_LOCK_DESTROY(ctx); | CTX_LOCK_DESTROY(ctx); | ||||
device_set_softc(ctx->ifc_dev, NULL); | device_set_softc(ctx->ifc_dev, NULL); | ||||
if (ctx->ifc_softc_ctx.isc_intr != IFLIB_INTR_LEGACY) { | iflib_free_intr_mem(ctx); | ||||
pci_release_msi(dev); | |||||
} | |||||
if (ctx->ifc_softc_ctx.isc_intr != IFLIB_INTR_MSIX) { | |||||
iflib_irq_free(ctx, &ctx->ifc_legacy_irq); | |||||
} | |||||
if (ctx->ifc_msix_mem != NULL) { | |||||
bus_release_resource(ctx->ifc_dev, SYS_RES_MEMORY, | |||||
ctx->ifc_softc_ctx.isc_msix_bar, ctx->ifc_msix_mem); | |||||
ctx->ifc_msix_mem = NULL; | |||||
} | |||||
bus_generic_detach(dev); | bus_generic_detach(dev); | ||||
if_free(ifp); | if_free(ifp); | ||||
iflib_tx_structures_free(ctx); | iflib_tx_structures_free(ctx); | ||||
iflib_rx_structures_free(ctx); | iflib_rx_structures_free(ctx); | ||||
if (ctx->ifc_flags & IFC_SC_ALLOCATED) | if (ctx->ifc_flags & IFC_SC_ALLOCATED) | ||||
free(ctx->ifc_softc, M_IFLIB); | free(ctx->ifc_softc, M_IFLIB); | ||||
STATE_LOCK_DESTROY(ctx); | |||||
free(ctx, M_IFLIB); | free(ctx, M_IFLIB); | ||||
return (0); | return (0); | ||||
} | } | ||||
static void | |||||
iflib_free_intr_mem(if_ctx_t ctx) | |||||
{ | |||||
if (ctx->ifc_softc_ctx.isc_intr != IFLIB_INTR_LEGACY) { | |||||
pci_release_msi(ctx->ifc_dev); | |||||
} | |||||
if (ctx->ifc_softc_ctx.isc_intr != IFLIB_INTR_MSIX) { | |||||
iflib_irq_free(ctx, &ctx->ifc_legacy_irq); | |||||
} | |||||
if (ctx->ifc_msix_mem != NULL) { | |||||
bus_release_resource(ctx->ifc_dev, SYS_RES_MEMORY, | |||||
ctx->ifc_softc_ctx.isc_msix_bar, ctx->ifc_msix_mem); | |||||
ctx->ifc_msix_mem = NULL; | |||||
} | |||||
} | |||||
int | int | ||||
iflib_device_detach(device_t dev) | iflib_device_detach(device_t dev) | ||||
{ | { | ||||
if_ctx_t ctx = device_get_softc(dev); | if_ctx_t ctx = device_get_softc(dev); | ||||
return (iflib_device_deregister(ctx)); | return (iflib_device_deregister(ctx)); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 154 Lines • ▼ Show 20 Lines | iflib_register(if_ctx_t ctx) | ||||
driver_t *driver = sctx->isc_driver; | driver_t *driver = sctx->isc_driver; | ||||
device_t dev = ctx->ifc_dev; | device_t dev = ctx->ifc_dev; | ||||
if_t ifp; | if_t ifp; | ||||
_iflib_assert(sctx); | _iflib_assert(sctx); | ||||
CTX_LOCK_INIT(ctx); | CTX_LOCK_INIT(ctx); | ||||
STATE_LOCK_INIT(ctx, device_get_nameunit(ctx->ifc_dev)); | STATE_LOCK_INIT(ctx, device_get_nameunit(ctx->ifc_dev)); | ||||
ifp = ctx->ifc_ifp = if_gethandle(IFT_ETHER); | ifp = ctx->ifc_ifp = if_alloc(IFT_ETHER); | ||||
if (ifp == NULL) { | if (ifp == NULL) { | ||||
device_printf(dev, "can not allocate ifnet structure\n"); | device_printf(dev, "can not allocate ifnet structure\n"); | ||||
return (ENOMEM); | return (ENOMEM); | ||||
} | } | ||||
/* | /* | ||||
* Initialize our context's device specific methods | * Initialize our context's device specific methods | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 167 Lines • ▼ Show 20 Lines | for (rxconf = i = 0; i < nrxqsets; i++, rxconf++, rxq++) { | ||||
} | } | ||||
rxq->ifr_fl = fl; | rxq->ifr_fl = fl; | ||||
for (j = 0; j < nfree_lists; j++) { | for (j = 0; j < nfree_lists; j++) { | ||||
fl[j].ifl_rxq = rxq; | fl[j].ifl_rxq = rxq; | ||||
fl[j].ifl_id = j; | fl[j].ifl_id = j; | ||||
fl[j].ifl_ifdi = &rxq->ifr_ifdi[j + rxq->ifr_fl_offset]; | fl[j].ifl_ifdi = &rxq->ifr_ifdi[j + rxq->ifr_fl_offset]; | ||||
fl[j].ifl_rxd_size = scctx->isc_rxd_size[j]; | fl[j].ifl_rxd_size = scctx->isc_rxd_size[j]; | ||||
} | } | ||||
/* Allocate receive buffers for the ring*/ | /* Allocate receive buffers for the ring */ | ||||
if (iflib_rxsd_alloc(rxq)) { | if (iflib_rxsd_alloc(rxq)) { | ||||
device_printf(dev, | device_printf(dev, | ||||
"Critical Failure setting up receive buffers\n"); | "Critical Failure setting up receive buffers\n"); | ||||
err = ENOMEM; | err = ENOMEM; | ||||
goto err_rx_desc; | goto err_rx_desc; | ||||
} | } | ||||
for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) | for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) | ||||
▲ Show 20 Lines • Show All 138 Lines • ▼ Show 20 Lines | |||||
static void | static void | ||||
iflib_rx_structures_free(if_ctx_t ctx) | iflib_rx_structures_free(if_ctx_t ctx) | ||||
{ | { | ||||
iflib_rxq_t rxq = ctx->ifc_rxqs; | iflib_rxq_t rxq = ctx->ifc_rxqs; | ||||
for (int i = 0; i < ctx->ifc_softc_ctx.isc_nrxqsets; i++, rxq++) { | for (int i = 0; i < ctx->ifc_softc_ctx.isc_nrxqsets; i++, rxq++) { | ||||
iflib_rx_sds_free(rxq); | iflib_rx_sds_free(rxq); | ||||
} | } | ||||
free(ctx->ifc_rxqs, M_IFLIB); | |||||
ctx->ifc_rxqs = NULL; | |||||
} | } | ||||
static int | static int | ||||
iflib_qset_structures_setup(if_ctx_t ctx) | iflib_qset_structures_setup(if_ctx_t ctx) | ||||
{ | { | ||||
int err; | int err; | ||||
/* | /* | ||||
▲ Show 20 Lines • Show All 244 Lines • ▼ Show 20 Lines | iflib_irq_alloc_generic(if_ctx_t ctx, if_irq_t irq, int rid, | ||||
} else { | } else { | ||||
taskqgroup_attach(tqg, gtask, q, rman_get_start(irq->ii_res), name); | taskqgroup_attach(tqg, gtask, q, rman_get_start(irq->ii_res), name); | ||||
} | } | ||||
return (0); | return (0); | ||||
} | } | ||||
void | void | ||||
iflib_softirq_alloc_generic(if_ctx_t ctx, if_irq_t irq, iflib_intr_type_t type, void *arg, int qid, const char *name) | iflib_softirq_alloc_generic(if_ctx_t ctx, if_irq_t irq, iflib_intr_type_t type, void *arg, int qid, const char *name) | ||||
{ | { | ||||
struct grouptask *gtask; | struct grouptask *gtask; | ||||
struct taskqgroup *tqg; | struct taskqgroup *tqg; | ||||
gtask_fn_t *fn; | gtask_fn_t *fn; | ||||
void *q; | void *q; | ||||
int irq_num = -1; | int irq_num = -1; | ||||
int err; | int err; | ||||
▲ Show 20 Lines • Show All 301 Lines • ▼ Show 20 Lines | else | ||||
tx_queues = mp_ncpus; | tx_queues = mp_ncpus; | ||||
if (tx_queues > scctx->isc_ntxqsets) | if (tx_queues > scctx->isc_ntxqsets) | ||||
tx_queues = scctx->isc_ntxqsets; | tx_queues = scctx->isc_ntxqsets; | ||||
if (ctx->ifc_sysctl_qs_eq_override == 0) { | if (ctx->ifc_sysctl_qs_eq_override == 0) { | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
if (tx_queues != rx_queues) | if (tx_queues != rx_queues) | ||||
device_printf(dev, "queue equality override not set, capping rx_queues at %d and tx_queues at %d\n", | device_printf(dev, | ||||
"queue equality override not set, capping rx_queues at %d and tx_queues at %d\n", | |||||
min(rx_queues, tx_queues), min(rx_queues, tx_queues)); | min(rx_queues, tx_queues), min(rx_queues, tx_queues)); | ||||
#endif | #endif | ||||
tx_queues = min(rx_queues, tx_queues); | tx_queues = min(rx_queues, tx_queues); | ||||
rx_queues = min(rx_queues, tx_queues); | rx_queues = min(rx_queues, tx_queues); | ||||
} | } | ||||
device_printf(dev, "using %d rx queues %d tx queues \n", rx_queues, tx_queues); | device_printf(dev, "using %d rx queues %d tx queues \n", rx_queues, tx_queues); | ||||
vectors = rx_queues + admincnt; | vectors = rx_queues + admincnt; | ||||
if ((err = pci_alloc_msix(dev, &vectors)) == 0) { | if ((err = pci_alloc_msix(dev, &vectors)) == 0) { | ||||
device_printf(dev, | device_printf(dev, "Using MSIX interrupts with %d vectors\n", vectors); | ||||
"Using MSIX interrupts with %d vectors\n", vectors); | |||||
scctx->isc_vectors = vectors; | scctx->isc_vectors = vectors; | ||||
scctx->isc_nrxqsets = rx_queues; | scctx->isc_nrxqsets = rx_queues; | ||||
scctx->isc_ntxqsets = tx_queues; | scctx->isc_ntxqsets = tx_queues; | ||||
scctx->isc_intr = IFLIB_INTR_MSIX; | scctx->isc_intr = IFLIB_INTR_MSIX; | ||||
return (vectors); | return (vectors); | ||||
} else { | } else { | ||||
device_printf(dev, "failed to allocate %d msix vectors, err: %d - using MSI\n", vectors, err); | device_printf(dev, | ||||
"failed to allocate %d msix vectors, err: %d - using MSI\n", vectors, err); | |||||
bus_release_resource(dev, SYS_RES_MEMORY, bar, | bus_release_resource(dev, SYS_RES_MEMORY, bar, | ||||
ctx->ifc_msix_mem); | ctx->ifc_msix_mem); | ||||
ctx->ifc_msix_mem = NULL; | ctx->ifc_msix_mem = NULL; | ||||
} | } | ||||
msi: | msi: | ||||
vectors = pci_msi_count(dev); | vectors = pci_msi_count(dev); | ||||
scctx->isc_nrxqsets = 1; | scctx->isc_nrxqsets = 1; | ||||
scctx->isc_ntxqsets = 1; | scctx->isc_ntxqsets = 1; | ||||
▲ Show 20 Lines • Show All 292 Lines • ▼ Show 20 Lines | #if MEMORY_LOGGING | ||||
SYSCTL_ADD_QUAD(ctx_list, fl_list, OID_AUTO, "fl_cl_dequeued", | SYSCTL_ADD_QUAD(ctx_list, fl_list, OID_AUTO, "fl_cl_dequeued", | ||||
CTLFLAG_RD, | CTLFLAG_RD, | ||||
&fl->ifl_cl_dequeued, "clusters freed"); | &fl->ifl_cl_dequeued, "clusters freed"); | ||||
#endif | #endif | ||||
} | } | ||||
} | } | ||||
} | |||||
void | |||||
iflib_request_reset(if_ctx_t ctx) | |||||
{ | |||||
STATE_LOCK(ctx); | |||||
ctx->ifc_flags |= IFC_DO_RESET; | |||||
STATE_UNLOCK(ctx); | |||||
} | } | ||||
#ifndef __NO_STRICT_ALIGNMENT | #ifndef __NO_STRICT_ALIGNMENT | ||||
static struct mbuf * | static struct mbuf * | ||||
iflib_fixup_rx(struct mbuf *m) | iflib_fixup_rx(struct mbuf *m) | ||||
{ | { | ||||
struct mbuf *n; | struct mbuf *n; | ||||
▲ Show 20 Lines • Show All 105 Lines • Show Last 20 Lines |