Changeset View
Changeset View
Standalone View
Standalone View
sys/net/iflib.c
Show First 20 Lines • Show All 45 Lines • ▼ Show 20 Lines | |||||
#include <sys/sbuf.h> | #include <sys/sbuf.h> | ||||
#include <sys/smp.h> | #include <sys/smp.h> | ||||
#include <sys/socket.h> | #include <sys/socket.h> | ||||
#include <sys/sysctl.h> | #include <sys/sysctl.h> | ||||
#include <sys/syslog.h> | #include <sys/syslog.h> | ||||
#include <sys/taskqueue.h> | #include <sys/taskqueue.h> | ||||
#include <sys/limits.h> | #include <sys/limits.h> | ||||
#include <net/if.h> | #include <net/if.h> | ||||
#include <net/if_var.h> | #include <net/if_var.h> | ||||
#include <net/if_types.h> | #include <net/if_types.h> | ||||
#include <net/if_media.h> | #include <net/if_media.h> | ||||
#include <net/bpf.h> | #include <net/bpf.h> | ||||
#include <net/ethernet.h> | #include <net/ethernet.h> | ||||
#include <net/mp_ring.h> | #include <net/mp_ring.h> | ||||
▲ Show 20 Lines • Show All 89 Lines • ▼ Show 20 Lines | struct iflib_ctx { | ||||
void *ifc_softc; | void *ifc_softc; | ||||
device_t ifc_dev; | device_t ifc_dev; | ||||
if_t ifc_ifp; | if_t ifc_ifp; | ||||
cpuset_t ifc_cpus; | cpuset_t ifc_cpus; | ||||
if_shared_ctx_t ifc_sctx; | if_shared_ctx_t ifc_sctx; | ||||
struct if_softc_ctx ifc_softc_ctx; | struct if_softc_ctx ifc_softc_ctx; | ||||
struct mtx ifc_mtx; | struct sx ifc_sx; | ||||
uint16_t ifc_nhwtxqs; | uint16_t ifc_nhwtxqs; | ||||
uint16_t ifc_nhwrxqs; | uint16_t ifc_nhwrxqs; | ||||
iflib_txq_t ifc_txqs; | iflib_txq_t ifc_txqs; | ||||
iflib_rxq_t ifc_rxqs; | iflib_rxq_t ifc_rxqs; | ||||
uint32_t ifc_if_flags; | uint32_t ifc_if_flags; | ||||
uint32_t ifc_flags; | uint32_t ifc_flags; | ||||
▲ Show 20 Lines • Show All 353 Lines • ▼ Show 20 Lines | |||||
/* | /* | ||||
* Only allow a single packet to take up most 1/nth of the tx ring | * Only allow a single packet to take up most 1/nth of the tx ring | ||||
*/ | */ | ||||
#define MAX_SINGLE_PACKET_FRACTION 12 | #define MAX_SINGLE_PACKET_FRACTION 12 | ||||
#define IF_BAD_DMA (bus_addr_t)-1 | #define IF_BAD_DMA (bus_addr_t)-1 | ||||
#define CTX_ACTIVE(ctx) ((if_getdrvflags((ctx)->ifc_ifp) & IFF_DRV_RUNNING)) | #define CTX_ACTIVE(ctx) ((if_getdrvflags((ctx)->ifc_ifp) & IFF_DRV_RUNNING)) | ||||
/* | |||||
jtl: These should probably just be deleted. | |||||
kmacyAuthorUnsubmitted Not Done Inline Actionsyup. kmacy: yup. | |||||
#define CTX_LOCK_INIT(_sc, _name) mtx_init(&(_sc)->ifc_mtx, _name, "iflib ctx lock", MTX_DEF) | #define CTX_LOCK_INIT(_sc, _name) mtx_init(&(_sc)->ifc_mtx, _name, "iflib ctx lock", MTX_DEF) | ||||
#define CTX_LOCK(ctx) mtx_lock(&(ctx)->ifc_mtx) | #define CTX_LOCK(ctx) mtx_lock(&(ctx)->ifc_mtx) | ||||
#define CTX_UNLOCK(ctx) mtx_unlock(&(ctx)->ifc_mtx) | #define CTX_UNLOCK(ctx) mtx_unlock(&(ctx)->ifc_mtx) | ||||
#define CTX_LOCK_DESTROY(ctx) mtx_destroy(&(ctx)->ifc_mtx) | #define CTX_LOCK_DESTROY(ctx) mtx_destroy(&(ctx)->ifc_mtx) | ||||
*/ | |||||
#define CTX_LOCK_INIT(_sc, _name) sx_init(&(_sc)->ifc_sx, _name) | |||||
#define CTX_LOCK(ctx) sx_xlock(&(ctx)->ifc_sx) | |||||
#define CTX_UNLOCK(ctx) sx_xunlock(&(ctx)->ifc_sx) | |||||
#define CTX_LOCK_DESTROY(ctx) sx_destroy(&(ctx)->ifc_sx) | |||||
#define CALLOUT_LOCK(txq) mtx_lock(&txq->ift_mtx) | #define CALLOUT_LOCK(txq) mtx_lock(&txq->ift_mtx) | ||||
#define CALLOUT_UNLOCK(txq) mtx_unlock(&txq->ift_mtx) | #define CALLOUT_UNLOCK(txq) mtx_unlock(&txq->ift_mtx) | ||||
/* Our boot-time initialization hook */ | /* Our boot-time initialization hook */ | ||||
static int iflib_module_event_handler(module_t, int, void *); | static int iflib_module_event_handler(module_t, int, void *); | ||||
static moduledata_t iflib_moduledata = { | static moduledata_t iflib_moduledata = { | ||||
▲ Show 20 Lines • Show All 139 Lines • ▼ Show 20 Lines | iflib_tx_seen = iflib_tx_sent = iflib_tx_encap = iflib_rx_allocs = | ||||
iflib_rx_mbuf_null = iflib_rxd_flush = 0; | iflib_rx_mbuf_null = iflib_rxd_flush = 0; | ||||
} | } | ||||
#else | #else | ||||
#define DBG_COUNTER_INC(name) | #define DBG_COUNTER_INC(name) | ||||
static void iflib_debug_reset(void) {} | static void iflib_debug_reset(void) {} | ||||
#endif | #endif | ||||
typedef void async_gtask_fn_t(if_ctx_t ctx, void *arg); | |||||
struct async_task_arg { | |||||
async_gtask_fn_t *ata_fn; | |||||
if_ctx_t ata_ctx; | |||||
void *ata_arg; | |||||
struct grouptask *ata_gtask; | |||||
}; | |||||
#define IFLIB_DEBUG 0 | #define IFLIB_DEBUG 0 | ||||
static void iflib_tx_structures_free(if_ctx_t ctx); | static void iflib_tx_structures_free(if_ctx_t ctx); | ||||
static void iflib_rx_structures_free(if_ctx_t ctx); | static void iflib_rx_structures_free(if_ctx_t ctx); | ||||
static int iflib_queues_alloc(if_ctx_t ctx); | static int iflib_queues_alloc(if_ctx_t ctx); | ||||
static int iflib_tx_credits_update(if_ctx_t ctx, iflib_txq_t txq); | static int iflib_tx_credits_update(if_ctx_t ctx, iflib_txq_t txq); | ||||
static int iflib_rxd_avail(if_ctx_t ctx, iflib_rxq_t rxq, qidx_t cidx, qidx_t budget); | static int iflib_rxd_avail(if_ctx_t ctx, iflib_rxq_t rxq, qidx_t cidx, qidx_t budget); | ||||
static int iflib_qset_structures_setup(if_ctx_t ctx); | static int iflib_qset_structures_setup(if_ctx_t ctx); | ||||
static int iflib_msix_init(if_ctx_t ctx); | static int iflib_msix_init(if_ctx_t ctx); | ||||
static int iflib_legacy_setup(if_ctx_t ctx, driver_filter_t filter, void *filterarg, int *rid, char *str); | static int iflib_legacy_setup(if_ctx_t ctx, driver_filter_t filter, void *filterarg, int *rid, char *str); | ||||
static void iflib_txq_check_drain(iflib_txq_t txq, int budget); | static void iflib_txq_check_drain(iflib_txq_t txq, int budget); | ||||
static uint32_t iflib_txq_can_drain(struct ifmp_ring *); | static uint32_t iflib_txq_can_drain(struct ifmp_ring *); | ||||
static int iflib_register(if_ctx_t); | static int iflib_register(if_ctx_t); | ||||
static void iflib_init_locked(if_ctx_t ctx); | static void iflib_init_locked(if_ctx_t ctx); | ||||
static void iflib_add_device_sysctl_pre(if_ctx_t ctx); | static void iflib_add_device_sysctl_pre(if_ctx_t ctx); | ||||
static void iflib_add_device_sysctl_post(if_ctx_t ctx); | static void iflib_add_device_sysctl_post(if_ctx_t ctx); | ||||
static void iflib_ifmp_purge(iflib_txq_t txq); | static void iflib_ifmp_purge(iflib_txq_t txq); | ||||
static void _iflib_pre_assert(if_softc_ctx_t scctx); | static void _iflib_pre_assert(if_softc_ctx_t scctx); | ||||
static void iflib_stop(if_ctx_t ctx); | static void iflib_stop(if_ctx_t ctx); | ||||
static void iflib_if_init_locked(if_ctx_t ctx); | static void iflib_if_init_locked(if_ctx_t ctx); | ||||
static int async_if_ioctl(if_ctx_t ctx, u_long command, caddr_t data); | |||||
static int iflib_config_async_gtask_dispatch(if_ctx_t ctx, async_gtask_fn_t *fn, char *name, void *arg); | |||||
static void iflib_admin_reset_deferred(if_ctx_t ctx); | |||||
#ifndef __NO_STRICT_ALIGNMENT | #ifndef __NO_STRICT_ALIGNMENT | ||||
static struct mbuf * iflib_fixup_rx(struct mbuf *m); | static struct mbuf * iflib_fixup_rx(struct mbuf *m); | ||||
#endif | #endif | ||||
#ifdef DEV_NETMAP | #ifdef DEV_NETMAP | ||||
#include <sys/selinfo.h> | #include <sys/selinfo.h> | ||||
#include <net/netmap.h> | #include <net/netmap.h> | ||||
#include <dev/netmap/netmap_kern.h> | #include <dev/netmap/netmap_kern.h> | ||||
▲ Show 20 Lines • Show All 366 Lines • ▼ Show 20 Lines | |||||
} | } | ||||
static void | static void | ||||
iflib_netmap_intr(struct netmap_adapter *na, int onoff) | iflib_netmap_intr(struct netmap_adapter *na, int onoff) | ||||
{ | { | ||||
struct ifnet *ifp = na->ifp; | struct ifnet *ifp = na->ifp; | ||||
if_ctx_t ctx = ifp->if_softc; | if_ctx_t ctx = ifp->if_softc; | ||||
CTX_LOCK(ctx); | /* XXX - do we need synchronization here?*/ | ||||
if (onoff) { | if (onoff) { | ||||
IFDI_INTR_ENABLE(ctx); | IFDI_INTR_ENABLE(ctx); | ||||
} else { | } else { | ||||
IFDI_INTR_DISABLE(ctx); | IFDI_INTR_DISABLE(ctx); | ||||
} | } | ||||
CTX_UNLOCK(ctx); | |||||
} | } | ||||
static int | static int | ||||
iflib_netmap_attach(if_ctx_t ctx) | iflib_netmap_attach(if_ctx_t ctx) | ||||
{ | { | ||||
struct netmap_adapter na; | struct netmap_adapter na; | ||||
if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; | if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; | ||||
▲ Show 20 Lines • Show All 974 Lines • ▼ Show 20 Lines | for (i = 0; i < rxq->ifr_nfl; i++) { | ||||
fl->ifl_sds.ifsd_map = NULL; | fl->ifl_sds.ifsd_map = NULL; | ||||
} | } | ||||
free(rxq->ifr_fl, M_IFLIB); | free(rxq->ifr_fl, M_IFLIB); | ||||
rxq->ifr_fl = NULL; | rxq->ifr_fl = NULL; | ||||
rxq->ifr_cq_gen = rxq->ifr_cq_cidx = rxq->ifr_cq_pidx = 0; | rxq->ifr_cq_gen = rxq->ifr_cq_cidx = rxq->ifr_cq_pidx = 0; | ||||
} | } | ||||
} | } | ||||
/* CONFIG context only */ | |||||
static void | |||||
iflib_handle_hang(if_ctx_t ctx, void *arg) | |||||
{ | |||||
iflib_txq_t txq = arg; | |||||
CTX_LOCK(ctx); | |||||
if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); | |||||
device_printf(ctx->ifc_dev, "TX(%d) desc avail = %d, pidx = %d\n", | |||||
txq->ift_id, TXQ_AVAIL(txq), txq->ift_pidx); | |||||
IFDI_WATCHDOG_RESET(ctx); | |||||
ctx->ifc_watchdog_events++; | |||||
ctx->ifc_flags |= IFC_DO_RESET; | |||||
iflib_admin_intr_deferred(ctx); | |||||
CTX_UNLOCK(ctx); | |||||
} | |||||
/* | /* | ||||
* MI independent logic | * MI independent logic | ||||
* | * | ||||
*/ | */ | ||||
static void | static void | ||||
iflib_timer(void *arg) | iflib_timer(void *arg) | ||||
{ | { | ||||
iflib_txq_t txq = arg; | iflib_txq_t txq = arg; | ||||
Show All 20 Lines | iflib_timer(void *arg) | ||||
if (txq->ift_db_pending) | if (txq->ift_db_pending) | ||||
GROUPTASK_ENQUEUE(&txq->ift_task); | GROUPTASK_ENQUEUE(&txq->ift_task); | ||||
sctx->isc_pause_frames = 0; | sctx->isc_pause_frames = 0; | ||||
if (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING) | if (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING) | ||||
callout_reset_on(&txq->ift_timer, hz/2, iflib_timer, txq, txq->ift_timer.c_cpu); | callout_reset_on(&txq->ift_timer, hz/2, iflib_timer, txq, txq->ift_timer.c_cpu); | ||||
return; | return; | ||||
hung: | hung: | ||||
CTX_LOCK(ctx); | iflib_config_async_gtask_dispatch(ctx, iflib_handle_hang, "hang handler", txq); | ||||
if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); | |||||
device_printf(ctx->ifc_dev, "TX(%d) desc avail = %d, pidx = %d\n", | |||||
txq->ift_id, TXQ_AVAIL(txq), txq->ift_pidx); | |||||
IFDI_WATCHDOG_RESET(ctx); | |||||
ctx->ifc_watchdog_events++; | |||||
ctx->ifc_flags |= IFC_DO_RESET; | |||||
iflib_admin_intr_deferred(ctx); | |||||
CTX_UNLOCK(ctx); | |||||
} | } | ||||
static void | static void | ||||
iflib_init_locked(if_ctx_t ctx) | iflib_init_locked(if_ctx_t ctx) | ||||
{ | { | ||||
if_softc_ctx_t sctx = &ctx->ifc_softc_ctx; | if_softc_ctx_t sctx = &ctx->ifc_softc_ctx; | ||||
if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; | if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; | ||||
if_t ifp = ctx->ifc_ifp; | if_t ifp = ctx->ifc_ifp; | ||||
▲ Show 20 Lines • Show All 49 Lines • ▼ Show 20 Lines | #endif | ||||
if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); | if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); | ||||
IFDI_INTR_ENABLE(ctx); | IFDI_INTR_ENABLE(ctx); | ||||
txq = ctx->ifc_txqs; | txq = ctx->ifc_txqs; | ||||
for (i = 0; i < sctx->isc_ntxqsets; i++, txq++) | for (i = 0; i < sctx->isc_ntxqsets; i++, txq++) | ||||
callout_reset_on(&txq->ift_timer, hz/2, iflib_timer, txq, | callout_reset_on(&txq->ift_timer, hz/2, iflib_timer, txq, | ||||
txq->ift_timer.c_cpu); | txq->ift_timer.c_cpu); | ||||
} | } | ||||
/* CONFIG context only */ | |||||
static int | static int | ||||
iflib_media_change(if_t ifp) | iflib_media_change(if_t ifp) | ||||
{ | { | ||||
if_ctx_t ctx = if_getsoftc(ifp); | if_ctx_t ctx = if_getsoftc(ifp); | ||||
int err; | int err; | ||||
CTX_LOCK(ctx); | CTX_LOCK(ctx); | ||||
if ((err = IFDI_MEDIA_CHANGE(ctx)) == 0) | if ((err = IFDI_MEDIA_CHANGE(ctx)) == 0) | ||||
iflib_init_locked(ctx); | iflib_init_locked(ctx); | ||||
CTX_UNLOCK(ctx); | CTX_UNLOCK(ctx); | ||||
return (err); | return (err); | ||||
} | } | ||||
/* CONFIG context only */ | |||||
static void | static void | ||||
iflib_media_status(if_t ifp, struct ifmediareq *ifmr) | iflib_media_status(if_t ifp, struct ifmediareq *ifmr) | ||||
{ | { | ||||
if_ctx_t ctx = if_getsoftc(ifp); | if_ctx_t ctx = if_getsoftc(ifp); | ||||
iflib_admin_intr_deferred(ctx); | |||||
CTX_LOCK(ctx); | CTX_LOCK(ctx); | ||||
IFDI_UPDATE_ADMIN_STATUS(ctx); | |||||
IFDI_MEDIA_STATUS(ctx, ifmr); | IFDI_MEDIA_STATUS(ctx, ifmr); | ||||
CTX_UNLOCK(ctx); | CTX_UNLOCK(ctx); | ||||
} | } | ||||
/* CONFIG context only */ | |||||
static void | static void | ||||
iflib_stop(if_ctx_t ctx) | iflib_stop(if_ctx_t ctx) | ||||
{ | { | ||||
iflib_txq_t txq = ctx->ifc_txqs; | iflib_txq_t txq = ctx->ifc_txqs; | ||||
iflib_rxq_t rxq = ctx->ifc_rxqs; | iflib_rxq_t rxq = ctx->ifc_rxqs; | ||||
if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; | if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; | ||||
iflib_dma_info_t di; | iflib_dma_info_t di; | ||||
iflib_fl_t fl; | iflib_fl_t fl; | ||||
int i, j; | int i, j; | ||||
/* Tell the stack that the interface is no longer active */ | /* Tell the stack that the interface is no longer active */ | ||||
if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); | if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); | ||||
IFDI_INTR_DISABLE(ctx); | IFDI_INTR_DISABLE(ctx); | ||||
DELAY(1000); | |||||
IFDI_STOP(ctx); | IFDI_STOP(ctx); | ||||
DELAY(1000); | |||||
iflib_debug_reset(); | iflib_debug_reset(); | ||||
/* Wait for current tx queue users to exit to disarm watchdog timer. */ | /* Wait for current tx queue users to exit to disarm watchdog timer. */ | ||||
for (i = 0; i < scctx->isc_ntxqsets; i++, txq++) { | for (i = 0; i < scctx->isc_ntxqsets; i++, txq++) { | ||||
/* make sure all transmitters have completed before proceeding XXX */ | /* make sure all transmitters have completed before proceeding XXX */ | ||||
/* clean any enqueued buffers */ | /* clean any enqueued buffers */ | ||||
iflib_ifmp_purge(txq); | iflib_ifmp_purge(txq); | ||||
▲ Show 20 Lines • Show All 322 Lines • ▼ Show 20 Lines | #endif | ||||
*/ | */ | ||||
#if defined(INET6) || defined(INET) | #if defined(INET6) || defined(INET) | ||||
tcp_lro_flush_all(&rxq->ifr_lc); | tcp_lro_flush_all(&rxq->ifr_lc); | ||||
#endif | #endif | ||||
if (avail) | if (avail) | ||||
return true; | return true; | ||||
return (iflib_rxd_avail(ctx, rxq, *cidxp, 1)); | return (iflib_rxd_avail(ctx, rxq, *cidxp, 1)); | ||||
err: | err: | ||||
CTX_LOCK(ctx); | iflib_admin_reset_deferred(ctx); | ||||
ctx->ifc_flags |= IFC_DO_RESET; | |||||
iflib_admin_intr_deferred(ctx); | |||||
CTX_UNLOCK(ctx); | |||||
return (false); | return (false); | ||||
} | } | ||||
#define TXD_NOTIFY_COUNT(txq) (((txq)->ift_size / (txq)->ift_update_freq)-1) | #define TXD_NOTIFY_COUNT(txq) (((txq)->ift_size / (txq)->ift_update_freq)-1) | ||||
static inline qidx_t | static inline qidx_t | ||||
txq_max_db_deferred(iflib_txq_t txq, qidx_t in_use) | txq_max_db_deferred(iflib_txq_t txq, qidx_t in_use) | ||||
{ | { | ||||
qidx_t notify_count = TXD_NOTIFY_COUNT(txq); | qidx_t notify_count = TXD_NOTIFY_COUNT(txq); | ||||
▲ Show 20 Lines • Show All 946 Lines • ▼ Show 20 Lines | if ((more = iflib_rxeof(rxq, 16 /* XXX */)) == false) { | ||||
} | } | ||||
} | } | ||||
if (__predict_false(!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING))) | if (__predict_false(!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING))) | ||||
return; | return; | ||||
if (more) | if (more) | ||||
GROUPTASK_ENQUEUE(&rxq->ifr_task); | GROUPTASK_ENQUEUE(&rxq->ifr_task); | ||||
} | } | ||||
/* CONFIG context only */ | |||||
static void | static void | ||||
_task_fn_admin(void *context) | _task_fn_admin(void *context) | ||||
{ | { | ||||
if_ctx_t ctx = context; | if_ctx_t ctx = context; | ||||
if_softc_ctx_t sctx = &ctx->ifc_softc_ctx; | if_softc_ctx_t sctx = &ctx->ifc_softc_ctx; | ||||
iflib_txq_t txq; | iflib_txq_t txq; | ||||
int i; | int i, running; | ||||
if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)) { | running = !!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING); | ||||
if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_OACTIVE)) { | |||||
return; | |||||
} | |||||
} | |||||
CTX_LOCK(ctx); | CTX_LOCK(ctx); | ||||
for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) { | for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) { | ||||
CALLOUT_LOCK(txq); | CALLOUT_LOCK(txq); | ||||
callout_stop(&txq->ift_timer); | callout_stop(&txq->ift_timer); | ||||
CALLOUT_UNLOCK(txq); | CALLOUT_UNLOCK(txq); | ||||
} | } | ||||
IFDI_UPDATE_ADMIN_STATUS(ctx); | if (running) { | ||||
for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) | for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) | ||||
callout_reset_on(&txq->ift_timer, hz/2, iflib_timer, txq, txq->ift_timer.c_cpu); | callout_reset_on(&txq->ift_timer, hz/2, iflib_timer, | ||||
txq, txq->ift_timer.c_cpu); | |||||
IFDI_LINK_INTR_ENABLE(ctx); | IFDI_LINK_INTR_ENABLE(ctx); | ||||
} | |||||
if (ctx->ifc_flags & IFC_DO_RESET) { | if (ctx->ifc_flags & IFC_DO_RESET) { | ||||
ctx->ifc_flags &= ~IFC_DO_RESET; | ctx->ifc_flags &= ~IFC_DO_RESET; | ||||
iflib_if_init_locked(ctx); | iflib_if_init_locked(ctx); | ||||
} | } | ||||
CTX_UNLOCK(ctx); | CTX_UNLOCK(ctx); | ||||
if (LINK_ACTIVE(ctx) == 0) | IFDI_UPDATE_ADMIN_STATUS(ctx); | ||||
if (LINK_ACTIVE(ctx) == 0 || !running) | |||||
return; | return; | ||||
for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) | for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) | ||||
iflib_txq_check_drain(txq, IFLIB_RESTART_BUDGET); | iflib_txq_check_drain(txq, IFLIB_RESTART_BUDGET); | ||||
} | } | ||||
/* CONFIG context only */ | |||||
static void | static void | ||||
_task_fn_iov(void *context) | _task_fn_iov(void *context) | ||||
{ | { | ||||
if_ctx_t ctx = context; | if_ctx_t ctx = context; | ||||
if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)) | if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)) | ||||
return; | return; | ||||
▲ Show 20 Lines • Show All 113 Lines • ▼ Show 20 Lines | #endif | ||||
m_freem(m); | m_freem(m); | ||||
} else if (TXQ_AVAIL(txq) < (txq->ift_size >> 1)) { | } else if (TXQ_AVAIL(txq) < (txq->ift_size >> 1)) { | ||||
GROUPTASK_ENQUEUE(&txq->ift_task); | GROUPTASK_ENQUEUE(&txq->ift_task); | ||||
} | } | ||||
return (err); | return (err); | ||||
} | } | ||||
/* CONFIG context only */ | |||||
static void | static void | ||||
iflib_if_qflush(if_t ifp) | iflib_if_qflush(if_t ifp) | ||||
{ | { | ||||
if_ctx_t ctx = if_getsoftc(ifp); | if_ctx_t ctx = if_getsoftc(ifp); | ||||
iflib_txq_t txq = ctx->ifc_txqs; | iflib_txq_t txq = ctx->ifc_txqs; | ||||
int i; | int i; | ||||
CTX_LOCK(ctx); | CTX_LOCK(ctx); | ||||
▲ Show 20 Lines • Show All 67 Lines • ▼ Show 20 Lines | if ((err = IFDI_MTU_SET(ctx, ifr->ifr_mtu)) == 0) { | ||||
ctx->ifc_flags &= ~IFC_MULTISEG; | ctx->ifc_flags &= ~IFC_MULTISEG; | ||||
err = if_setmtu(ifp, ifr->ifr_mtu); | err = if_setmtu(ifp, ifr->ifr_mtu); | ||||
} | } | ||||
iflib_init_locked(ctx); | iflib_init_locked(ctx); | ||||
if_setdrvflags(ifp, bits); | if_setdrvflags(ifp, bits); | ||||
CTX_UNLOCK(ctx); | CTX_UNLOCK(ctx); | ||||
break; | break; | ||||
case SIOCSIFFLAGS: | case SIOCSIFFLAGS: | ||||
CTX_LOCK(ctx); | err = async_if_ioctl(ctx, command, data); | ||||
if (if_getflags(ifp) & IFF_UP) { | |||||
if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { | |||||
if ((if_getflags(ifp) ^ ctx->ifc_if_flags) & | |||||
(IFF_PROMISC | IFF_ALLMULTI)) { | |||||
err = IFDI_PROMISC_SET(ctx, if_getflags(ifp)); | |||||
} | |||||
} else | |||||
reinit = 1; | |||||
} else if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { | |||||
iflib_stop(ctx); | |||||
} | |||||
ctx->ifc_if_flags = if_getflags(ifp); | |||||
CTX_UNLOCK(ctx); | |||||
break; | break; | ||||
case SIOCADDMULTI: | case SIOCADDMULTI: | ||||
case SIOCDELMULTI: | case SIOCDELMULTI: | ||||
if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { | if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { | ||||
CTX_LOCK(ctx); | err = async_if_ioctl(ctx, command, data); | ||||
IFDI_INTR_DISABLE(ctx); | |||||
IFDI_MULTI_SET(ctx); | |||||
IFDI_INTR_ENABLE(ctx); | |||||
CTX_UNLOCK(ctx); | |||||
} | } | ||||
break; | break; | ||||
case SIOCSIFMEDIA: | case SIOCSIFMEDIA: | ||||
CTX_LOCK(ctx); | CTX_LOCK(ctx); | ||||
IFDI_MEDIA_SET(ctx); | IFDI_MEDIA_SET(ctx); | ||||
CTX_UNLOCK(ctx); | CTX_UNLOCK(ctx); | ||||
/* falls thru */ | /* falls thru */ | ||||
case SIOCGIFMEDIA: | case SIOCGIFMEDIA: | ||||
▲ Show 20 Lines • Show All 77 Lines • ▼ Show 20 Lines | |||||
} | } | ||||
/********************************************************************* | /********************************************************************* | ||||
* | * | ||||
* OTHER FUNCTIONS EXPORTED TO THE STACK | * OTHER FUNCTIONS EXPORTED TO THE STACK | ||||
* | * | ||||
**********************************************************************/ | **********************************************************************/ | ||||
/* CONFIG context only */ | |||||
static void | static void | ||||
iflib_vlan_register(void *arg, if_t ifp, uint16_t vtag) | iflib_vlan_register(void *arg, if_t ifp, uint16_t vtag) | ||||
{ | { | ||||
if_ctx_t ctx = if_getsoftc(ifp); | if_ctx_t ctx = if_getsoftc(ifp); | ||||
if ((void *)ctx != arg) | if ((void *)ctx != arg) | ||||
return; | return; | ||||
if ((vtag == 0) || (vtag > 4095)) | if ((vtag == 0) || (vtag > 4095)) | ||||
return; | return; | ||||
CTX_LOCK(ctx); | CTX_LOCK(ctx); | ||||
IFDI_VLAN_REGISTER(ctx, vtag); | IFDI_VLAN_REGISTER(ctx, vtag); | ||||
/* Re-init to load the changes */ | /* Re-init to load the changes */ | ||||
if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER) | if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER) | ||||
iflib_init_locked(ctx); | iflib_init_locked(ctx); | ||||
CTX_UNLOCK(ctx); | CTX_UNLOCK(ctx); | ||||
} | } | ||||
/* CONFIG context only */ | |||||
static void | static void | ||||
iflib_vlan_unregister(void *arg, if_t ifp, uint16_t vtag) | iflib_vlan_unregister(void *arg, if_t ifp, uint16_t vtag) | ||||
{ | { | ||||
if_ctx_t ctx = if_getsoftc(ifp); | if_ctx_t ctx = if_getsoftc(ifp); | ||||
if ((void *)ctx != arg) | if ((void *)ctx != arg) | ||||
return; | return; | ||||
if ((vtag == 0) || (vtag > 4095)) | if ((vtag == 0) || (vtag > 4095)) | ||||
return; | return; | ||||
CTX_LOCK(ctx); | CTX_LOCK(ctx); | ||||
IFDI_VLAN_UNREGISTER(ctx, vtag); | IFDI_VLAN_UNREGISTER(ctx, vtag); | ||||
/* Re-init to load the changes */ | /* Re-init to load the changes */ | ||||
if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER) | if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER) | ||||
iflib_init_locked(ctx); | iflib_init_locked(ctx); | ||||
CTX_UNLOCK(ctx); | CTX_UNLOCK(ctx); | ||||
} | } | ||||
/* CONFIG context only */ | |||||
static void | static void | ||||
iflib_led_func(void *arg, int onoff) | iflib_led_func(void *arg, int onoff) | ||||
{ | { | ||||
if_ctx_t ctx = arg; | if_ctx_t ctx = arg; | ||||
CTX_LOCK(ctx); | CTX_LOCK(ctx); | ||||
IFDI_LED_FUNC(ctx, onoff); | IFDI_LED_FUNC(ctx, onoff); | ||||
CTX_UNLOCK(ctx); | CTX_UNLOCK(ctx); | ||||
▲ Show 20 Lines • Show All 169 Lines • ▼ Show 20 Lines | #endif | ||||
/* XXX change for per-queue sizes */ | /* XXX change for per-queue sizes */ | ||||
device_printf(dev, "using %d tx descriptors and %d rx descriptors\n", | device_printf(dev, "using %d tx descriptors and %d rx descriptors\n", | ||||
scctx->isc_ntxd[main_txq], scctx->isc_nrxd[main_rxq]); | scctx->isc_ntxd[main_txq], scctx->isc_nrxd[main_rxq]); | ||||
for (i = 0; i < sctx->isc_nrxqs; i++) { | for (i = 0; i < sctx->isc_nrxqs; i++) { | ||||
if (!powerof2(scctx->isc_nrxd[i])) { | if (!powerof2(scctx->isc_nrxd[i])) { | ||||
/* round down instead? */ | /* round down instead? */ | ||||
device_printf(dev, "# rx descriptors must be a power of 2\n"); | device_printf(dev, "# rx descriptors must be a power of 2\n"); | ||||
err = EINVAL; | err = EINVAL; | ||||
goto fail; | goto fail; | ||||
} | } | ||||
} | } | ||||
for (i = 0; i < sctx->isc_ntxqs; i++) { | for (i = 0; i < sctx->isc_ntxqs; i++) { | ||||
if (!powerof2(scctx->isc_ntxd[i])) { | if (!powerof2(scctx->isc_ntxd[i])) { | ||||
device_printf(dev, | device_printf(dev, | ||||
"# tx descriptors must be a power of 2"); | "# tx descriptors must be a power of 2"); | ||||
▲ Show 20 Lines • Show All 961 Lines • ▼ Show 20 Lines | #ifdef INVARIANTS | ||||
gtask = &ctx->ifc_admin_task; | gtask = &ctx->ifc_admin_task; | ||||
MPASS(gtask->gt_taskqueue != NULL); | MPASS(gtask->gt_taskqueue != NULL); | ||||
#endif | #endif | ||||
GROUPTASK_ENQUEUE(&ctx->ifc_admin_task); | GROUPTASK_ENQUEUE(&ctx->ifc_admin_task); | ||||
} | } | ||||
/* CONFIG context only */ | |||||
static void | |||||
iflib_handle_reset(if_ctx_t ctx, void *arg) | |||||
{ | |||||
CTX_LOCK(ctx); | |||||
ctx->ifc_flags |= IFC_DO_RESET; | |||||
iflib_admin_intr_deferred(ctx); | |||||
CTX_UNLOCK(ctx); | |||||
} | |||||
static void | |||||
iflib_admin_reset_deferred(if_ctx_t ctx) | |||||
{ | |||||
iflib_config_async_gtask_dispatch(ctx, iflib_handle_reset, "reset handler", NULL); | |||||
} | |||||
void | void | ||||
iflib_iov_intr_deferred(if_ctx_t ctx) | iflib_iov_intr_deferred(if_ctx_t ctx) | ||||
{ | { | ||||
GROUPTASK_ENQUEUE(&ctx->ifc_vflr_task); | GROUPTASK_ENQUEUE(&ctx->ifc_vflr_task); | ||||
} | } | ||||
void | void | ||||
iflib_io_tqg_attach(struct grouptask *gt, void *uniq, int cpu, char *name) | iflib_io_tqg_attach(struct grouptask *gt, void *uniq, int cpu, char *name) | ||||
{ | { | ||||
taskqgroup_attach_cpu(qgroup_if_io_tqg, gt, uniq, cpu, -1, name); | taskqgroup_attach_cpu(qgroup_if_io_tqg, gt, uniq, cpu, -1, name); | ||||
} | } | ||||
void | void | ||||
iflib_config_gtask_init(if_ctx_t ctx, struct grouptask *gtask, gtask_fn_t *fn, | iflib_config_gtask_init(if_ctx_t ctx, struct grouptask *gtask, gtask_fn_t *fn, | ||||
char *name) | char *name) | ||||
{ | { | ||||
GROUPTASK_INIT(gtask, 0, fn, ctx); | GROUPTASK_INIT(gtask, 0, fn, ctx); | ||||
taskqgroup_attach(qgroup_if_config_tqg, gtask, gtask, -1, name); | taskqgroup_attach(qgroup_if_config_tqg, gtask, gtask, -1, name); | ||||
} | } | ||||
static void | |||||
iflib_multi_set(if_ctx_t ctx, void *arg) | |||||
{ | |||||
CTX_LOCK(ctx); | |||||
IFDI_INTR_DISABLE(ctx); | |||||
IFDI_MULTI_SET(ctx); | |||||
IFDI_INTR_ENABLE(ctx); | |||||
CTX_UNLOCK(ctx); | |||||
} | |||||
static void | |||||
iflib_flags_set(if_ctx_t ctx, void *arg) | |||||
{ | |||||
int reinit, err; | |||||
if_t ifp = ctx->ifc_ifp; | |||||
err = reinit = 0; | |||||
CTX_LOCK(ctx); | |||||
if (if_getflags(ifp) & IFF_UP) { | |||||
if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { | |||||
if ((if_getflags(ifp) ^ ctx->ifc_if_flags) & | |||||
(IFF_PROMISC | IFF_ALLMULTI)) { | |||||
err = IFDI_PROMISC_SET(ctx, if_getflags(ifp)); | |||||
} | |||||
} else | |||||
reinit = 1; | |||||
} else if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { | |||||
iflib_stop(ctx); | |||||
} | |||||
ctx->ifc_if_flags = if_getflags(ifp); | |||||
if (reinit) | |||||
iflib_if_init_locked(ctx); | |||||
CTX_UNLOCK(ctx); | |||||
if (err) | |||||
log(LOG_WARNING, "IFDI_PROMISC_SET returned %d\n", err); | |||||
} | |||||
static void | |||||
async_gtask(void *ctx) | |||||
{ | |||||
struct async_task_arg *at_arg = ctx; | |||||
if_ctx_t if_ctx = at_arg->ata_ctx; | |||||
void *arg = at_arg->ata_arg; | |||||
at_arg->ata_fn(if_ctx, arg); | |||||
taskqgroup_detach(qgroup_if_config_tqg, at_arg->ata_gtask); | |||||
free(at_arg->ata_gtask, M_IFLIB); | |||||
} | |||||
static int | |||||
iflib_config_async_gtask_dispatch(if_ctx_t ctx, async_gtask_fn_t *fn, char *name, void *arg) | |||||
{ | |||||
struct grouptask *gtask; | |||||
struct async_task_arg *at_arg; | |||||
if ((gtask = malloc(sizeof(struct grouptask) + sizeof(struct async_task_arg), M_IFLIB, M_NOWAIT|M_ZERO)) == NULL) | |||||
return (ENOMEM); | |||||
at_arg = (struct async_task_arg *)(gtask + 1); | |||||
at_arg->ata_fn = fn; | |||||
at_arg->ata_ctx = ctx; | |||||
at_arg->ata_arg = arg; | |||||
at_arg->ata_gtask = gtask; | |||||
GROUPTASK_INIT(gtask, 0, async_gtask, at_arg); | |||||
taskqgroup_attach(qgroup_if_config_tqg, gtask, gtask, -1, name); | |||||
GROUPTASK_ENQUEUE(gtask); | |||||
return (0); | |||||
} | |||||
gallatinUnsubmitted Not Done Inline ActionsHow does the actual error status get returned via ifioctl? If I'm reading this right, all you can return via this mechanism is the whether you were able to enqueue the task, not the status of the task's work. gallatin: How does the actual error status get returned via ifioctl? If I'm reading this right, all you… | |||||
kmacyAuthorUnsubmitted Not Done Inline ActionsCurrently these are only used for actions that can't fail or that the driver won't report failure on. The model is currently busted to begin with in that these hardware transactions can take longer than we should be holding non-sleepable locks. Ideally multicast would be reworked, but it's not in anyone's critical path. kmacy: Currently these are only used for actions that can't fail or that the driver won't report… | |||||
static int | |||||
async_if_ioctl(if_ctx_t ctx, u_long command, caddr_t data) | |||||
{ | |||||
int rc; | |||||
switch (command) { | |||||
case SIOCADDMULTI: | |||||
case SIOCDELMULTI: | |||||
rc = iflib_config_async_gtask_dispatch(ctx, iflib_multi_set, "async_if_multi", NULL); | |||||
break; | |||||
case SIOCSIFFLAGS: | |||||
rc = iflib_config_async_gtask_dispatch(ctx, iflib_flags_set, "async_if_flags", NULL); | |||||
break; | |||||
default: | |||||
panic("unknown command %lx", command); | |||||
} | |||||
return (rc); | |||||
} | |||||
void | void | ||||
iflib_config_gtask_deinit(struct grouptask *gtask) | iflib_config_gtask_deinit(struct grouptask *gtask) | ||||
{ | { | ||||
taskqgroup_detach(qgroup_if_config_tqg, gtask); | taskqgroup_detach(qgroup_if_config_tqg, gtask); | ||||
} | } | ||||
void | void | ||||
iflib_link_state_change(if_ctx_t ctx, int link_state, uint64_t baudrate) | iflib_link_state_change(if_ctx_t ctx, int link_state, uint64_t baudrate) | ||||
{ | { | ||||
if_t ifp = ctx->ifc_ifp; | if_t ifp = ctx->ifc_ifp; | ||||
iflib_txq_t txq = ctx->ifc_txqs; | iflib_txq_t txq = ctx->ifc_txqs; | ||||
▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Lines | iflib_add_int_delay_sysctl(if_ctx_t ctx, const char *name, | ||||
info->iidi_offset = offset; | info->iidi_offset = offset; | ||||
info->iidi_value = value; | info->iidi_value = value; | ||||
SYSCTL_ADD_PROC(device_get_sysctl_ctx(ctx->ifc_dev), | SYSCTL_ADD_PROC(device_get_sysctl_ctx(ctx->ifc_dev), | ||||
SYSCTL_CHILDREN(device_get_sysctl_tree(ctx->ifc_dev)), | SYSCTL_CHILDREN(device_get_sysctl_tree(ctx->ifc_dev)), | ||||
OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, | OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, | ||||
info, 0, iflib_sysctl_int_delay, "I", description); | info, 0, iflib_sysctl_int_delay, "I", description); | ||||
} | } | ||||
struct mtx * | struct sx * | ||||
iflib_ctx_lock_get(if_ctx_t ctx) | iflib_ctx_lock_get(if_ctx_t ctx) | ||||
{ | { | ||||
return (&ctx->ifc_mtx); | return (&ctx->ifc_sx); | ||||
} | } | ||||
static int | static int | ||||
iflib_msix_init(if_ctx_t ctx) | iflib_msix_init(if_ctx_t ctx) | ||||
{ | { | ||||
device_t dev = ctx->ifc_dev; | device_t dev = ctx->ifc_dev; | ||||
if_shared_ctx_t sctx = ctx->ifc_sctx; | if_shared_ctx_t sctx = ctx->ifc_sctx; | ||||
if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; | if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; | ||||
▲ Show 20 Lines • Show All 449 Lines • Show Last 20 Lines |
These should probably just be deleted.