Changeset View
Changeset View
Standalone View
Standalone View
sys/net/iflib.c
Show First 20 Lines • Show All 45 Lines • ▼ Show 20 Lines | |||||
#include <sys/sbuf.h> | #include <sys/sbuf.h> | ||||
#include <sys/smp.h> | #include <sys/smp.h> | ||||
#include <sys/socket.h> | #include <sys/socket.h> | ||||
#include <sys/sysctl.h> | #include <sys/sysctl.h> | ||||
#include <sys/syslog.h> | #include <sys/syslog.h> | ||||
#include <sys/taskqueue.h> | #include <sys/taskqueue.h> | ||||
#include <sys/limits.h> | #include <sys/limits.h> | ||||
#include <net/if.h> | #include <net/if.h> | ||||
#include <net/if_var.h> | #include <net/if_var.h> | ||||
#include <net/if_types.h> | #include <net/if_types.h> | ||||
#include <net/if_media.h> | #include <net/if_media.h> | ||||
#include <net/bpf.h> | #include <net/bpf.h> | ||||
#include <net/ethernet.h> | #include <net/ethernet.h> | ||||
#include <net/mp_ring.h> | #include <net/mp_ring.h> | ||||
▲ Show 20 Lines • Show All 89 Lines • ▼ Show 20 Lines | struct iflib_ctx { | ||||
void *ifc_softc; | void *ifc_softc; | ||||
device_t ifc_dev; | device_t ifc_dev; | ||||
if_t ifc_ifp; | if_t ifc_ifp; | ||||
cpuset_t ifc_cpus; | cpuset_t ifc_cpus; | ||||
if_shared_ctx_t ifc_sctx; | if_shared_ctx_t ifc_sctx; | ||||
struct if_softc_ctx ifc_softc_ctx; | struct if_softc_ctx ifc_softc_ctx; | ||||
struct mtx ifc_mtx; | struct sx ifc_sx; | ||||
uint16_t ifc_nhwtxqs; | uint16_t ifc_nhwtxqs; | ||||
uint16_t ifc_nhwrxqs; | uint16_t ifc_nhwrxqs; | ||||
iflib_txq_t ifc_txqs; | iflib_txq_t ifc_txqs; | ||||
iflib_rxq_t ifc_rxqs; | iflib_rxq_t ifc_rxqs; | ||||
uint32_t ifc_if_flags; | uint32_t ifc_if_flags; | ||||
uint32_t ifc_flags; | uint32_t ifc_flags; | ||||
Show All 11 Lines | struct iflib_ctx { | ||||
struct grouptask ifc_vflr_task; | struct grouptask ifc_vflr_task; | ||||
struct iflib_filter_info ifc_filter_info; | struct iflib_filter_info ifc_filter_info; | ||||
struct ifmedia ifc_media; | struct ifmedia ifc_media; | ||||
struct sysctl_oid *ifc_sysctl_node; | struct sysctl_oid *ifc_sysctl_node; | ||||
uint16_t ifc_sysctl_ntxqs; | uint16_t ifc_sysctl_ntxqs; | ||||
uint16_t ifc_sysctl_nrxqs; | uint16_t ifc_sysctl_nrxqs; | ||||
uint16_t ifc_sysctl_qs_eq_override; | uint16_t ifc_sysctl_qs_eq_override; | ||||
uint16_t ifc_cpuid_highest; | |||||
uint16_t ifc_sysctl_rx_budget; | |||||
qidx_t ifc_sysctl_ntxds[8]; | qidx_t ifc_sysctl_ntxds[8]; | ||||
qidx_t ifc_sysctl_nrxds[8]; | qidx_t ifc_sysctl_nrxds[8]; | ||||
struct if_txrx ifc_txrx; | struct if_txrx ifc_txrx; | ||||
#define isc_txd_encap ifc_txrx.ift_txd_encap | #define isc_txd_encap ifc_txrx.ift_txd_encap | ||||
#define isc_txd_flush ifc_txrx.ift_txd_flush | #define isc_txd_flush ifc_txrx.ift_txd_flush | ||||
#define isc_txd_credits_update ifc_txrx.ift_txd_credits_update | #define isc_txd_credits_update ifc_txrx.ift_txd_credits_update | ||||
#define isc_rxd_available ifc_txrx.ift_rxd_available | #define isc_rxd_available ifc_txrx.ift_rxd_available | ||||
#define isc_rxd_pkt_get ifc_txrx.ift_rxd_pkt_get | #define isc_rxd_pkt_get ifc_txrx.ift_rxd_pkt_get | ||||
#define isc_rxd_refill ifc_txrx.ift_rxd_refill | #define isc_rxd_refill ifc_txrx.ift_rxd_refill | ||||
#define isc_rxd_flush ifc_txrx.ift_rxd_flush | #define isc_rxd_flush ifc_txrx.ift_rxd_flush | ||||
#define isc_rxd_refill ifc_txrx.ift_rxd_refill | #define isc_rxd_refill ifc_txrx.ift_rxd_refill | ||||
#define isc_rxd_refill ifc_txrx.ift_rxd_refill | #define isc_rxd_refill ifc_txrx.ift_rxd_refill | ||||
#define isc_legacy_intr ifc_txrx.ift_legacy_intr | #define isc_legacy_intr ifc_txrx.ift_legacy_intr | ||||
eventhandler_tag ifc_vlan_attach_event; | eventhandler_tag ifc_vlan_attach_event; | ||||
eventhandler_tag ifc_vlan_detach_event; | eventhandler_tag ifc_vlan_detach_event; | ||||
uint8_t ifc_mac[ETHER_ADDR_LEN]; | uint8_t ifc_mac[ETHER_ADDR_LEN]; | ||||
char ifc_mtx_name[16]; | char ifc_mtx_name[16]; | ||||
LIST_ENTRY(iflib_ctx) ifc_next; | |||||
}; | }; | ||||
static LIST_HEAD(ctx_head, iflib_ctx) ctx_list; | |||||
static struct mtx ctx_list_lock; | |||||
TASKQGROUP_DEFINE(if_io, mp_ncpus, 1, true, PI_NET); | |||||
TASKQGROUP_DEFINE(if_config, 1, 1, false, PI_SOFT); | |||||
static void | |||||
iflib_ctx_apply(void (*fn)(if_ctx_t ctx, void *arg), void *arg) | |||||
{ | |||||
if_ctx_t ctx; | |||||
mtx_lock(&ctx_list_lock); | |||||
LIST_FOREACH(ctx, &ctx_list, ifc_next) { | |||||
(fn)(ctx, arg); | |||||
} | |||||
mtx_unlock(&ctx_list_lock); | |||||
} | |||||
static void | |||||
_iflib_cpuid_highest(if_ctx_t ctx, void *arg) { | |||||
int *cpuid = arg; | |||||
if (*cpuid < ctx->ifc_cpuid_highest) | |||||
*cpuid = ctx->ifc_cpuid_highest; | |||||
} | |||||
static int | |||||
iflib_cpuid_highest(void) | |||||
{ | |||||
int cpuid = 0; | |||||
iflib_ctx_apply(_iflib_cpuid_highest, &cpuid); | |||||
return (cpuid); | |||||
} | |||||
static void | |||||
iflib_ctx_insert(if_ctx_t ctx) | |||||
{ | |||||
mtx_lock(&ctx_list_lock); | |||||
LIST_INSERT_HEAD(&ctx_list, ctx, ifc_next); | |||||
mtx_unlock(&ctx_list_lock); | |||||
} | |||||
static void | |||||
iflib_ctx_remove(if_ctx_t ctx) | |||||
{ | |||||
int max_cpuid_prev, max_cpuid_new; | |||||
max_cpuid_prev = iflib_cpuid_highest(); | |||||
mtx_lock(&ctx_list_lock); | |||||
LIST_REMOVE(ctx, ifc_next); | |||||
mtx_unlock(&ctx_list_lock); | |||||
max_cpuid_new = max(1, iflib_cpuid_highest()); | |||||
if (max_cpuid_new < max_cpuid_prev) { | |||||
taskqgroup_adjust(qgroup_if_io, max_cpuid_new, 1, true, PI_NET); | |||||
} | |||||
} | |||||
void * | void * | ||||
iflib_get_softc(if_ctx_t ctx) | iflib_get_softc(if_ctx_t ctx) | ||||
{ | { | ||||
return (ctx->ifc_softc); | return (ctx->ifc_softc); | ||||
} | } | ||||
device_t | device_t | ||||
▲ Show 20 Lines • Show All 41 Lines • ▼ Show 20 Lines | |||||
#define IP_ALIGNED(m) ((((uintptr_t)(m)->m_data) & 0x3) == 0x2) | #define IP_ALIGNED(m) ((((uintptr_t)(m)->m_data) & 0x3) == 0x2) | ||||
#define CACHE_PTR_INCREMENT (CACHE_LINE_SIZE/sizeof(void*)) | #define CACHE_PTR_INCREMENT (CACHE_LINE_SIZE/sizeof(void*)) | ||||
#define CACHE_PTR_NEXT(ptr) ((void *)(((uintptr_t)(ptr)+CACHE_LINE_SIZE-1) & (CACHE_LINE_SIZE-1))) | #define CACHE_PTR_NEXT(ptr) ((void *)(((uintptr_t)(ptr)+CACHE_LINE_SIZE-1) & (CACHE_LINE_SIZE-1))) | ||||
#define LINK_ACTIVE(ctx) ((ctx)->ifc_link_state == LINK_STATE_UP) | #define LINK_ACTIVE(ctx) ((ctx)->ifc_link_state == LINK_STATE_UP) | ||||
#define CTX_IS_VF(ctx) ((ctx)->ifc_sctx->isc_flags & IFLIB_IS_VF) | #define CTX_IS_VF(ctx) ((ctx)->ifc_sctx->isc_flags & IFLIB_IS_VF) | ||||
#define RX_SW_DESC_MAP_CREATED (1 << 0) | #define RX_SW_DESC_MAP_CREATED (1 << 0) | ||||
#define TX_SW_DESC_MAP_CREATED (1 << 1) | #define RX_SW_DESC_INUSE (1 << 1) | ||||
#define RX_SW_DESC_INUSE (1 << 3) | #define RX_NETMAP_INUSE (1 << 2) | ||||
#define TX_SW_DESC_MAPPED (1 << 4) | |||||
#define TX_SW_DESC_MAP_CREATED (1 << 0) | |||||
#define TX_SW_DESC_MAPPED (1 << 1) | |||||
#define M_TOOBIG M_PROTO1 | #define M_TOOBIG M_PROTO1 | ||||
typedef struct iflib_sw_rx_desc_array { | typedef struct iflib_sw_rx_desc_array { | ||||
bus_dmamap_t *ifsd_map; /* bus_dma maps for packet */ | bus_dmamap_t *ifsd_map; /* bus_dma maps for packet */ | ||||
struct mbuf **ifsd_m; /* pkthdr mbufs */ | struct mbuf **ifsd_m; /* pkthdr mbufs */ | ||||
caddr_t *ifsd_cl; /* direct cluster pointer for rx */ | caddr_t *ifsd_cl; /* direct cluster pointer for rx */ | ||||
uint8_t *ifsd_flags; | uint8_t *ifsd_flags; | ||||
} iflib_rxsd_array_t; | } iflib_rxsd_array_t; | ||||
▲ Show 20 Lines • Show All 74 Lines • ▼ Show 20 Lines | #endif | ||||
qidx_t ift_size; | qidx_t ift_size; | ||||
uint16_t ift_id; | uint16_t ift_id; | ||||
struct callout ift_timer; | struct callout ift_timer; | ||||
if_txsd_vec_t ift_sds; | if_txsd_vec_t ift_sds; | ||||
uint8_t ift_qstatus; | uint8_t ift_qstatus; | ||||
uint8_t ift_closed; | uint8_t ift_closed; | ||||
uint8_t ift_update_freq; | uint8_t ift_update_freq; | ||||
uint8_t ift_stall_count; | |||||
struct iflib_filter_info ift_filter_info; | struct iflib_filter_info ift_filter_info; | ||||
bus_dma_tag_t ift_desc_tag; | bus_dma_tag_t ift_desc_tag; | ||||
bus_dma_tag_t ift_tso_desc_tag; | bus_dma_tag_t ift_tso_desc_tag; | ||||
iflib_dma_info_t ift_ifdi; | iflib_dma_info_t ift_ifdi; | ||||
#define MTX_NAME_LEN 16 | #define MTX_NAME_LEN 16 | ||||
char ift_mtx_name[MTX_NAME_LEN]; | char ift_mtx_name[MTX_NAME_LEN]; | ||||
char ift_db_mtx_name[MTX_NAME_LEN]; | char ift_db_mtx_name[MTX_NAME_LEN]; | ||||
bus_dma_segment_t ift_segs[IFLIB_MAX_TX_SEGS] __aligned(CACHE_LINE_SIZE); | bus_dma_segment_t ift_segs[IFLIB_MAX_TX_SEGS] __aligned(CACHE_LINE_SIZE); | ||||
▲ Show 20 Lines • Show All 75 Lines • ▼ Show 20 Lines | struct iflib_rxq { | ||||
uint8_t ifr_lro_enabled; | uint8_t ifr_lro_enabled; | ||||
uint8_t ifr_nfl; | uint8_t ifr_nfl; | ||||
uint8_t ifr_ntxqirq; | uint8_t ifr_ntxqirq; | ||||
uint8_t ifr_txqid[IFLIB_MAX_TX_SHARED_INTR]; | uint8_t ifr_txqid[IFLIB_MAX_TX_SHARED_INTR]; | ||||
struct lro_ctrl ifr_lc; | struct lro_ctrl ifr_lc; | ||||
struct grouptask ifr_task; | struct grouptask ifr_task; | ||||
struct iflib_filter_info ifr_filter_info; | struct iflib_filter_info ifr_filter_info; | ||||
iflib_dma_info_t ifr_ifdi; | iflib_dma_info_t ifr_ifdi; | ||||
struct if_rxd_info ifr_ri; | |||||
struct if_rxd_update ifr_iru; | |||||
/* dynamically allocate if any drivers need a value substantially larger than this */ | /* dynamically allocate if any drivers need a value substantially larger than this */ | ||||
struct if_rxd_frag ifr_frags[IFLIB_MAX_RX_SEGS] __aligned(CACHE_LINE_SIZE); | struct if_rxd_frag ifr_frags[IFLIB_MAX_RX_SEGS] __aligned(CACHE_LINE_SIZE); | ||||
#ifdef IFLIB_DIAGNOSTICS | #ifdef IFLIB_DIAGNOSTICS | ||||
uint64_t ifr_cpu_exec_count[256]; | uint64_t ifr_cpu_exec_count[256]; | ||||
#endif | #endif | ||||
} __aligned(CACHE_LINE_SIZE); | } __aligned(CACHE_LINE_SIZE); | ||||
typedef struct if_rxsd { | typedef struct if_rxsd { | ||||
caddr_t *ifsd_cl; | caddr_t *ifsd_cl; | ||||
struct mbuf **ifsd_m; | struct mbuf **ifsd_m; | ||||
iflib_fl_t ifsd_fl; | iflib_fl_t ifsd_fl; | ||||
qidx_t ifsd_cidx; | qidx_t ifsd_cidx; | ||||
} *if_rxsd_t; | } *if_rxsd_t; | ||||
/* multiple of word size */ | /* multiple of word size */ | ||||
#ifdef __LP64__ | #ifdef __LP64__ | ||||
#define PKT_INFO_SIZE 6 | #define PKT_INFO_SIZE 7 | ||||
#define RXD_INFO_SIZE 5 | #define RXD_INFO_SIZE 5 | ||||
#define PKT_TYPE uint64_t | #define PKT_TYPE uint64_t | ||||
#else | #else | ||||
#define PKT_INFO_SIZE 11 | #define PKT_INFO_SIZE 12 | ||||
#define RXD_INFO_SIZE 8 | #define RXD_INFO_SIZE 8 | ||||
#define PKT_TYPE uint32_t | #define PKT_TYPE uint32_t | ||||
#endif | #endif | ||||
#define PKT_LOOP_BOUND ((PKT_INFO_SIZE/3)*3) | #define PKT_LOOP_BOUND ((PKT_INFO_SIZE/3)*3) | ||||
#define RXD_LOOP_BOUND ((RXD_INFO_SIZE/4)*4) | #define RXD_LOOP_BOUND ((RXD_INFO_SIZE/4)*4) | ||||
typedef struct if_pkt_info_pad { | typedef struct if_pkt_info_pad { | ||||
PKT_TYPE pkt_val[PKT_INFO_SIZE]; | PKT_TYPE pkt_val[PKT_INFO_SIZE]; | ||||
Show All 9 Lines | |||||
static inline void | static inline void | ||||
pkt_info_zero(if_pkt_info_t pi) | pkt_info_zero(if_pkt_info_t pi) | ||||
{ | { | ||||
if_pkt_info_pad_t pi_pad; | if_pkt_info_pad_t pi_pad; | ||||
pi_pad = (if_pkt_info_pad_t)pi; | pi_pad = (if_pkt_info_pad_t)pi; | ||||
pi_pad->pkt_val[0] = 0; pi_pad->pkt_val[1] = 0; pi_pad->pkt_val[2] = 0; | pi_pad->pkt_val[0] = 0; pi_pad->pkt_val[1] = 0; pi_pad->pkt_val[2] = 0; | ||||
pi_pad->pkt_val[3] = 0; pi_pad->pkt_val[4] = 0; pi_pad->pkt_val[5] = 0; | pi_pad->pkt_val[3] = 0; pi_pad->pkt_val[4] = 0; pi_pad->pkt_val[5] = 0; | ||||
pi_pad->pkt_val[6] = 0; | |||||
#ifndef __LP64__ | #ifndef __LP64__ | ||||
pi_pad->pkt_val[6] = 0; pi_pad->pkt_val[7] = 0; pi_pad->pkt_val[8] = 0; | pi_pad->pkt_val[7] = 0; pi_pad->pkt_val[8] = 0; pi_pad->pkt_val[9] = 0; | ||||
pi_pad->pkt_val[9] = 0; pi_pad->pkt_val[10] = 0; | pi_pad->pkt_val[10] = 0; pi_pad->pkt_val[11] = 0; | ||||
#endif | #endif | ||||
} | } | ||||
static inline void | static inline void | ||||
rxd_info_zero(if_rxd_info_t ri) | rxd_info_zero(if_rxd_info_t ri) | ||||
{ | { | ||||
if_rxd_info_pad_t ri_pad; | if_rxd_info_pad_t ri_pad; | ||||
int i; | int i; | ||||
Show All 11 Lines | |||||
} | } | ||||
/* | /* | ||||
* Only allow a single packet to take up most 1/nth of the tx ring | * Only allow a single packet to take up most 1/nth of the tx ring | ||||
*/ | */ | ||||
#define MAX_SINGLE_PACKET_FRACTION 12 | #define MAX_SINGLE_PACKET_FRACTION 12 | ||||
#define IF_BAD_DMA (bus_addr_t)-1 | #define IF_BAD_DMA (bus_addr_t)-1 | ||||
#define CTX_ACTIVE(ctx) ((if_getdrvflags((ctx)->ifc_ifp) & IFF_DRV_RUNNING)) | static SYSCTL_NODE(_net, OID_AUTO, iflib, CTLFLAG_RD, 0, | ||||
"iflib driver parameters"); | |||||
#define CTX_LOCK_INIT(_sc, _name) mtx_init(&(_sc)->ifc_mtx, _name, "iflib ctx lock", MTX_DEF) | static int iflib_timer_int; | ||||
SYSCTL_INT(_net_iflib, OID_AUTO, timer_int, CTLFLAG_RW, &iflib_timer_int, | |||||
0, "interval at which to run per-queue timers (in ticks)"); | |||||
#define CTX_LOCK(ctx) mtx_lock(&(ctx)->ifc_mtx) | static int force_busdma = 0; | ||||
#define CTX_UNLOCK(ctx) mtx_unlock(&(ctx)->ifc_mtx) | SYSCTL_INT(_net_iflib, OID_AUTO, force_busdma, CTLFLAG_RDTUN, &force_busdma, | ||||
#define CTX_LOCK_DESTROY(ctx) mtx_destroy(&(ctx)->ifc_mtx) | 1, "force busdma"); | ||||
#define CTX_ACTIVE(ctx) ((if_getdrvflags((ctx)->ifc_ifp) & IFF_DRV_RUNNING)) | |||||
#define CTX_LOCK_INIT(_sc, _name) sx_init(&(_sc)->ifc_sx, _name) | |||||
cramerj_intel.com: Are these sysctls per-device? They probably should be, no? And there doesn't seem to be any… | |||||
Not Done Inline ActionsThey are not per-device... I don't see a reason they couldn't be, but they're also fairly blunt tools that hopefully wouldn't need to be used in a production system. force_busdma and iflib_timer_int are both used, but enable_msix is not. I'll remove enable_msix. shurd: They are not per-device... I don't see a reason they couldn't be, but they're also fairly blunt… | |||||
#define CTX_LOCK(ctx) sx_xlock(&(ctx)->ifc_sx) | |||||
#define CTX_UNLOCK(ctx) sx_xunlock(&(ctx)->ifc_sx) | |||||
#define CTX_LOCK_DESTROY(ctx) sx_destroy(&(ctx)->ifc_sx) | |||||
#define CALLOUT_LOCK(txq) mtx_lock(&txq->ift_mtx) | #define CALLOUT_LOCK(txq) mtx_lock(&txq->ift_mtx) | ||||
#define CALLOUT_UNLOCK(txq) mtx_unlock(&txq->ift_mtx) | #define CALLOUT_UNLOCK(txq) mtx_unlock(&txq->ift_mtx) | ||||
/* Our boot-time initialization hook */ | /* Our boot-time initialization hook */ | ||||
static int iflib_module_event_handler(module_t, int, void *); | static int iflib_module_event_handler(module_t, int, void *); | ||||
static moduledata_t iflib_moduledata = { | static moduledata_t iflib_moduledata = { | ||||
"iflib", | "iflib", | ||||
iflib_module_event_handler, | iflib_module_event_handler, | ||||
NULL | NULL | ||||
}; | }; | ||||
DECLARE_MODULE(iflib, iflib_moduledata, SI_SUB_INIT_IF, SI_ORDER_ANY); | DECLARE_MODULE(iflib, iflib_moduledata, SI_SUB_INIT_IF, SI_ORDER_ANY); | ||||
MODULE_VERSION(iflib, 1); | MODULE_VERSION(iflib, 1); | ||||
MODULE_DEPEND(iflib, pci, 1, 1, 1); | MODULE_DEPEND(iflib, pci, 1, 1, 1); | ||||
MODULE_DEPEND(iflib, ether, 1, 1, 1); | MODULE_DEPEND(iflib, ether, 1, 1, 1); | ||||
TASKQGROUP_DEFINE(if_io_tqg, mp_ncpus, 1); | |||||
TASKQGROUP_DEFINE(if_config_tqg, 1, 1); | |||||
#ifndef IFLIB_DEBUG_COUNTERS | #ifndef IFLIB_DEBUG_COUNTERS | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
#define IFLIB_DEBUG_COUNTERS 1 | #define IFLIB_DEBUG_COUNTERS 1 | ||||
#else | #else | ||||
#define IFLIB_DEBUG_COUNTERS 0 | #define IFLIB_DEBUG_COUNTERS 0 | ||||
#endif /* !INVARIANTS */ | #endif /* !INVARIANTS */ | ||||
#endif | #endif | ||||
static SYSCTL_NODE(_net, OID_AUTO, iflib, CTLFLAG_RD, 0, | |||||
"iflib driver parameters"); | |||||
/* | /* | ||||
* XXX need to ensure that this can't accidentally cause the head to be moved backwards | * XXX need to ensure that this can't accidentally cause the head to be moved backwards | ||||
*/ | */ | ||||
static int iflib_min_tx_latency = 0; | static int iflib_min_tx_latency = 0; | ||||
SYSCTL_INT(_net_iflib, OID_AUTO, min_tx_latency, CTLFLAG_RW, | SYSCTL_INT(_net_iflib, OID_AUTO, min_tx_latency, CTLFLAG_RW, | ||||
&iflib_min_tx_latency, 0, "minimize transmit latency at the possible expense of throughput"); | &iflib_min_tx_latency, 0, "minimize transmit latency at the possible expense of throughput"); | ||||
static int iflib_no_tx_batch = 0; | static int iflib_no_tx_batch = 0; | ||||
SYSCTL_INT(_net_iflib, OID_AUTO, no_tx_batch, CTLFLAG_RW, | SYSCTL_INT(_net_iflib, OID_AUTO, no_tx_batch, CTLFLAG_RW, | ||||
▲ Show 20 Lines • Show All 106 Lines • ▼ Show 20 Lines | iflib_tx_seen = iflib_tx_sent = iflib_tx_encap = iflib_rx_allocs = | ||||
iflib_rx_mbuf_null = iflib_rxd_flush = 0; | iflib_rx_mbuf_null = iflib_rxd_flush = 0; | ||||
} | } | ||||
#else | #else | ||||
#define DBG_COUNTER_INC(name) | #define DBG_COUNTER_INC(name) | ||||
static void iflib_debug_reset(void) {} | static void iflib_debug_reset(void) {} | ||||
#endif | #endif | ||||
typedef void async_gtask_fn_t(if_ctx_t ctx, void *arg); | |||||
struct async_task_arg { | |||||
async_gtask_fn_t *ata_fn; | |||||
if_ctx_t ata_ctx; | |||||
void *ata_arg; | |||||
struct grouptask *ata_gtask; | |||||
}; | |||||
#define IFLIB_DEBUG 0 | #define IFLIB_DEBUG 0 | ||||
static void iflib_tx_structures_free(if_ctx_t ctx); | static void iflib_tx_structures_free(if_ctx_t ctx); | ||||
static void iflib_rx_structures_free(if_ctx_t ctx); | static void iflib_rx_structures_free(if_ctx_t ctx); | ||||
static int iflib_queues_alloc(if_ctx_t ctx); | static int iflib_queues_alloc(if_ctx_t ctx); | ||||
static int iflib_tx_credits_update(if_ctx_t ctx, iflib_txq_t txq); | static int iflib_tx_credits_update(if_ctx_t ctx, iflib_txq_t txq); | ||||
static int iflib_rxd_avail(if_ctx_t ctx, iflib_rxq_t rxq, qidx_t cidx, qidx_t budget); | static int iflib_rxd_avail(if_ctx_t ctx, iflib_rxq_t rxq, qidx_t cidx, qidx_t budget); | ||||
static int iflib_qset_structures_setup(if_ctx_t ctx); | static int iflib_qset_structures_setup(if_ctx_t ctx); | ||||
static int iflib_msix_init(if_ctx_t ctx); | static int iflib_msix_init(if_ctx_t ctx); | ||||
static int iflib_legacy_setup(if_ctx_t ctx, driver_filter_t filter, void *filterarg, int *rid, char *str); | static int iflib_legacy_setup(if_ctx_t ctx, driver_filter_t filter, void *filterarg, int *rid, char *str); | ||||
static void iflib_txq_check_drain(iflib_txq_t txq, int budget); | static void iflib_txq_check_drain(iflib_txq_t txq, int budget); | ||||
static uint32_t iflib_txq_can_drain(struct ifmp_ring *); | static uint32_t iflib_txq_can_drain(struct ifmp_ring *); | ||||
static int iflib_register(if_ctx_t); | static int iflib_register(if_ctx_t); | ||||
static void iflib_init_locked(if_ctx_t ctx); | static void iflib_init_locked(if_ctx_t ctx); | ||||
static void iflib_add_device_sysctl_pre(if_ctx_t ctx); | static void iflib_add_device_sysctl_pre(if_ctx_t ctx); | ||||
static void iflib_add_device_sysctl_post(if_ctx_t ctx); | static void iflib_add_device_sysctl_post(if_ctx_t ctx); | ||||
static void iflib_ifmp_purge(iflib_txq_t txq); | static void iflib_ifmp_purge(iflib_txq_t txq); | ||||
static void _iflib_pre_assert(if_softc_ctx_t scctx); | static void _iflib_pre_assert(if_softc_ctx_t scctx); | ||||
static void iflib_stop(if_ctx_t ctx); | static void iflib_stop(if_ctx_t ctx); | ||||
static void iflib_if_init_locked(if_ctx_t ctx); | static void iflib_if_init_locked(if_ctx_t ctx); | ||||
static int async_if_ioctl(if_ctx_t ctx, u_long command, caddr_t data); | |||||
static int iflib_config_async_gtask_dispatch(if_ctx_t ctx, async_gtask_fn_t *fn, char *name, void *arg); | |||||
static void iflib_admin_reset_deferred(if_ctx_t ctx); | |||||
#ifndef __NO_STRICT_ALIGNMENT | #ifndef __NO_STRICT_ALIGNMENT | ||||
static struct mbuf * iflib_fixup_rx(struct mbuf *m); | static struct mbuf * iflib_fixup_rx(struct mbuf *m); | ||||
#endif | #endif | ||||
#ifdef DEV_NETMAP | #ifdef DEV_NETMAP | ||||
#include <sys/selinfo.h> | #include <sys/selinfo.h> | ||||
#include <net/netmap.h> | #include <net/netmap.h> | ||||
#include <dev/netmap/netmap_kern.h> | #include <dev/netmap/netmap_kern.h> | ||||
▲ Show 20 Lines • Show All 57 Lines • ▼ Show 20 Lines | iflib_netmap_register(struct netmap_adapter *na, int onoff) | ||||
IFDI_CRCSTRIP_SET(ctx, onoff, iflib_crcstrip); // XXX why twice ? | IFDI_CRCSTRIP_SET(ctx, onoff, iflib_crcstrip); // XXX why twice ? | ||||
status = ifp->if_drv_flags & IFF_DRV_RUNNING ? 0 : 1; | status = ifp->if_drv_flags & IFF_DRV_RUNNING ? 0 : 1; | ||||
if (status) | if (status) | ||||
nm_clear_native_flags(na); | nm_clear_native_flags(na); | ||||
CTX_UNLOCK(ctx); | CTX_UNLOCK(ctx); | ||||
return (status); | return (status); | ||||
} | } | ||||
static void | |||||
iru_init(if_rxd_update_t iru, iflib_rxq_t rxq, uint8_t flid) | |||||
{ | |||||
iflib_fl_t fl; | |||||
fl = &rxq->ifr_fl[flid]; | |||||
iru->iru_paddrs = fl->ifl_bus_addrs; | |||||
iru->iru_vaddrs = &fl->ifl_vm_addrs[0]; | |||||
iru->iru_idxs = fl->ifl_rxd_idxs; | |||||
iru->iru_qsidx = rxq->ifr_id; | |||||
iru->iru_buf_size = fl->ifl_buf_size; | |||||
iru->iru_flidx = fl->ifl_id; | |||||
} | |||||
static int | |||||
netmap_fl_refill(iflib_rxq_t rxq, struct netmap_kring *kring, uint32_t nm_i, bool init) | |||||
{ | |||||
struct netmap_adapter *na = kring->na; | |||||
u_int const lim = kring->nkr_num_slots - 1; | |||||
u_int head = kring->rhead; | |||||
struct netmap_ring *ring = kring->ring; | |||||
bus_dmamap_t *map; | |||||
if_rxd_update_t iru; | |||||
if_ctx_t ctx = rxq->ifr_ctx; | |||||
iflib_fl_t fl = &rxq->ifr_fl[0]; | |||||
uint32_t refill_pidx, nic_i; | |||||
iru = &rxq->ifr_iru; | |||||
iru_init(iru, rxq, 0 /* flid */); | |||||
map = fl->ifl_sds.ifsd_map; | |||||
refill_pidx = netmap_idx_k2n(kring, nm_i); | |||||
if (init && (nm_i == head)) | |||||
head = nm_prev(head, lim); | |||||
for (int tmp_pidx = 0; nm_i != head; tmp_pidx++) { | |||||
struct netmap_slot *slot = &ring->slot[nm_i]; | |||||
void *addr = PNMB(na, slot, &fl->ifl_bus_addrs[tmp_pidx]); | |||||
uint32_t nic_i_dma = refill_pidx; | |||||
nic_i = netmap_idx_k2n(kring, nm_i); | |||||
MPASS(tmp_pidx < IFLIB_MAX_RX_REFRESH); | |||||
if (addr == NETMAP_BUF_BASE(na)) /* bad buf */ | |||||
return netmap_ring_reinit(kring); | |||||
fl->ifl_vm_addrs[tmp_pidx] = addr; | |||||
if (__predict_false(init) && map) { | |||||
netmap_load_map(na, fl->ifl_ifdi->idi_tag, map[nic_i], addr); | |||||
} else if (map && (slot->flags & NS_BUF_CHANGED)) { | |||||
/* buffer has changed, reload map */ | |||||
netmap_reload_map(na, fl->ifl_ifdi->idi_tag, map[nic_i], addr); | |||||
} | |||||
slot->flags &= ~NS_BUF_CHANGED; | |||||
nm_i = nm_next(nm_i, lim); | |||||
fl->ifl_rxd_idxs[tmp_pidx] = nic_i = nm_next(nic_i, lim); | |||||
if (nm_i != head && tmp_pidx < IFLIB_MAX_RX_REFRESH-1) | |||||
continue; | |||||
iru->iru_pidx = refill_pidx; | |||||
iru->iru_count = tmp_pidx+1; | |||||
ctx->isc_rxd_refill(ctx->ifc_softc, iru); | |||||
tmp_pidx = 0; | |||||
refill_pidx = nic_i; | |||||
if (map == NULL) | |||||
continue; | |||||
for (int n = 0; n < iru->iru_count; n++) { | |||||
bus_dmamap_sync(fl->ifl_ifdi->idi_tag, map[nic_i_dma], | |||||
BUS_DMASYNC_PREREAD); | |||||
/* XXX - change this to not use the netmap func*/ | |||||
nic_i_dma = nm_next(nic_i_dma, lim); | |||||
} | |||||
} | |||||
kring->nr_hwcur = head; | |||||
if (map) | |||||
bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, | |||||
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | |||||
/* | /* | ||||
* IMPORTANT: we must leave one free slot in the ring, | |||||
* so move nic_i back by one unit | |||||
*/ | |||||
nic_i = nm_prev(nic_i, lim); | |||||
ctx->isc_rxd_flush(ctx->ifc_softc, rxq->ifr_id, fl->ifl_id, nic_i); | |||||
return (0); | |||||
} | |||||
/* | |||||
* Reconcile kernel and user view of the transmit ring. | * Reconcile kernel and user view of the transmit ring. | ||||
* | * | ||||
* All information is in the kring. | * All information is in the kring. | ||||
* Userspace wants to send packets up to the one before kring->rhead, | * Userspace wants to send packets up to the one before kring->rhead, | ||||
* kernel knows kring->nr_hwcur is the first unsent packet. | * kernel knows kring->nr_hwcur is the first unsent packet. | ||||
* | * | ||||
* Here we push packets out (as many as possible), and possibly | * Here we push packets out (as many as possible), and possibly | ||||
* reclaim buffers from previously completed transmission. | * reclaim buffers from previously completed transmission. | ||||
▲ Show 20 Lines • Show All 140 Lines • ▼ Show 20 Lines | |||||
* of whether or not we received an interrupt. | * of whether or not we received an interrupt. | ||||
*/ | */ | ||||
static int | static int | ||||
iflib_netmap_rxsync(struct netmap_kring *kring, int flags) | iflib_netmap_rxsync(struct netmap_kring *kring, int flags) | ||||
{ | { | ||||
struct netmap_adapter *na = kring->na; | struct netmap_adapter *na = kring->na; | ||||
struct netmap_ring *ring = kring->ring; | struct netmap_ring *ring = kring->ring; | ||||
uint32_t nm_i; /* index into the netmap ring */ | uint32_t nm_i; /* index into the netmap ring */ | ||||
uint32_t nic_i, nic_i_start; /* index into the NIC ring */ | uint32_t nic_i; /* index into the NIC ring */ | ||||
u_int i, n; | u_int i, n; | ||||
u_int const lim = kring->nkr_num_slots - 1; | u_int const lim = kring->nkr_num_slots - 1; | ||||
u_int const head = kring->rhead; | u_int const head = kring->rhead; | ||||
int force_update = (flags & NAF_FORCE_READ) || kring->nr_kflags & NKR_PENDINTR; | int force_update = (flags & NAF_FORCE_READ) || kring->nr_kflags & NKR_PENDINTR; | ||||
struct if_rxd_info ri; | struct if_rxd_info *ri; | ||||
struct if_rxd_update iru; | struct if_rxd_update *iru; | ||||
struct ifnet *ifp = na->ifp; | struct ifnet *ifp = na->ifp; | ||||
if_ctx_t ctx = ifp->if_softc; | if_ctx_t ctx = ifp->if_softc; | ||||
iflib_rxq_t rxq = &ctx->ifc_rxqs[kring->ring_id]; | iflib_rxq_t rxq = &ctx->ifc_rxqs[kring->ring_id]; | ||||
iflib_fl_t fl = rxq->ifr_fl; | iflib_fl_t fl = rxq->ifr_fl; | ||||
ri = &rxq->ifr_ri; | |||||
iru = &rxq->ifr_iru; | |||||
if (head > lim) | if (head > lim) | ||||
return netmap_ring_reinit(kring); | return netmap_ring_reinit(kring); | ||||
/* XXX check sync modes */ | /* XXX check sync modes */ | ||||
for (i = 0, fl = rxq->ifr_fl; i < rxq->ifr_nfl; i++, fl++) { | for (i = 0, fl = rxq->ifr_fl; i < rxq->ifr_nfl; i++, fl++) { | ||||
if (fl->ifl_sds.ifsd_map == NULL) | if (fl->ifl_sds.ifsd_map == NULL) | ||||
continue; | continue; | ||||
bus_dmamap_sync(rxq->ifr_fl[i].ifl_desc_tag, fl->ifl_ifdi->idi_map, | bus_dmamap_sync(rxq->ifr_fl[i].ifl_desc_tag, fl->ifl_ifdi->idi_map, | ||||
Show All 19 Lines | if (netmap_no_pendintr || force_update) { | ||||
int error, avail; | int error, avail; | ||||
uint16_t slot_flags = kring->nkr_slot_flags; | uint16_t slot_flags = kring->nkr_slot_flags; | ||||
for (fl = rxq->ifr_fl, i = 0; i < rxq->ifr_nfl; i++, fl++) { | for (fl = rxq->ifr_fl, i = 0; i < rxq->ifr_nfl; i++, fl++) { | ||||
nic_i = fl->ifl_cidx; | nic_i = fl->ifl_cidx; | ||||
nm_i = netmap_idx_n2k(kring, nic_i); | nm_i = netmap_idx_n2k(kring, nic_i); | ||||
avail = iflib_rxd_avail(ctx, rxq, nic_i, USHRT_MAX); | avail = iflib_rxd_avail(ctx, rxq, nic_i, USHRT_MAX); | ||||
for (n = 0; avail > 0; n++, avail--) { | for (n = 0; avail > 0; n++, avail--) { | ||||
rxd_info_zero(&ri); | rxd_info_zero(ri); | ||||
ri.iri_frags = rxq->ifr_frags; | ri->iri_frags = rxq->ifr_frags; | ||||
ri.iri_qsidx = kring->ring_id; | ri->iri_qsidx = kring->ring_id; | ||||
ri.iri_ifp = ctx->ifc_ifp; | ri->iri_ifp = ctx->ifc_ifp; | ||||
ri.iri_cidx = nic_i; | ri->iri_cidx = nic_i; | ||||
error = ctx->isc_rxd_pkt_get(ctx->ifc_softc, &ri); | error = ctx->isc_rxd_pkt_get(ctx->ifc_softc, ri); | ||||
ring->slot[nm_i].len = error ? 0 : ri.iri_len - crclen; | ring->slot[nm_i].len = error ? 0 : ri->iri_len - crclen; | ||||
ring->slot[nm_i].flags = slot_flags; | ring->slot[nm_i].flags = slot_flags; | ||||
if (fl->ifl_sds.ifsd_map) | if (fl->ifl_sds.ifsd_map) | ||||
bus_dmamap_sync(fl->ifl_ifdi->idi_tag, | bus_dmamap_sync(fl->ifl_ifdi->idi_tag, | ||||
fl->ifl_sds.ifsd_map[nic_i], BUS_DMASYNC_POSTREAD); | fl->ifl_sds.ifsd_map[nic_i], BUS_DMASYNC_POSTREAD); | ||||
nm_i = nm_next(nm_i, lim); | nm_i = nm_next(nm_i, lim); | ||||
nic_i = nm_next(nic_i, lim); | nic_i = nm_next(nic_i, lim); | ||||
} | } | ||||
if (n) { /* update the state variables */ | if (n) { /* update the state variables */ | ||||
Show All 16 Lines | iflib_netmap_rxsync(struct netmap_kring *kring, int flags) | ||||
* nic_i is the index in the NIC ring, and | * nic_i is the index in the NIC ring, and | ||||
* nm_i == (nic_i + kring->nkr_hwofs) % ring_size | * nm_i == (nic_i + kring->nkr_hwofs) % ring_size | ||||
*/ | */ | ||||
/* XXX not sure how this will work with multiple free lists */ | /* XXX not sure how this will work with multiple free lists */ | ||||
nm_i = kring->nr_hwcur; | nm_i = kring->nr_hwcur; | ||||
if (nm_i == head) | if (nm_i == head) | ||||
return (0); | return (0); | ||||
iru.iru_paddrs = fl->ifl_bus_addrs; | return (netmap_fl_refill(rxq, kring, nm_i, false)); | ||||
iru.iru_vaddrs = &fl->ifl_vm_addrs[0]; | |||||
iru.iru_idxs = fl->ifl_rxd_idxs; | |||||
iru.iru_qsidx = rxq->ifr_id; | |||||
iru.iru_buf_size = fl->ifl_buf_size; | |||||
iru.iru_flidx = fl->ifl_id; | |||||
nic_i_start = nic_i = netmap_idx_k2n(kring, nm_i); | |||||
for (i = 0; nm_i != head; i++) { | |||||
struct netmap_slot *slot = &ring->slot[nm_i]; | |||||
void *addr = PNMB(na, slot, &fl->ifl_bus_addrs[i]); | |||||
if (addr == NETMAP_BUF_BASE(na)) /* bad buf */ | |||||
goto ring_reset; | |||||
fl->ifl_vm_addrs[i] = addr; | |||||
if (fl->ifl_sds.ifsd_map && (slot->flags & NS_BUF_CHANGED)) { | |||||
/* buffer has changed, reload map */ | |||||
netmap_reload_map(na, fl->ifl_ifdi->idi_tag, fl->ifl_sds.ifsd_map[nic_i], addr); | |||||
} | } | ||||
slot->flags &= ~NS_BUF_CHANGED; | |||||
nm_i = nm_next(nm_i, lim); | |||||
fl->ifl_rxd_idxs[i] = nic_i = nm_next(nic_i, lim); | |||||
if (nm_i != head && i < IFLIB_MAX_RX_REFRESH) | |||||
continue; | |||||
iru.iru_pidx = nic_i_start; | |||||
iru.iru_count = i; | |||||
i = 0; | |||||
ctx->isc_rxd_refill(ctx->ifc_softc, &iru); | |||||
if (fl->ifl_sds.ifsd_map == NULL) { | |||||
nic_i_start = nic_i; | |||||
continue; | |||||
} | |||||
nic_i = nic_i_start; | |||||
for (n = 0; n < iru.iru_count; n++) { | |||||
bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_sds.ifsd_map[nic_i], | |||||
BUS_DMASYNC_PREREAD); | |||||
nic_i = nm_next(nic_i, lim); | |||||
} | |||||
nic_i_start = nic_i; | |||||
} | |||||
kring->nr_hwcur = head; | |||||
if (fl->ifl_sds.ifsd_map) | |||||
bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, | |||||
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | |||||
/* | |||||
* IMPORTANT: we must leave one free slot in the ring, | |||||
* so move nic_i back by one unit | |||||
*/ | |||||
nic_i = nm_prev(nic_i, lim); | |||||
ctx->isc_rxd_flush(ctx->ifc_softc, rxq->ifr_id, fl->ifl_id, nic_i); | |||||
return 0; | |||||
ring_reset: | |||||
return netmap_ring_reinit(kring); | |||||
} | |||||
static void | static void | ||||
iflib_netmap_intr(struct netmap_adapter *na, int onoff) | iflib_netmap_intr(struct netmap_adapter *na, int onoff) | ||||
{ | { | ||||
struct ifnet *ifp = na->ifp; | struct ifnet *ifp = na->ifp; | ||||
if_ctx_t ctx = ifp->if_softc; | if_ctx_t ctx = ifp->if_softc; | ||||
CTX_LOCK(ctx); | /* XXX - do we need synchronization here?*/ | ||||
if (onoff) { | if (onoff) { | ||||
IFDI_INTR_ENABLE(ctx); | IFDI_INTR_ENABLE(ctx); | ||||
} else { | } else { | ||||
IFDI_INTR_DISABLE(ctx); | IFDI_INTR_DISABLE(ctx); | ||||
} | } | ||||
CTX_UNLOCK(ctx); | |||||
} | } | ||||
static int | static int | ||||
iflib_netmap_attach(if_ctx_t ctx) | iflib_netmap_attach(if_ctx_t ctx) | ||||
{ | { | ||||
struct netmap_adapter na; | struct netmap_adapter na; | ||||
if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; | if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; | ||||
Show All 40 Lines | for (int i = 0; i < ctx->ifc_softc_ctx.isc_ntxd[0]; i++) { | ||||
int si = netmap_idx_n2k(&na->tx_rings[txq->ift_id], i); | int si = netmap_idx_n2k(&na->tx_rings[txq->ift_id], i); | ||||
netmap_load_map(na, txq->ift_desc_tag, txq->ift_sds.ifsd_map[i], NMB(na, slot + si)); | netmap_load_map(na, txq->ift_desc_tag, txq->ift_sds.ifsd_map[i], NMB(na, slot + si)); | ||||
} | } | ||||
} | } | ||||
static void | static void | ||||
iflib_netmap_rxq_init(if_ctx_t ctx, iflib_rxq_t rxq) | iflib_netmap_rxq_init(if_ctx_t ctx, iflib_rxq_t rxq) | ||||
{ | { | ||||
struct netmap_adapter *na = NA(ctx->ifc_ifp); | struct netmap_adapter *na = NA(ctx->ifc_ifp); | ||||
struct netmap_kring *kring = &na->rx_rings[rxq->ifr_id]; | |||||
struct netmap_slot *slot; | struct netmap_slot *slot; | ||||
struct if_rxd_update iru; | uint32_t nm_i; | ||||
iflib_fl_t fl; | |||||
bus_dmamap_t *map; | |||||
int nrxd; | |||||
uint32_t i, j, pidx_start; | |||||
slot = netmap_reset(na, NR_RX, rxq->ifr_id, 0); | slot = netmap_reset(na, NR_RX, rxq->ifr_id, 0); | ||||
if (slot == NULL) | if (slot == NULL) | ||||
return; | return; | ||||
fl = &rxq->ifr_fl[0]; | nm_i = netmap_idx_n2k(kring, 0); | ||||
map = fl->ifl_sds.ifsd_map; | netmap_fl_refill(rxq, kring, nm_i, true); | ||||
nrxd = ctx->ifc_softc_ctx.isc_nrxd[0]; | |||||
iru.iru_paddrs = fl->ifl_bus_addrs; | |||||
iru.iru_vaddrs = &fl->ifl_vm_addrs[0]; | |||||
iru.iru_idxs = fl->ifl_rxd_idxs; | |||||
iru.iru_qsidx = rxq->ifr_id; | |||||
iru.iru_buf_size = rxq->ifr_fl[0].ifl_buf_size; | |||||
iru.iru_flidx = 0; | |||||
for (pidx_start = i = j = 0; i < nrxd; i++, j++) { | |||||
int sj = netmap_idx_n2k(&na->rx_rings[rxq->ifr_id], i); | |||||
void *addr; | |||||
fl->ifl_rxd_idxs[j] = i; | |||||
addr = fl->ifl_vm_addrs[j] = PNMB(na, slot + sj, &fl->ifl_bus_addrs[j]); | |||||
if (map) { | |||||
netmap_load_map(na, rxq->ifr_fl[0].ifl_ifdi->idi_tag, *map, addr); | |||||
map++; | |||||
} | } | ||||
if (j < IFLIB_MAX_RX_REFRESH && i < nrxd - 1) | |||||
continue; | |||||
iru.iru_pidx = pidx_start; | |||||
pidx_start = i; | |||||
iru.iru_count = j; | |||||
j = 0; | |||||
MPASS(pidx_start + j <= nrxd); | |||||
/* Update descriptors and the cached value */ | |||||
ctx->isc_rxd_refill(ctx->ifc_softc, &iru); | |||||
} | |||||
/* preserve queue */ | |||||
if (ctx->ifc_ifp->if_capenable & IFCAP_NETMAP) { | |||||
struct netmap_kring *kring = &na->rx_rings[rxq->ifr_id]; | |||||
int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring); | |||||
ctx->isc_rxd_flush(ctx->ifc_softc, rxq->ifr_id, 0 /* fl_id */, t); | |||||
} else | |||||
ctx->isc_rxd_flush(ctx->ifc_softc, rxq->ifr_id, 0 /* fl_id */, nrxd-1); | |||||
} | |||||
#define iflib_netmap_detach(ifp) netmap_detach(ifp) | #define iflib_netmap_detach(ifp) netmap_detach(ifp) | ||||
#else | #else | ||||
#define iflib_netmap_txq_init(ctx, txq) | #define iflib_netmap_txq_init(ctx, txq) | ||||
#define iflib_netmap_rxq_init(ctx, rxq) | #define iflib_netmap_rxq_init(ctx, rxq) | ||||
#define iflib_netmap_detach(ifp) | #define iflib_netmap_detach(ifp) | ||||
#define iflib_netmap_attach(ctx) (0) | #define iflib_netmap_attach(ctx) (0) | ||||
#define netmap_rx_irq(ifp, qid, budget) (0) | #define netmap_rx_irq(ifp, qid, budget) (0) | ||||
#define netmap_tx_irq(ifp, qid) do {} while (0) | #define netmap_tx_irq(ifp, qid) do {} while (0) | ||||
#endif | #endif | ||||
#if defined(__i386__) || defined(__amd64__) | #if defined(__i386__) || defined(__amd64__) | ||||
static __inline void | static __inline void | ||||
prefetch(void *x) | prefetch(void *x) | ||||
{ | { | ||||
__asm volatile("prefetcht0 %0" :: "m" (*(unsigned long *)x)); | __asm volatile("prefetcht0 %0" :: "m" (*(unsigned long *)x)); | ||||
} | } | ||||
static __inline void | |||||
prefetch2(void *x) | |||||
{ | |||||
__asm volatile("prefetcht0 %0" :: "m" (*(unsigned long *)x)); | |||||
#if (CACHE_LINE_SIZE < 128) | |||||
__asm volatile("prefetcht0 %0" :: "m" (*(((unsigned long *)x)+CACHE_LINE_SIZE/(sizeof(unsigned long))))); | |||||
#endif | |||||
} | |||||
#else | #else | ||||
#define prefetch(x) | #define prefetch(x) | ||||
#define prefetch2(x) | |||||
#endif | #endif | ||||
static void | static void | ||||
_iflib_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int err) | _iflib_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int err) | ||||
{ | { | ||||
if (err) | if (err) | ||||
return; | return; | ||||
*(bus_addr_t *) arg = segs[0].ds_addr; | *(bus_addr_t *) arg = segs[0].ds_addr; | ||||
▲ Show 20 Lines • Show All 99 Lines • ▼ Show 20 Lines | |||||
{ | { | ||||
int i; | int i; | ||||
iflib_dma_info_t *dmaiter = dmalist; | iflib_dma_info_t *dmaiter = dmalist; | ||||
for (i = 0; i < count; i++, dmaiter++) | for (i = 0; i < count; i++, dmaiter++) | ||||
iflib_dma_free(*dmaiter); | iflib_dma_free(*dmaiter); | ||||
} | } | ||||
static void | |||||
txq_validate(iflib_txq_t txq) { | |||||
#ifdef INVARIANTS | |||||
uint32_t cidx = txq->ift_cidx; | |||||
struct mbuf **ifsd_m = txq->ift_sds.ifsd_m; | |||||
if (txq->ift_pidx > cidx) { | |||||
int i; | |||||
for (i = txq->ift_pidx; i < txq->ift_size; i++) | |||||
MPASS(ifsd_m[i] == NULL); | |||||
for (i = 0; i < cidx; i++) | |||||
MPASS(ifsd_m[i] == NULL); | |||||
} else if (txq->ift_pidx < cidx) { | |||||
int i; | |||||
for (i = txq->ift_pidx; i < cidx; i++) | |||||
MPASS(ifsd_m[i] == NULL); | |||||
} | |||||
#endif | |||||
} | |||||
#ifdef EARLY_AP_STARTUP | #ifdef EARLY_AP_STARTUP | ||||
static const int iflib_started = 1; | static const int iflib_started = 1; | ||||
#else | #else | ||||
/* | /* | ||||
* We used to abuse the smp_started flag to decide if the queues have been | * We used to abuse the smp_started flag to decide if the queues have been | ||||
* fully initialized (by late taskqgroup_adjust() calls in a SYSINIT()). | * fully initialized (by late taskqgroup_adjust() calls in a SYSINIT()). | ||||
* That gave bad races, since the SYSINIT() runs strictly after smp_started | * That gave bad races, since the SYSINIT() runs strictly after smp_started | ||||
* is set. Run a SYSINIT() strictly after that to just set a usable | * is set. Run a SYSINIT() strictly after that to just set a usable | ||||
Show All 12 Lines | SYSINIT(iflib_record_started, SI_SUB_SMP + 1, SI_ORDER_FIRST, | ||||
iflib_record_started, NULL); | iflib_record_started, NULL); | ||||
#endif | #endif | ||||
static int | static int | ||||
iflib_fast_intr(void *arg) | iflib_fast_intr(void *arg) | ||||
{ | { | ||||
iflib_filter_info_t info = arg; | iflib_filter_info_t info = arg; | ||||
struct grouptask *gtask = info->ifi_task; | struct grouptask *gtask = info->ifi_task; | ||||
if (!iflib_started) | if (!iflib_started) | ||||
return (FILTER_HANDLED); | return (FILTER_HANDLED); | ||||
DBG_COUNTER_INC(fast_intrs); | DBG_COUNTER_INC(fast_intrs); | ||||
if (info->ifi_filter != NULL && info->ifi_filter(info->ifi_filter_arg) == FILTER_HANDLED) | if (info->ifi_filter != NULL && info->ifi_filter(info->ifi_filter_arg) == FILTER_HANDLED) | ||||
return (FILTER_HANDLED); | return (FILTER_HANDLED); | ||||
GROUPTASK_ENQUEUE(gtask); | GROUPTASK_ENQUEUE(gtask); | ||||
return (FILTER_HANDLED); | return (FILTER_HANDLED); | ||||
} | } | ||||
static int | static int | ||||
iflib_fast_intr_rx(void *arg) | |||||
{ | |||||
iflib_filter_info_t info = arg; | |||||
struct grouptask *gtask = info->ifi_task; | |||||
iflib_rxq_t rxq = (iflib_rxq_t)info->ifi_ctx; | |||||
if_ctx_t ctx; | |||||
int cidx; | |||||
if (!iflib_started) | |||||
return (FILTER_HANDLED); | |||||
DBG_COUNTER_INC(fast_intrs); | |||||
if (info->ifi_filter != NULL && info->ifi_filter(info->ifi_filter_arg) == FILTER_HANDLED) | |||||
return (FILTER_HANDLED); | |||||
ctx = rxq->ifr_ctx; | |||||
if (ctx->ifc_sctx->isc_flags & IFLIB_HAS_RXCQ) | |||||
cidx = rxq->ifr_cq_cidx; | |||||
else | |||||
cidx = rxq->ifr_fl[0].ifl_cidx; | |||||
if (iflib_rxd_avail(ctx, rxq, cidx, 1)) | |||||
GROUPTASK_ENQUEUE(gtask); | |||||
else | |||||
IFDI_RX_QUEUE_INTR_ENABLE(ctx, rxq->ifr_id); | |||||
return (FILTER_HANDLED); | |||||
} | |||||
static int | |||||
iflib_fast_intr_rxtx(void *arg) | iflib_fast_intr_rxtx(void *arg) | ||||
{ | { | ||||
iflib_filter_info_t info = arg; | iflib_filter_info_t info = arg; | ||||
struct grouptask *gtask = info->ifi_task; | struct grouptask *gtask = info->ifi_task; | ||||
iflib_rxq_t rxq = (iflib_rxq_t)info->ifi_ctx; | iflib_rxq_t rxq = (iflib_rxq_t)info->ifi_ctx; | ||||
if_ctx_t ctx; | if_ctx_t ctx; | ||||
int i, cidx; | int i, cidx; | ||||
if (!iflib_started) | if (!iflib_started) | ||||
return (FILTER_HANDLED); | return (FILTER_HANDLED); | ||||
DBG_COUNTER_INC(fast_intrs); | DBG_COUNTER_INC(fast_intrs); | ||||
if (info->ifi_filter != NULL && info->ifi_filter(info->ifi_filter_arg) == FILTER_HANDLED) | if (info->ifi_filter != NULL && info->ifi_filter(info->ifi_filter_arg) == FILTER_HANDLED) | ||||
return (FILTER_HANDLED); | return (FILTER_HANDLED); | ||||
ctx = rxq->ifr_ctx; | |||||
for (i = 0; i < rxq->ifr_ntxqirq; i++) { | for (i = 0; i < rxq->ifr_ntxqirq; i++) { | ||||
qidx_t txqid = rxq->ifr_txqid[i]; | qidx_t txqid = rxq->ifr_txqid[i]; | ||||
ctx = rxq->ifr_ctx; | |||||
if (!ctx->isc_txd_credits_update(ctx->ifc_softc, txqid, false)) { | if (!ctx->isc_txd_credits_update(ctx->ifc_softc, txqid, false)) { | ||||
IFDI_TX_QUEUE_INTR_ENABLE(ctx, txqid); | IFDI_TX_QUEUE_INTR_ENABLE(ctx, txqid); | ||||
continue; | continue; | ||||
} | } | ||||
GROUPTASK_ENQUEUE(&ctx->ifc_txqs[txqid].ift_task); | GROUPTASK_ENQUEUE(&ctx->ifc_txqs[txqid].ift_task); | ||||
} | } | ||||
if (ctx->ifc_sctx->isc_flags & IFLIB_HAS_RXCQ) | if (ctx->ifc_sctx->isc_flags & IFLIB_HAS_RXCQ) | ||||
cidx = rxq->ifr_cq_cidx; | cidx = rxq->ifr_cq_cidx; | ||||
▲ Show 20 Lines • Show All 555 Lines • ▼ Show 20 Lines | if (*sd_flags & RX_SW_DESC_INUSE) { | ||||
} | } | ||||
if (*sd_m != NULL) { | if (*sd_m != NULL) { | ||||
m_init(*sd_m, M_NOWAIT, MT_DATA, 0); | m_init(*sd_m, M_NOWAIT, MT_DATA, 0); | ||||
uma_zfree(zone_mbuf, *sd_m); | uma_zfree(zone_mbuf, *sd_m); | ||||
} | } | ||||
if (*sd_cl != NULL) | if (*sd_cl != NULL) | ||||
uma_zfree(fl->ifl_zone, *sd_cl); | uma_zfree(fl->ifl_zone, *sd_cl); | ||||
*sd_flags = 0; | *sd_flags = 0; | ||||
} else if (*sd_flags & RX_NETMAP_INUSE) { | |||||
if (fl->ifl_sds.ifsd_map != NULL) { | |||||
bus_dmamap_t sd_map = fl->ifl_sds.ifsd_map[i]; | |||||
bus_dmamap_unload(fl->ifl_desc_tag, sd_map); | |||||
bus_dmamap_destroy(fl->ifl_desc_tag, sd_map); | |||||
} | |||||
*sd_flags = 0; | |||||
MPASS(*sd_cl == NULL); | |||||
MPASS(*sd_m == NULL); | |||||
} else { | } else { | ||||
MPASS(*sd_cl == NULL); | MPASS(*sd_cl == NULL); | ||||
MPASS(*sd_m == NULL); | MPASS(*sd_m == NULL); | ||||
} | } | ||||
#if MEMORY_LOGGING | #if MEMORY_LOGGING | ||||
if (*sd_m != NULL) | |||||
fl->ifl_m_dequeued++; | fl->ifl_m_dequeued++; | ||||
if (*sd_cl != NULL) | |||||
fl->ifl_cl_dequeued++; | fl->ifl_cl_dequeued++; | ||||
#endif | #endif | ||||
*sd_cl = NULL; | *sd_cl = NULL; | ||||
*sd_m = NULL; | *sd_m = NULL; | ||||
} | } | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
for (i = 0; i < fl->ifl_size; i++) { | for (i = 0; i < fl->ifl_size; i++) { | ||||
MPASS(fl->ifl_sds.ifsd_flags[i] == 0); | KASSERT(fl->ifl_sds.ifsd_flags[i] == 0, ("fl->ifl_sds.ifsd_flags[%d]=0x%x, expected 0", | ||||
i, fl->ifl_sds.ifsd_flags[i])); | |||||
MPASS(fl->ifl_sds.ifsd_cl[i] == NULL); | MPASS(fl->ifl_sds.ifsd_cl[i] == NULL); | ||||
MPASS(fl->ifl_sds.ifsd_m[i] == NULL); | MPASS(fl->ifl_sds.ifsd_m[i] == NULL); | ||||
} | } | ||||
#endif | #endif | ||||
/* | /* | ||||
* Reset free list values | * Reset free list values | ||||
*/ | */ | ||||
fl->ifl_credits = fl->ifl_cidx = fl->ifl_pidx = fl->ifl_gen = fl->ifl_fragidx = 0; | fl->ifl_credits = fl->ifl_cidx = fl->ifl_pidx = fl->ifl_gen = fl->ifl_fragidx = 0; | ||||
bzero(idi->idi_vaddr, idi->idi_size); | bzero(idi->idi_vaddr, idi->idi_size); | ||||
} | } | ||||
/********************************************************************* | /********************************************************************* | ||||
* | * | ||||
* Initialize a receive ring and its buffers. | * Initialize a receive ring and its buffers. | ||||
* | * | ||||
**********************************************************************/ | **********************************************************************/ | ||||
static int | static int | ||||
iflib_fl_setup(iflib_fl_t fl) | iflib_fl_setup(iflib_fl_t fl) | ||||
{ | { | ||||
iflib_rxq_t rxq = fl->ifl_rxq; | iflib_rxq_t rxq = fl->ifl_rxq; | ||||
if_ctx_t ctx = rxq->ifr_ctx; | if_ctx_t ctx = rxq->ifr_ctx; | ||||
if_softc_ctx_t sctx = &ctx->ifc_softc_ctx; | if_softc_ctx_t sctx = &ctx->ifc_softc_ctx; | ||||
bit_nclear(fl->ifl_rx_bitmap, 0, fl->ifl_size); | bit_nclear(fl->ifl_rx_bitmap, 0, fl->ifl_size-1); | ||||
/* | /* | ||||
** Free current RX buffer structs and their mbufs | ** Free current RX buffer structs and their mbufs | ||||
*/ | */ | ||||
iflib_fl_bufs_free(fl); | iflib_fl_bufs_free(fl); | ||||
/* Now replenish the mbufs */ | /* Now replenish the mbufs */ | ||||
MPASS(fl->ifl_credits == 0); | MPASS(fl->ifl_credits == 0); | ||||
/* | /* | ||||
* XXX don't set the max_frame_size to larger | * XXX don't set the max_frame_size to larger | ||||
▲ Show 20 Lines • Show All 62 Lines • ▼ Show 20 Lines | for (i = 0; i < rxq->ifr_nfl; i++) { | ||||
fl->ifl_sds.ifsd_map = NULL; | fl->ifl_sds.ifsd_map = NULL; | ||||
} | } | ||||
free(rxq->ifr_fl, M_IFLIB); | free(rxq->ifr_fl, M_IFLIB); | ||||
rxq->ifr_fl = NULL; | rxq->ifr_fl = NULL; | ||||
rxq->ifr_cq_gen = rxq->ifr_cq_cidx = rxq->ifr_cq_pidx = 0; | rxq->ifr_cq_gen = rxq->ifr_cq_cidx = rxq->ifr_cq_pidx = 0; | ||||
} | } | ||||
} | } | ||||
/* CONFIG context only */ | |||||
static void | |||||
iflib_handle_hang(if_ctx_t ctx, void *arg __unused) | |||||
{ | |||||
CTX_LOCK(ctx); | |||||
if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); | |||||
IFDI_WATCHDOG_RESET(ctx); | |||||
ctx->ifc_watchdog_events++; | |||||
iflib_if_init_locked(ctx); | |||||
CTX_UNLOCK(ctx); | |||||
} | |||||
/* | /* | ||||
* MI independent logic | * MI independent logic | ||||
* | * | ||||
*/ | */ | ||||
static void | static void | ||||
iflib_timer(void *arg) | iflib_timer(void *arg) | ||||
{ | { | ||||
iflib_txq_t txq = arg; | iflib_txq_t txq_i, txq = arg; | ||||
if_ctx_t ctx = txq->ift_ctx; | if_ctx_t ctx = txq->ift_ctx; | ||||
if_softc_ctx_t sctx = &ctx->ifc_softc_ctx; | |||||
if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)) | if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)) | ||||
return; | return; | ||||
/* | |||||
** Check on the state of the TX queue(s), this | |||||
** can be done without the lock because its RO | |||||
** and the HUNG state will be static if set. | |||||
*/ | |||||
IFDI_TIMER(ctx, txq->ift_id); | |||||
if ((txq->ift_qstatus == IFLIB_QUEUE_HUNG) && | |||||
((txq->ift_cleaned_prev == txq->ift_cleaned) || | |||||
(sctx->isc_pause_frames == 0))) | |||||
goto hung; | |||||
if (ifmp_ring_is_stalled(txq->ift_br)) | |||||
txq->ift_qstatus = IFLIB_QUEUE_HUNG; | |||||
txq->ift_cleaned_prev = txq->ift_cleaned; | |||||
/* handle any laggards */ | /* handle any laggards */ | ||||
if (txq->ift_db_pending) | if (txq->ift_db_pending) | ||||
GROUPTASK_ENQUEUE(&txq->ift_task); | GROUPTASK_ENQUEUE(&txq->ift_task); | ||||
IFDI_TIMER(ctx, txq->ift_id); | |||||
sctx->isc_pause_frames = 0; | if (ifmp_ring_is_stalled(txq->ift_br) && | ||||
if (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING) | txq->ift_cleaned_prev == txq->ift_cleaned) | ||||
callout_reset_on(&txq->ift_timer, hz/2, iflib_timer, txq, txq->ift_timer.c_cpu); | txq->ift_stall_count++; | ||||
return; | txq->ift_cleaned_prev = txq->ift_cleaned; | ||||
hung: | if (txq->ift_stall_count > 2) { | ||||
CTX_LOCK(ctx); | txq->ift_qstatus = IFLIB_QUEUE_HUNG; | ||||
if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); | |||||
device_printf(ctx->ifc_dev, "TX(%d) desc avail = %d, pidx = %d\n", | device_printf(ctx->ifc_dev, "TX(%d) desc avail = %d, pidx = %d\n", | ||||
txq->ift_id, TXQ_AVAIL(txq), txq->ift_pidx); | txq->ift_id, TXQ_AVAIL(txq), txq->ift_pidx); | ||||
} | |||||
if (txq->ift_id != 0) { | |||||
if (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING) | |||||
callout_reset_on(&txq->ift_timer, iflib_timer_int, iflib_timer, | |||||
txq, txq->ift_timer.c_cpu); | |||||
return; | |||||
} | |||||
/* | |||||
** Check on the state of the TX queue(s), this | |||||
** can be done without the lock because its RO | |||||
** and the HUNG state will be static if set. | |||||
*/ | |||||
txq_i = ctx->ifc_txqs; | |||||
for (int i = 0; i < ctx->ifc_softc_ctx.isc_ntxqsets; i++, txq_i++) { | |||||
if (txq_i->ift_qstatus == IFLIB_QUEUE_HUNG) { | |||||
iflib_config_async_gtask_dispatch(ctx, iflib_handle_hang, "hang handler", txq); | |||||
/* init will reset the callout */ | |||||
return; | |||||
} | |||||
} | |||||
IFDI_WATCHDOG_RESET(ctx); | |||||
ctx->ifc_watchdog_events++; | |||||
ctx->ifc_flags |= IFC_DO_RESET; | if (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING) | ||||
iflib_admin_intr_deferred(ctx); | callout_reset_on(&txq->ift_timer, iflib_timer_int, iflib_timer, | ||||
CTX_UNLOCK(ctx); | txq, txq->ift_timer.c_cpu); | ||||
} | } | ||||
static void | static void | ||||
iflib_init_locked(if_ctx_t ctx) | iflib_init_locked(if_ctx_t ctx) | ||||
{ | { | ||||
if_softc_ctx_t sctx = &ctx->ifc_softc_ctx; | if_softc_ctx_t sctx = &ctx->ifc_softc_ctx; | ||||
if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; | if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; | ||||
if_t ifp = ctx->ifc_ifp; | if_t ifp = ctx->ifc_ifp; | ||||
iflib_fl_t fl; | iflib_fl_t fl; | ||||
iflib_txq_t txq; | iflib_txq_t txq; | ||||
iflib_rxq_t rxq; | iflib_rxq_t rxq; | ||||
int i, j, tx_ip_csum_flags, tx_ip6_csum_flags; | int i, j, tx_ip_csum_flags, tx_ip6_csum_flags, running, reset; | ||||
running = !!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING); | |||||
reset = !!(ctx->ifc_flags & IFC_DO_RESET); | |||||
if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); | if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); | ||||
IFDI_INTR_DISABLE(ctx); | IFDI_INTR_DISABLE(ctx); | ||||
tx_ip_csum_flags = scctx->isc_tx_csum_flags & (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_SCTP); | tx_ip_csum_flags = scctx->isc_tx_csum_flags & (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_SCTP); | ||||
tx_ip6_csum_flags = scctx->isc_tx_csum_flags & (CSUM_IP6_TCP | CSUM_IP6_UDP | CSUM_IP6_SCTP); | tx_ip6_csum_flags = scctx->isc_tx_csum_flags & (CSUM_IP6_TCP | CSUM_IP6_UDP | CSUM_IP6_SCTP); | ||||
/* Set hardware offload abilities */ | /* Set hardware offload abilities */ | ||||
if_clearhwassist(ifp); | if_clearhwassist(ifp); | ||||
if (if_getcapenable(ifp) & IFCAP_TXCSUM) | if (if_getcapenable(ifp) & IFCAP_TXCSUM) | ||||
if_sethwassistbits(ifp, tx_ip_csum_flags, 0); | if_sethwassistbits(ifp, tx_ip_csum_flags, 0); | ||||
if (if_getcapenable(ifp) & IFCAP_TXCSUM_IPV6) | if (if_getcapenable(ifp) & IFCAP_TXCSUM_IPV6) | ||||
if_sethwassistbits(ifp, tx_ip6_csum_flags, 0); | if_sethwassistbits(ifp, tx_ip6_csum_flags, 0); | ||||
if (if_getcapenable(ifp) & IFCAP_TSO4) | if (if_getcapenable(ifp) & IFCAP_TSO4) | ||||
if_sethwassistbits(ifp, CSUM_IP_TSO, 0); | if_sethwassistbits(ifp, CSUM_IP_TSO, 0); | ||||
if (if_getcapenable(ifp) & IFCAP_TSO6) | if (if_getcapenable(ifp) & IFCAP_TSO6) | ||||
if_sethwassistbits(ifp, CSUM_IP6_TSO, 0); | if_sethwassistbits(ifp, CSUM_IP6_TSO, 0); | ||||
for (i = 0, txq = ctx->ifc_txqs; i < sctx->isc_ntxqsets; i++, txq++) { | for (i = 0, txq = ctx->ifc_txqs; i < sctx->isc_ntxqsets; i++, txq++) { | ||||
CALLOUT_LOCK(txq); | CALLOUT_LOCK(txq); | ||||
callout_stop(&txq->ift_timer); | callout_stop(&txq->ift_timer); | ||||
CALLOUT_UNLOCK(txq); | CALLOUT_UNLOCK(txq); | ||||
iflib_netmap_txq_init(ctx, txq); | iflib_netmap_txq_init(ctx, txq); | ||||
} | } | ||||
for (i = 0, rxq = ctx->ifc_rxqs; i < sctx->isc_nrxqsets; i++, rxq++) { | |||||
MPASS(rxq->ifr_id == i); | |||||
iflib_netmap_rxq_init(ctx, rxq); | |||||
} | |||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
i = if_getdrvflags(ifp); | i = if_getdrvflags(ifp); | ||||
#endif | #endif | ||||
IFDI_INIT(ctx); | IFDI_INIT(ctx); | ||||
MPASS(if_getdrvflags(ifp) == i); | MPASS(if_getdrvflags(ifp) == i); | ||||
if (!running && reset) | |||||
return; | |||||
for (i = 0, rxq = ctx->ifc_rxqs; i < sctx->isc_nrxqsets; i++, rxq++) { | for (i = 0, rxq = ctx->ifc_rxqs; i < sctx->isc_nrxqsets; i++, rxq++) { | ||||
/* XXX this should really be done on a per-queue basis */ | /* XXX this should really be done on a per-queue basis */ | ||||
if (if_getcapenable(ifp) & IFCAP_NETMAP) | if (if_getcapenable(ifp) & IFCAP_NETMAP) { | ||||
MPASS(rxq->ifr_id == i); | |||||
iflib_netmap_rxq_init(ctx, rxq); | |||||
continue; | continue; | ||||
} | |||||
for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) { | for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) { | ||||
if (iflib_fl_setup(fl)) { | if (iflib_fl_setup(fl)) { | ||||
device_printf(ctx->ifc_dev, "freelist setup failed - check cluster settings\n"); | device_printf(ctx->ifc_dev, "freelist setup failed - check cluster settings\n"); | ||||
goto done; | goto done; | ||||
} | } | ||||
} | } | ||||
} | } | ||||
done: | done: | ||||
if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); | if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); | ||||
IFDI_INTR_ENABLE(ctx); | IFDI_INTR_ENABLE(ctx); | ||||
txq = ctx->ifc_txqs; | txq = ctx->ifc_txqs; | ||||
for (i = 0; i < sctx->isc_ntxqsets; i++, txq++) | for (i = 0; i < sctx->isc_ntxqsets; i++, txq++) | ||||
callout_reset_on(&txq->ift_timer, hz/2, iflib_timer, txq, | callout_reset_on(&txq->ift_timer, iflib_timer_int, iflib_timer, | ||||
txq->ift_timer.c_cpu); | txq, txq->ift_timer.c_cpu); | ||||
} | } | ||||
/* CONFIG context only */ | |||||
static int | static int | ||||
iflib_media_change(if_t ifp) | iflib_media_change(if_t ifp) | ||||
{ | { | ||||
if_ctx_t ctx = if_getsoftc(ifp); | if_ctx_t ctx = if_getsoftc(ifp); | ||||
int err; | int err; | ||||
CTX_LOCK(ctx); | CTX_LOCK(ctx); | ||||
if ((err = IFDI_MEDIA_CHANGE(ctx)) == 0) | if ((err = IFDI_MEDIA_CHANGE(ctx)) == 0) | ||||
iflib_init_locked(ctx); | iflib_init_locked(ctx); | ||||
CTX_UNLOCK(ctx); | CTX_UNLOCK(ctx); | ||||
return (err); | return (err); | ||||
} | } | ||||
/* CONFIG context only */ | |||||
static void | static void | ||||
iflib_media_status(if_t ifp, struct ifmediareq *ifmr) | iflib_media_status(if_t ifp, struct ifmediareq *ifmr) | ||||
{ | { | ||||
if_ctx_t ctx = if_getsoftc(ifp); | if_ctx_t ctx = if_getsoftc(ifp); | ||||
iflib_admin_intr_deferred(ctx); | |||||
CTX_LOCK(ctx); | CTX_LOCK(ctx); | ||||
IFDI_UPDATE_ADMIN_STATUS(ctx); | |||||
IFDI_MEDIA_STATUS(ctx, ifmr); | IFDI_MEDIA_STATUS(ctx, ifmr); | ||||
CTX_UNLOCK(ctx); | CTX_UNLOCK(ctx); | ||||
} | } | ||||
/* CONFIG context only */ | |||||
static void | static void | ||||
iflib_stop(if_ctx_t ctx) | iflib_stop(if_ctx_t ctx) | ||||
{ | { | ||||
iflib_txq_t txq = ctx->ifc_txqs; | iflib_txq_t txq = ctx->ifc_txqs; | ||||
iflib_rxq_t rxq = ctx->ifc_rxqs; | iflib_rxq_t rxq = ctx->ifc_rxqs; | ||||
if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; | if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; | ||||
iflib_dma_info_t di; | iflib_dma_info_t di; | ||||
iflib_fl_t fl; | iflib_fl_t fl; | ||||
int i, j; | int i, j; | ||||
/* Tell the stack that the interface is no longer active */ | /* Tell the stack that the interface is no longer active */ | ||||
if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); | if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); | ||||
IFDI_INTR_DISABLE(ctx); | IFDI_INTR_DISABLE(ctx); | ||||
DELAY(1000); | |||||
IFDI_STOP(ctx); | IFDI_STOP(ctx); | ||||
DELAY(1000); | |||||
iflib_debug_reset(); | iflib_debug_reset(); | ||||
/* Wait for current tx queue users to exit to disarm watchdog timer. */ | /* Wait for current tx queue users to exit to disarm watchdog timer. */ | ||||
for (i = 0; i < scctx->isc_ntxqsets; i++, txq++) { | for (i = 0; i < scctx->isc_ntxqsets; i++, txq++) { | ||||
/* make sure all transmitters have completed before proceeding XXX */ | /* make sure all transmitters have completed before proceeding XXX */ | ||||
/* clean any enqueued buffers */ | /* clean any enqueued buffers */ | ||||
iflib_ifmp_purge(txq); | iflib_ifmp_purge(txq); | ||||
/* Free any existing tx buffers. */ | /* Free any existing tx buffers. */ | ||||
for (j = 0; j < txq->ift_size; j++) { | for (j = 0; j < txq->ift_size; j++) { | ||||
iflib_txsd_free(ctx, txq, j); | iflib_txsd_free(ctx, txq, j); | ||||
} | } | ||||
txq->ift_processed = txq->ift_cleaned = txq->ift_cidx_processed = 0; | /* XXX please rewrite to simply bzero this range */ | ||||
txq->ift_in_use = txq->ift_gen = txq->ift_cidx = txq->ift_pidx = txq->ift_no_desc_avail = 0; | txq->ift_processed = txq->ift_cleaned = txq->ift_cleaned_prev = 0; | ||||
txq->ift_stall_count = txq->ift_cidx_processed = 0; | |||||
txq->ift_in_use = txq->ift_gen = txq->ift_cidx = txq->ift_pidx = 0; | |||||
txq->ift_closed = txq->ift_mbuf_defrag = txq->ift_mbuf_defrag_failed = 0; | txq->ift_closed = txq->ift_mbuf_defrag = txq->ift_mbuf_defrag_failed = 0; | ||||
txq->ift_no_tx_dma_setup = txq->ift_txd_encap_efbig = txq->ift_map_failed = 0; | txq->ift_no_tx_dma_setup = txq->ift_txd_encap_efbig = txq->ift_map_failed = 0; | ||||
txq->ift_pullups = 0; | txq->ift_no_desc_avail = txq->ift_pullups = 0; | ||||
ifmp_ring_reset_stats(txq->ift_br); | ifmp_ring_reset_stats(txq->ift_br); | ||||
for (j = 0, di = txq->ift_ifdi; j < ctx->ifc_nhwtxqs; j++, di++) | for (j = 0, di = txq->ift_ifdi; j < ctx->ifc_nhwtxqs; j++, di++) | ||||
bzero((void *)di->idi_vaddr, di->idi_size); | bzero((void *)di->idi_vaddr, di->idi_size); | ||||
} | } | ||||
for (i = 0; i < scctx->isc_nrxqsets; i++, rxq++) { | for (i = 0; i < scctx->isc_nrxqsets; i++, rxq++) { | ||||
/* make sure all transmitters have completed before proceeding XXX */ | /* make sure all transmitters have completed before proceeding XXX */ | ||||
for (j = 0, di = txq->ift_ifdi; j < ctx->ifc_nhwrxqs; j++, di++) | for (j = 0, di = txq->ift_ifdi; j < ctx->ifc_nhwrxqs; j++, di++) | ||||
▲ Show 20 Lines • Show All 126 Lines • ▼ Show 20 Lines | if (mh == NULL) { | ||||
flags = M_EXT; | flags = M_EXT; | ||||
mt->m_next = m; | mt->m_next = m; | ||||
mt = m; | mt = m; | ||||
/* assuming padding is only on the first fragment */ | /* assuming padding is only on the first fragment */ | ||||
padlen = 0; | padlen = 0; | ||||
} | } | ||||
cl = *sd->ifsd_cl; | cl = *sd->ifsd_cl; | ||||
*sd->ifsd_cl = NULL; | *sd->ifsd_cl = NULL; | ||||
#if MEMORY_LOGGING | |||||
sd->ifsd_fl->ifl_cl_dequeued++; | |||||
#endif | |||||
/* Can these two be made one ? */ | /* Can these two be made one ? */ | ||||
m_init(m, M_NOWAIT, MT_DATA, flags); | m_init(m, M_NOWAIT, MT_DATA, flags); | ||||
m_cljset(m, cl, sd->ifsd_fl->ifl_cltype); | m_cljset(m, cl, sd->ifsd_fl->ifl_cltype); | ||||
/* | /* | ||||
* These must follow m_init and m_cljset | * These must follow m_init and m_cljset | ||||
*/ | */ | ||||
m->m_data += padlen; | m->m_data += padlen; | ||||
▲ Show 20 Lines • Show All 53 Lines • ▼ Show 20 Lines | iflib_rxeof(iflib_rxq_t rxq, qidx_t budget) | ||||
iflib_fl_t fl; | iflib_fl_t fl; | ||||
struct ifnet *ifp; | struct ifnet *ifp; | ||||
int lro_enabled; | int lro_enabled; | ||||
/* | /* | ||||
* XXX early demux data packets so that if_input processing only handles | * XXX early demux data packets so that if_input processing only handles | ||||
* acks in interrupt context | * acks in interrupt context | ||||
*/ | */ | ||||
struct mbuf *m, *mh, *mt; | struct mbuf *m, *mh, *mt, *mf; | ||||
ifp = ctx->ifc_ifp; | ifp = ctx->ifc_ifp; | ||||
#ifdef DEV_NETMAP | |||||
if (ifp->if_capenable & IFCAP_NETMAP) { | |||||
u_int work = 0; | |||||
if (netmap_rx_irq(ifp, rxq->ifr_id, &work)) | |||||
return (FALSE); | |||||
} | |||||
#endif | |||||
mh = mt = NULL; | mh = mt = NULL; | ||||
MPASS(budget > 0); | MPASS(budget > 0); | ||||
rx_pkts = rx_bytes = 0; | rx_pkts = rx_bytes = 0; | ||||
if (sctx->isc_flags & IFLIB_HAS_RXCQ) | if (sctx->isc_flags & IFLIB_HAS_RXCQ) | ||||
cidxp = &rxq->ifr_cq_cidx; | cidxp = &rxq->ifr_cq_cidx; | ||||
else | else | ||||
cidxp = &rxq->ifr_fl[0].ifl_cidx; | cidxp = &rxq->ifr_fl[0].ifl_cidx; | ||||
if ((avail = iflib_rxd_avail(ctx, rxq, *cidxp, budget)) == 0) { | if ((avail = iflib_rxd_avail(ctx, rxq, *cidxp, budget)) == 0) { | ||||
for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++) | for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++) | ||||
__iflib_fl_refill_lt(ctx, fl, budget + 8); | __iflib_fl_refill_lt(ctx, fl, budget + 8); | ||||
DBG_COUNTER_INC(rx_unavail); | DBG_COUNTER_INC(rx_unavail); | ||||
▲ Show 20 Lines • Show All 46 Lines • ▼ Show 20 Lines | if (mh == NULL) | ||||
mh = mt = m; | mh = mt = m; | ||||
else { | else { | ||||
mt->m_nextpkt = m; | mt->m_nextpkt = m; | ||||
mt = m; | mt = m; | ||||
} | } | ||||
} | } | ||||
/* make sure that we can refill faster than drain */ | /* make sure that we can refill faster than drain */ | ||||
for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++) | for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++) | ||||
__iflib_fl_refill_lt(ctx, fl, budget + 8); | __iflib_fl_refill_lt(ctx, fl, 2*budget + 8); | ||||
lro_enabled = (if_getcapenable(ifp) & IFCAP_LRO); | lro_enabled = (if_getcapenable(ifp) & IFCAP_LRO); | ||||
mt = mf = NULL; | |||||
while (mh != NULL) { | while (mh != NULL) { | ||||
m = mh; | m = mh; | ||||
if (mf == NULL) | |||||
mf = m; | |||||
mh = mh->m_nextpkt; | mh = mh->m_nextpkt; | ||||
m->m_nextpkt = NULL; | m->m_nextpkt = NULL; | ||||
#ifndef __NO_STRICT_ALIGNMENT | #ifndef __NO_STRICT_ALIGNMENT | ||||
if (!IP_ALIGNED(m) && (m = iflib_fixup_rx(m)) == NULL) | if (!IP_ALIGNED(m) && (m = iflib_fixup_rx(m)) == NULL) | ||||
continue; | continue; | ||||
#endif | #endif | ||||
rx_bytes += m->m_pkthdr.len; | rx_bytes += m->m_pkthdr.len; | ||||
rx_pkts++; | rx_pkts++; | ||||
#if defined(INET6) || defined(INET) | #if defined(INET6) || defined(INET) | ||||
if (lro_enabled && tcp_lro_rx(&rxq->ifr_lc, m, 0) == 0) | if (lro_enabled && tcp_lro_rx(&rxq->ifr_lc, m, 0) == 0) { | ||||
if (mf == m) | |||||
mf = NULL; | |||||
continue; | continue; | ||||
} | |||||
#endif | #endif | ||||
if (mt != NULL) | |||||
mt->m_nextpkt = m; | |||||
mt = m; | |||||
} | |||||
if (mf != NULL) { | |||||
ifp->if_input(ifp, mf); | |||||
DBG_COUNTER_INC(rx_if_input); | DBG_COUNTER_INC(rx_if_input); | ||||
ifp->if_input(ifp, m); | |||||
} | } | ||||
if (rx_pkts) { | |||||
if_inc_counter(ifp, IFCOUNTER_IBYTES, rx_bytes); | if_inc_counter(ifp, IFCOUNTER_IBYTES, rx_bytes); | ||||
if_inc_counter(ifp, IFCOUNTER_IPACKETS, rx_pkts); | if_inc_counter(ifp, IFCOUNTER_IPACKETS, rx_pkts); | ||||
} | |||||
/* | /* | ||||
* Flush any outstanding LRO work | * Flush any outstanding LRO work | ||||
*/ | */ | ||||
#if defined(INET6) || defined(INET) | #if defined(INET6) || defined(INET) | ||||
tcp_lro_flush_all(&rxq->ifr_lc); | tcp_lro_flush_all(&rxq->ifr_lc); | ||||
#endif | #endif | ||||
if (avail) | return (avail || iflib_rxd_avail(ctx, rxq, *cidxp, 1)); | ||||
return true; | |||||
return (iflib_rxd_avail(ctx, rxq, *cidxp, 1)); | |||||
err: | err: | ||||
CTX_LOCK(ctx); | iflib_admin_reset_deferred(ctx); | ||||
ctx->ifc_flags |= IFC_DO_RESET; | |||||
iflib_admin_intr_deferred(ctx); | |||||
CTX_UNLOCK(ctx); | |||||
return (false); | return (false); | ||||
} | } | ||||
#define TXD_NOTIFY_COUNT(txq) (((txq)->ift_size / (txq)->ift_update_freq)-1) | #define TXD_NOTIFY_COUNT(txq) (((txq)->ift_size / (txq)->ift_update_freq)-1) | ||||
static inline qidx_t | static inline qidx_t | ||||
txq_max_db_deferred(iflib_txq_t txq, qidx_t in_use) | txq_max_db_deferred(iflib_txq_t txq, qidx_t in_use) | ||||
{ | { | ||||
qidx_t notify_count = TXD_NOTIFY_COUNT(txq); | qidx_t notify_count = TXD_NOTIFY_COUNT(txq); | ||||
▲ Show 20 Lines • Show All 70 Lines • ▼ Show 20 Lines | |||||
#endif | #endif | ||||
#define IS_TSO4(pi) ((pi)->ipi_csum_flags & CSUM_IP_TSO) | #define IS_TSO4(pi) ((pi)->ipi_csum_flags & CSUM_IP_TSO) | ||||
#define IS_TSO6(pi) ((pi)->ipi_csum_flags & CSUM_IP6_TSO) | #define IS_TSO6(pi) ((pi)->ipi_csum_flags & CSUM_IP6_TSO) | ||||
static int | static int | ||||
iflib_parse_header(iflib_txq_t txq, if_pkt_info_t pi, struct mbuf **mp) | iflib_parse_header(iflib_txq_t txq, if_pkt_info_t pi, struct mbuf **mp) | ||||
{ | { | ||||
if_shared_ctx_t sctx = txq->ift_ctx->ifc_sctx; | if_ctx_t ctx = txq->ift_ctx; | ||||
#ifdef INET | |||||
if_shared_ctx_t sctx = ctx->ifc_sctx; | |||||
#endif | |||||
if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; | |||||
struct ether_vlan_header *eh; | struct ether_vlan_header *eh; | ||||
struct mbuf *m, *n; | struct mbuf *m, *n; | ||||
int err; | |||||
if (scctx->isc_txrx->ift_txd_errata && | |||||
(err = scctx->isc_txrx->ift_txd_errata(ctx->ifc_softc, mp))) | |||||
return (err); | |||||
n = m = *mp; | n = m = *mp; | ||||
if ((sctx->isc_flags & IFLIB_NEED_SCRATCH) && | |||||
M_WRITABLE(m) == 0) { | |||||
if ((m = m_dup(m, M_NOWAIT)) == NULL) { | |||||
return (ENOMEM); | |||||
} else { | |||||
m_freem(*mp); | |||||
n = *mp = m; | |||||
} | |||||
} | |||||
/* | /* | ||||
* Determine where frame payload starts. | * Determine where frame payload starts. | ||||
* Jump over vlan headers if already present, | * Jump over vlan headers if already present, | ||||
* helpful for QinQ too. | * helpful for QinQ too. | ||||
*/ | */ | ||||
if (__predict_false(m->m_len < sizeof(*eh))) { | if (__predict_false(m->m_len < sizeof(*eh))) { | ||||
txq->ift_pullups++; | txq->ift_pullups++; | ||||
if (__predict_false((m = m_pullup(m, sizeof(*eh))) == NULL)) | if (__predict_false((m = m_pullup(m, sizeof(*eh))) == NULL)) | ||||
return (ENOMEM); | return (ENOMEM); | ||||
} | } | ||||
eh = mtod(m, struct ether_vlan_header *); | eh = mtod(m, struct ether_vlan_header *); | ||||
if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { | if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { | ||||
pi->ipi_etype = ntohs(eh->evl_proto); | pi->ipi_etype = ntohs(eh->evl_proto); | ||||
pi->ipi_ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; | pi->ipi_ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; | ||||
} else { | } else { | ||||
pi->ipi_etype = ntohs(eh->evl_encap_proto); | pi->ipi_etype = ntohs(eh->evl_encap_proto); | ||||
pi->ipi_ehdrlen = ETHER_HDR_LEN; | pi->ipi_ehdrlen = ETHER_HDR_LEN; | ||||
} | } | ||||
if (if_getmtu(txq->ift_ctx->ifc_ifp) >= pi->ipi_len) { | |||||
pi->ipi_csum_flags &= ~(CSUM_IP_TSO|CSUM_IP6_TSO); | |||||
} | |||||
switch (pi->ipi_etype) { | switch (pi->ipi_etype) { | ||||
#ifdef INET | #ifdef INET | ||||
case ETHERTYPE_IP: | case ETHERTYPE_IP: | ||||
{ | { | ||||
struct ip *ip = NULL; | struct ip *ip = NULL; | ||||
struct tcphdr *th = NULL; | struct tcphdr *th = NULL; | ||||
int minthlen; | int minthlen; | ||||
Show All 28 Lines | if (__predict_false(m->m_len < minthlen)) { | ||||
ip = (struct ip *)(m->m_data + pi->ipi_ehdrlen); | ip = (struct ip *)(m->m_data + pi->ipi_ehdrlen); | ||||
if (m->m_len >= (ip->ip_hl << 2) + sizeof(*th)) | if (m->m_len >= (ip->ip_hl << 2) + sizeof(*th)) | ||||
th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2)); | th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2)); | ||||
} | } | ||||
pi->ipi_ip_hlen = ip->ip_hl << 2; | pi->ipi_ip_hlen = ip->ip_hl << 2; | ||||
pi->ipi_ipproto = ip->ip_p; | pi->ipi_ipproto = ip->ip_p; | ||||
pi->ipi_flags |= IPI_TX_IPV4; | pi->ipi_flags |= IPI_TX_IPV4; | ||||
if (pi->ipi_csum_flags & CSUM_IP) | if ((sctx->isc_flags & IFLIB_NEED_ZERO_CSUM) && (pi->ipi_csum_flags & CSUM_IP)) | ||||
ip->ip_sum = 0; | ip->ip_sum = 0; | ||||
if (IS_TSO4(pi)) { | |||||
if (pi->ipi_ipproto == IPPROTO_TCP) { | if (pi->ipi_ipproto == IPPROTO_TCP) { | ||||
if (__predict_false(th == NULL)) { | if (__predict_false(th == NULL)) { | ||||
txq->ift_pullups++; | txq->ift_pullups++; | ||||
if (__predict_false((m = m_pullup(m, (ip->ip_hl << 2) + sizeof(*th))) == NULL)) | if (__predict_false((m = m_pullup(m, (ip->ip_hl << 2) + sizeof(*th))) == NULL)) | ||||
return (ENOMEM); | return (ENOMEM); | ||||
th = (struct tcphdr *)((caddr_t)ip + pi->ipi_ip_hlen); | th = (struct tcphdr *)((caddr_t)ip + pi->ipi_ip_hlen); | ||||
} | } | ||||
pi->ipi_tcp_hflags = th->th_flags; | pi->ipi_tcp_hflags = th->th_flags; | ||||
pi->ipi_tcp_hlen = th->th_off << 2; | pi->ipi_tcp_hlen = th->th_off << 2; | ||||
pi->ipi_tcp_seq = th->th_seq; | pi->ipi_tcp_seq = th->th_seq; | ||||
} | } | ||||
if (IS_TSO4(pi)) { | |||||
if (__predict_false(ip->ip_p != IPPROTO_TCP)) | if (__predict_false(ip->ip_p != IPPROTO_TCP)) | ||||
return (ENXIO); | return (ENXIO); | ||||
th->th_sum = in_pseudo(ip->ip_src.s_addr, | th->th_sum = in_pseudo(ip->ip_src.s_addr, | ||||
ip->ip_dst.s_addr, htons(IPPROTO_TCP)); | ip->ip_dst.s_addr, htons(IPPROTO_TCP)); | ||||
pi->ipi_tso_segsz = m->m_pkthdr.tso_segsz; | pi->ipi_tso_segsz = m->m_pkthdr.tso_segsz; | ||||
if (sctx->isc_flags & IFLIB_TSO_INIT_IP) { | if (sctx->isc_flags & IFLIB_TSO_INIT_IP) { | ||||
ip->ip_sum = 0; | ip->ip_sum = 0; | ||||
ip->ip_len = htons(pi->ipi_ip_hlen + pi->ipi_tcp_hlen + pi->ipi_tso_segsz); | ip->ip_len = htons(pi->ipi_ip_hlen + pi->ipi_tcp_hlen + pi->ipi_tso_segsz); | ||||
Show All 14 Lines | if (__predict_false(m->m_len < pi->ipi_ehdrlen + sizeof(struct ip6_hdr))) { | ||||
return (ENOMEM); | return (ENOMEM); | ||||
} | } | ||||
th = (struct tcphdr *)((caddr_t)ip6 + pi->ipi_ip_hlen); | th = (struct tcphdr *)((caddr_t)ip6 + pi->ipi_ip_hlen); | ||||
/* XXX-BZ this will go badly in case of ext hdrs. */ | /* XXX-BZ this will go badly in case of ext hdrs. */ | ||||
pi->ipi_ipproto = ip6->ip6_nxt; | pi->ipi_ipproto = ip6->ip6_nxt; | ||||
pi->ipi_flags |= IPI_TX_IPV6; | pi->ipi_flags |= IPI_TX_IPV6; | ||||
if (IS_TSO6(pi)) { | |||||
if (pi->ipi_ipproto == IPPROTO_TCP) { | if (pi->ipi_ipproto == IPPROTO_TCP) { | ||||
if (__predict_false(m->m_len < pi->ipi_ehdrlen + sizeof(struct ip6_hdr) + sizeof(struct tcphdr))) { | if (__predict_false(m->m_len < pi->ipi_ehdrlen + sizeof(struct ip6_hdr) + sizeof(struct tcphdr))) { | ||||
if (__predict_false((m = m_pullup(m, pi->ipi_ehdrlen + sizeof(struct ip6_hdr) + sizeof(struct tcphdr))) == NULL)) | if (__predict_false((m = m_pullup(m, pi->ipi_ehdrlen + sizeof(struct ip6_hdr) + sizeof(struct tcphdr))) == NULL)) | ||||
return (ENOMEM); | return (ENOMEM); | ||||
} | } | ||||
pi->ipi_tcp_hflags = th->th_flags; | pi->ipi_tcp_hflags = th->th_flags; | ||||
pi->ipi_tcp_hlen = th->th_off << 2; | pi->ipi_tcp_hlen = th->th_off << 2; | ||||
} | } | ||||
if (IS_TSO6(pi)) { | |||||
if (__predict_false(ip6->ip6_nxt != IPPROTO_TCP)) | if (__predict_false(ip6->ip6_nxt != IPPROTO_TCP)) | ||||
return (ENXIO); | return (ENXIO); | ||||
/* | /* | ||||
* The corresponding flag is set by the stack in the IPv4 | * The corresponding flag is set by the stack in the IPv4 | ||||
* TSO case, but not in IPv6 (at least in FreeBSD 10.2). | * TSO case, but not in IPv6 (at least in FreeBSD 10.2). | ||||
* So, set it here because the rest of the flow requires it. | * So, set it here because the rest of the flow requires it. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 92 Lines • ▼ Show 20 Lines | if (__predict_false(m->m_len == 0)) | ||||
*m0 = m = collapse_pkthdr(m); | *m0 = m = collapse_pkthdr(m); | ||||
ctx = txq->ift_ctx; | ctx = txq->ift_ctx; | ||||
sctx = ctx->ifc_sctx; | sctx = ctx->ifc_sctx; | ||||
scctx = &ctx->ifc_softc_ctx; | scctx = &ctx->ifc_softc_ctx; | ||||
ifsd_m = txq->ift_sds.ifsd_m; | ifsd_m = txq->ift_sds.ifsd_m; | ||||
ntxd = txq->ift_size; | ntxd = txq->ift_size; | ||||
pidx = txq->ift_pidx; | pidx = txq->ift_pidx; | ||||
if (map != NULL) { | MPASS(ifsd_m[pidx] == NULL); | ||||
if (force_busdma || map != NULL) { | |||||
uint8_t *ifsd_flags = txq->ift_sds.ifsd_flags; | uint8_t *ifsd_flags = txq->ift_sds.ifsd_flags; | ||||
err = bus_dmamap_load_mbuf_sg(tag, map, | err = bus_dmamap_load_mbuf_sg(tag, map, | ||||
*m0, segs, nsegs, BUS_DMA_NOWAIT); | *m0, segs, nsegs, BUS_DMA_NOWAIT); | ||||
if (err) | if (err) | ||||
return (err); | return (err); | ||||
ifsd_flags[pidx] |= TX_SW_DESC_MAPPED; | ifsd_flags[pidx] |= TX_SW_DESC_MAPPED; | ||||
count = 0; | count = 0; | ||||
m = *m0; | m = *m0; | ||||
do { | do { | ||||
▲ Show 20 Lines • Show All 136 Lines • ▼ Show 20 Lines | if (ctx->ifc_flags & IFC_PREFETCH) { | ||||
/* prefetch the next cache line of mbuf pointers and flags */ | /* prefetch the next cache line of mbuf pointers and flags */ | ||||
prefetch(&txq->ift_sds.ifsd_m[next]); | prefetch(&txq->ift_sds.ifsd_m[next]); | ||||
if (txq->ift_sds.ifsd_map != NULL) { | if (txq->ift_sds.ifsd_map != NULL) { | ||||
prefetch(&txq->ift_sds.ifsd_map[next]); | prefetch(&txq->ift_sds.ifsd_map[next]); | ||||
next = (cidx + CACHE_LINE_SIZE) & (ntxd-1); | next = (cidx + CACHE_LINE_SIZE) & (ntxd-1); | ||||
prefetch(&txq->ift_sds.ifsd_flags[next]); | prefetch(&txq->ift_sds.ifsd_flags[next]); | ||||
} | } | ||||
} else if (txq->ift_sds.ifsd_map != NULL) | } | ||||
if (txq->ift_sds.ifsd_map != NULL) | |||||
map = txq->ift_sds.ifsd_map[pidx]; | map = txq->ift_sds.ifsd_map[pidx]; | ||||
if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { | if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { | ||||
desc_tag = txq->ift_tso_desc_tag; | desc_tag = txq->ift_tso_desc_tag; | ||||
max_segs = scctx->isc_tx_tso_segments_max; | max_segs = scctx->isc_tx_tso_segments_max; | ||||
} else { | } else { | ||||
desc_tag = txq->ift_desc_tag; | desc_tag = txq->ift_desc_tag; | ||||
max_segs = scctx->isc_tx_nsegments; | max_segs = scctx->isc_tx_nsegments; | ||||
} | } | ||||
m_head = *m_headp; | m_head = *m_headp; | ||||
pkt_info_zero(&pi); | pkt_info_zero(&pi); | ||||
pi.ipi_len = m_head->m_pkthdr.len; | |||||
pi.ipi_mflags = (m_head->m_flags & (M_VLANTAG|M_BCAST|M_MCAST)); | pi.ipi_mflags = (m_head->m_flags & (M_VLANTAG|M_BCAST|M_MCAST)); | ||||
pi.ipi_csum_flags = m_head->m_pkthdr.csum_flags; | |||||
pi.ipi_vtag = (m_head->m_flags & M_VLANTAG) ? m_head->m_pkthdr.ether_vtag : 0; | |||||
pi.ipi_pidx = pidx; | pi.ipi_pidx = pidx; | ||||
pi.ipi_qsidx = txq->ift_id; | pi.ipi_qsidx = txq->ift_id; | ||||
pi.ipi_len = m_head->m_pkthdr.len; | |||||
pi.ipi_csum_flags = m_head->m_pkthdr.csum_flags; | |||||
pi.ipi_vtag = (m_head->m_flags & M_VLANTAG) ? m_head->m_pkthdr.ether_vtag : 0; | |||||
/* deliberate bitwise OR to make one condition */ | /* deliberate bitwise OR to make one condition */ | ||||
if (__predict_true((pi.ipi_csum_flags | pi.ipi_vtag))) { | if (__predict_true((pi.ipi_csum_flags | pi.ipi_vtag))) { | ||||
if (__predict_false((err = iflib_parse_header(txq, &pi, m_headp)) != 0)) | if (__predict_false((err = iflib_parse_header(txq, &pi, m_headp)) != 0)) | ||||
return (err); | return (err); | ||||
m_head = *m_headp; | m_head = *m_headp; | ||||
pi.ipi_hdr_data = mtod(m_head, caddr_t); | |||||
} | } | ||||
retry: | retry: | ||||
err = iflib_busdma_load_mbuf_sg(txq, desc_tag, map, m_headp, segs, &nsegs, max_segs, BUS_DMA_NOWAIT); | err = iflib_busdma_load_mbuf_sg(txq, desc_tag, map, m_headp, segs, &nsegs, max_segs, BUS_DMA_NOWAIT); | ||||
defrag: | defrag: | ||||
if (__predict_false(err)) { | if (__predict_false(err)) { | ||||
switch (err) { | switch (err) { | ||||
case EFBIG: | case EFBIG: | ||||
▲ Show 20 Lines • Show All 160 Lines • ▼ Show 20 Lines | #endif | ||||
DBG_COUNTER_INC(tx_frees); | DBG_COUNTER_INC(tx_frees); | ||||
} | } | ||||
} | } | ||||
if (__predict_false(++cidx == qsize)) { | if (__predict_false(++cidx == qsize)) { | ||||
cidx = 0; | cidx = 0; | ||||
gen = 0; | gen = 0; | ||||
} | } | ||||
} | } | ||||
txq_validate(txq); | |||||
txq->ift_cidx = cidx; | txq->ift_cidx = cidx; | ||||
txq->ift_gen = gen; | txq->ift_gen = gen; | ||||
} | } | ||||
static __inline int | static __inline int | ||||
iflib_completed_tx_reclaim(iflib_txq_t txq, int thresh) | iflib_completed_tx_reclaim(iflib_txq_t txq, int thresh) | ||||
{ | { | ||||
int reclaim; | int reclaim; | ||||
Show All 33 Lines | _ring_peek_one(struct ifmp_ring *r, int cidx, int offset, int remaining) | ||||
struct mbuf **items; | struct mbuf **items; | ||||
size = r->size; | size = r->size; | ||||
next = (cidx + CACHE_PTR_INCREMENT) & (size-1); | next = (cidx + CACHE_PTR_INCREMENT) & (size-1); | ||||
items = __DEVOLATILE(struct mbuf **, &r->items[0]); | items = __DEVOLATILE(struct mbuf **, &r->items[0]); | ||||
prefetch(items[(cidx + offset) & (size-1)]); | prefetch(items[(cidx + offset) & (size-1)]); | ||||
if (remaining > 1) { | if (remaining > 1) { | ||||
prefetch(&items[next]); | prefetch2(&items[next]); | ||||
prefetch(items[(cidx + offset + 1) & (size-1)]); | prefetch2(items[(cidx + offset + 1) & (size-1)]); | ||||
prefetch(items[(cidx + offset + 2) & (size-1)]); | prefetch2(items[(cidx + offset + 2) & (size-1)]); | ||||
prefetch(items[(cidx + offset + 3) & (size-1)]); | prefetch2(items[(cidx + offset + 3) & (size-1)]); | ||||
} | } | ||||
return (__DEVOLATILE(struct mbuf **, &r->items[(cidx + offset) & (size-1)])); | return (__DEVOLATILE(struct mbuf **, &r->items[(cidx + offset) & (size-1)])); | ||||
} | } | ||||
static void | static void | ||||
iflib_txq_check_drain(iflib_txq_t txq, int budget) | iflib_txq_check_drain(iflib_txq_t txq, int budget) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 164 Lines • ▼ Show 20 Lines | _task_fn_tx(void *context) | ||||
struct ifnet *ifp = ctx->ifc_ifp; | struct ifnet *ifp = ctx->ifc_ifp; | ||||
int rc; | int rc; | ||||
#ifdef IFLIB_DIAGNOSTICS | #ifdef IFLIB_DIAGNOSTICS | ||||
txq->ift_cpu_exec_count[curcpu]++; | txq->ift_cpu_exec_count[curcpu]++; | ||||
#endif | #endif | ||||
if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)) | if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)) | ||||
return; | return; | ||||
if ((ifp->if_capenable & IFCAP_NETMAP)) { | if (if_getcapenable(ifp) & IFCAP_NETMAP) { | ||||
if (ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id, false)) | if (ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id, false)) | ||||
netmap_tx_irq(ifp, txq->ift_id); | netmap_tx_irq(ifp, txq->ift_id); | ||||
IFDI_TX_QUEUE_INTR_ENABLE(ctx, txq->ift_id); | IFDI_TX_QUEUE_INTR_ENABLE(ctx, txq->ift_id); | ||||
return; | return; | ||||
} | } | ||||
if (txq->ift_db_pending) | if (txq->ift_db_pending) | ||||
ifmp_ring_enqueue(txq->ift_br, (void **)&txq, 1, TX_BATCH_SIZE); | ifmp_ring_enqueue(txq->ift_br, (void **)&txq, 1, TX_BATCH_SIZE); | ||||
else | |||||
ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE); | ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE); | ||||
if (ctx->ifc_flags & IFC_LEGACY) | if (ctx->ifc_flags & IFC_LEGACY) | ||||
IFDI_INTR_ENABLE(ctx); | IFDI_INTR_ENABLE(ctx); | ||||
else { | else { | ||||
rc = IFDI_TX_QUEUE_INTR_ENABLE(ctx, txq->ift_id); | rc = IFDI_TX_QUEUE_INTR_ENABLE(ctx, txq->ift_id); | ||||
KASSERT(rc != ENOTSUP, ("MSI-X support requires queue_intr_enable, but not implemented in driver")); | KASSERT(rc != ENOTSUP, ("MSI-X support requires queue_intr_enable, but not implemented in driver")); | ||||
} | } | ||||
} | } | ||||
static void | static void | ||||
_task_fn_rx(void *context) | _task_fn_rx(void *context) | ||||
{ | { | ||||
iflib_rxq_t rxq = context; | iflib_rxq_t rxq = context; | ||||
if_ctx_t ctx = rxq->ifr_ctx; | if_ctx_t ctx = rxq->ifr_ctx; | ||||
bool more; | bool more; | ||||
int rc; | int rc; | ||||
uint16_t budget; | |||||
#ifdef IFLIB_DIAGNOSTICS | #ifdef IFLIB_DIAGNOSTICS | ||||
rxq->ifr_cpu_exec_count[curcpu]++; | rxq->ifr_cpu_exec_count[curcpu]++; | ||||
#endif | #endif | ||||
DBG_COUNTER_INC(task_fn_rxs); | DBG_COUNTER_INC(task_fn_rxs); | ||||
if (__predict_false(!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING))) | if (__predict_false(!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING))) | ||||
return; | return; | ||||
if ((more = iflib_rxeof(rxq, 16 /* XXX */)) == false) { | more = true; | ||||
#ifdef DEV_NETMAP | |||||
if (if_getcapenable(ctx->ifc_ifp) & IFCAP_NETMAP) { | |||||
u_int work = 0; | |||||
if (netmap_rx_irq(ctx->ifc_ifp, rxq->ifr_id, &work)) { | |||||
more = false; | |||||
} | |||||
} | |||||
#endif | |||||
budget = ctx->ifc_sysctl_rx_budget; | |||||
if (budget == 0) | |||||
budget = 16; /* XXX */ | |||||
if (more == false || (more = iflib_rxeof(rxq, budget)) == false) { | |||||
if (ctx->ifc_flags & IFC_LEGACY) | if (ctx->ifc_flags & IFC_LEGACY) | ||||
IFDI_INTR_ENABLE(ctx); | IFDI_INTR_ENABLE(ctx); | ||||
else { | else { | ||||
DBG_COUNTER_INC(rx_intr_enables); | DBG_COUNTER_INC(rx_intr_enables); | ||||
rc = IFDI_RX_QUEUE_INTR_ENABLE(ctx, rxq->ifr_id); | rc = IFDI_RX_QUEUE_INTR_ENABLE(ctx, rxq->ifr_id); | ||||
KASSERT(rc != ENOTSUP, ("MSI-X support requires queue_intr_enable, but not implemented in driver")); | KASSERT(rc != ENOTSUP, ("MSI-X support requires queue_intr_enable, but not implemented in driver")); | ||||
} | } | ||||
} | } | ||||
if (__predict_false(!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING))) | if (__predict_false(!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING))) | ||||
return; | return; | ||||
if (more) | if (more) | ||||
GROUPTASK_ENQUEUE(&rxq->ifr_task); | GROUPTASK_ENQUEUE(&rxq->ifr_task); | ||||
} | } | ||||
/* CONFIG context only */ | |||||
static void | static void | ||||
_task_fn_admin(void *context) | _task_fn_admin(void *context) | ||||
{ | { | ||||
if_ctx_t ctx = context; | if_ctx_t ctx = context; | ||||
if_softc_ctx_t sctx = &ctx->ifc_softc_ctx; | if_softc_ctx_t sctx = &ctx->ifc_softc_ctx; | ||||
iflib_txq_t txq; | iflib_txq_t txq; | ||||
int i; | int i, running; | ||||
if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)) { | |||||
if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_OACTIVE)) { | |||||
return; | |||||
} | |||||
} | |||||
CTX_LOCK(ctx); | CTX_LOCK(ctx); | ||||
running = !!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING); | |||||
for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) { | for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) { | ||||
CALLOUT_LOCK(txq); | CALLOUT_LOCK(txq); | ||||
callout_stop(&txq->ift_timer); | callout_stop(&txq->ift_timer); | ||||
CALLOUT_UNLOCK(txq); | CALLOUT_UNLOCK(txq); | ||||
} | } | ||||
IFDI_UPDATE_ADMIN_STATUS(ctx); | if (running) { | ||||
for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) | for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) | ||||
callout_reset_on(&txq->ift_timer, hz/2, iflib_timer, txq, txq->ift_timer.c_cpu); | callout_reset_on(&txq->ift_timer, iflib_timer_int, iflib_timer, | ||||
txq, txq->ift_timer.c_cpu); | |||||
IFDI_LINK_INTR_ENABLE(ctx); | IFDI_LINK_INTR_ENABLE(ctx); | ||||
} | |||||
if (ctx->ifc_flags & IFC_DO_RESET) { | if (ctx->ifc_flags & IFC_DO_RESET) { | ||||
ctx->ifc_flags &= ~IFC_DO_RESET; | |||||
iflib_if_init_locked(ctx); | iflib_if_init_locked(ctx); | ||||
ctx->ifc_flags &= ~IFC_DO_RESET; | |||||
} | } | ||||
IFDI_UPDATE_ADMIN_STATUS(ctx); | |||||
CTX_UNLOCK(ctx); | CTX_UNLOCK(ctx); | ||||
if (LINK_ACTIVE(ctx) == 0) | if (LINK_ACTIVE(ctx) == 0 || !running) | ||||
return; | return; | ||||
for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) | for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) | ||||
iflib_txq_check_drain(txq, IFLIB_RESTART_BUDGET); | iflib_txq_check_drain(txq, IFLIB_RESTART_BUDGET); | ||||
} | } | ||||
/* CONFIG context only */ | |||||
static void | static void | ||||
_task_fn_iov(void *context) | _task_fn_iov(void *context) | ||||
{ | { | ||||
if_ctx_t ctx = context; | if_ctx_t ctx = context; | ||||
if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)) | if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)) | ||||
return; | return; | ||||
▲ Show 20 Lines • Show All 98 Lines • ▼ Show 20 Lines | for (next = m, i = 0; next != NULL; i++) { | ||||
mp[i] = next; | mp[i] = next; | ||||
next = next->m_nextpkt; | next = next->m_nextpkt; | ||||
mp[i]->m_nextpkt = NULL; | mp[i]->m_nextpkt = NULL; | ||||
} | } | ||||
#endif | #endif | ||||
DBG_COUNTER_INC(tx_seen); | DBG_COUNTER_INC(tx_seen); | ||||
err = ifmp_ring_enqueue(txq->ift_br, (void **)&m, 1, TX_BATCH_SIZE); | err = ifmp_ring_enqueue(txq->ift_br, (void **)&m, 1, TX_BATCH_SIZE); | ||||
if (err) { | |||||
GROUPTASK_ENQUEUE(&txq->ift_task); | GROUPTASK_ENQUEUE(&txq->ift_task); | ||||
if (err) { | |||||
/* support forthcoming later */ | /* support forthcoming later */ | ||||
#ifdef DRIVER_BACKPRESSURE | #ifdef DRIVER_BACKPRESSURE | ||||
txq->ift_closed = TRUE; | txq->ift_closed = TRUE; | ||||
#endif | #endif | ||||
ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE); | ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE); | ||||
m_freem(m); | m_freem(m); | ||||
} else if (TXQ_AVAIL(txq) < (txq->ift_size >> 1)) { | |||||
GROUPTASK_ENQUEUE(&txq->ift_task); | |||||
} | } | ||||
return (err); | return (err); | ||||
} | } | ||||
/* CONFIG context only */ | |||||
static void | static void | ||||
iflib_if_qflush(if_t ifp) | iflib_if_qflush(if_t ifp) | ||||
{ | { | ||||
if_ctx_t ctx = if_getsoftc(ifp); | if_ctx_t ctx = if_getsoftc(ifp); | ||||
iflib_txq_t txq = ctx->ifc_txqs; | iflib_txq_t txq = ctx->ifc_txqs; | ||||
int i; | int i; | ||||
CTX_LOCK(ctx); | CTX_LOCK(ctx); | ||||
▲ Show 20 Lines • Show All 67 Lines • ▼ Show 20 Lines | if ((err = IFDI_MTU_SET(ctx, ifr->ifr_mtu)) == 0) { | ||||
ctx->ifc_flags &= ~IFC_MULTISEG; | ctx->ifc_flags &= ~IFC_MULTISEG; | ||||
err = if_setmtu(ifp, ifr->ifr_mtu); | err = if_setmtu(ifp, ifr->ifr_mtu); | ||||
} | } | ||||
iflib_init_locked(ctx); | iflib_init_locked(ctx); | ||||
if_setdrvflags(ifp, bits); | if_setdrvflags(ifp, bits); | ||||
CTX_UNLOCK(ctx); | CTX_UNLOCK(ctx); | ||||
break; | break; | ||||
case SIOCSIFFLAGS: | case SIOCSIFFLAGS: | ||||
CTX_LOCK(ctx); | err = async_if_ioctl(ctx, command, data); | ||||
if (if_getflags(ifp) & IFF_UP) { | |||||
if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { | |||||
if ((if_getflags(ifp) ^ ctx->ifc_if_flags) & | |||||
(IFF_PROMISC | IFF_ALLMULTI)) { | |||||
err = IFDI_PROMISC_SET(ctx, if_getflags(ifp)); | |||||
} | |||||
} else | |||||
reinit = 1; | |||||
} else if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { | |||||
iflib_stop(ctx); | |||||
} | |||||
ctx->ifc_if_flags = if_getflags(ifp); | |||||
CTX_UNLOCK(ctx); | |||||
break; | break; | ||||
case SIOCADDMULTI: | case SIOCADDMULTI: | ||||
case SIOCDELMULTI: | case SIOCDELMULTI: | ||||
if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { | if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { | ||||
CTX_LOCK(ctx); | err = async_if_ioctl(ctx, command, data); | ||||
IFDI_INTR_DISABLE(ctx); | |||||
IFDI_MULTI_SET(ctx); | |||||
IFDI_INTR_ENABLE(ctx); | |||||
CTX_UNLOCK(ctx); | |||||
} | } | ||||
break; | break; | ||||
case SIOCSIFMEDIA: | case SIOCSIFMEDIA: | ||||
CTX_LOCK(ctx); | CTX_LOCK(ctx); | ||||
IFDI_MEDIA_SET(ctx); | IFDI_MEDIA_SET(ctx); | ||||
CTX_UNLOCK(ctx); | CTX_UNLOCK(ctx); | ||||
/* falls thru */ | /* falls thru */ | ||||
case SIOCGIFMEDIA: | case SIOCGIFMEDIA: | ||||
▲ Show 20 Lines • Show All 77 Lines • ▼ Show 20 Lines | |||||
} | } | ||||
/********************************************************************* | /********************************************************************* | ||||
* | * | ||||
* OTHER FUNCTIONS EXPORTED TO THE STACK | * OTHER FUNCTIONS EXPORTED TO THE STACK | ||||
* | * | ||||
**********************************************************************/ | **********************************************************************/ | ||||
/* CONFIG context only */ | |||||
static void | static void | ||||
iflib_vlan_register(void *arg, if_t ifp, uint16_t vtag) | iflib_vlan_register(void *arg, if_t ifp, uint16_t vtag) | ||||
{ | { | ||||
if_ctx_t ctx = if_getsoftc(ifp); | if_ctx_t ctx = if_getsoftc(ifp); | ||||
if ((void *)ctx != arg) | if ((void *)ctx != arg) | ||||
return; | return; | ||||
if ((vtag == 0) || (vtag > 4095)) | if ((vtag == 0) || (vtag > 4095)) | ||||
return; | return; | ||||
CTX_LOCK(ctx); | CTX_LOCK(ctx); | ||||
IFDI_VLAN_REGISTER(ctx, vtag); | IFDI_VLAN_REGISTER(ctx, vtag); | ||||
/* Re-init to load the changes */ | /* Re-init to load the changes */ | ||||
if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER) | if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER) | ||||
iflib_if_init_locked(ctx); | iflib_if_init_locked(ctx); | ||||
CTX_UNLOCK(ctx); | CTX_UNLOCK(ctx); | ||||
} | } | ||||
/* CONFIG context only */ | |||||
static void | static void | ||||
iflib_vlan_unregister(void *arg, if_t ifp, uint16_t vtag) | iflib_vlan_unregister(void *arg, if_t ifp, uint16_t vtag) | ||||
{ | { | ||||
if_ctx_t ctx = if_getsoftc(ifp); | if_ctx_t ctx = if_getsoftc(ifp); | ||||
if ((void *)ctx != arg) | if ((void *)ctx != arg) | ||||
return; | return; | ||||
if ((vtag == 0) || (vtag > 4095)) | if ((vtag == 0) || (vtag > 4095)) | ||||
return; | return; | ||||
CTX_LOCK(ctx); | CTX_LOCK(ctx); | ||||
IFDI_VLAN_UNREGISTER(ctx, vtag); | IFDI_VLAN_UNREGISTER(ctx, vtag); | ||||
/* Re-init to load the changes */ | /* Re-init to load the changes */ | ||||
if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER) | if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER) | ||||
iflib_if_init_locked(ctx); | iflib_if_init_locked(ctx); | ||||
CTX_UNLOCK(ctx); | CTX_UNLOCK(ctx); | ||||
} | } | ||||
/* CONFIG context only */ | |||||
static void | static void | ||||
iflib_led_func(void *arg, int onoff) | iflib_led_func(void *arg, int onoff) | ||||
{ | { | ||||
if_ctx_t ctx = arg; | if_ctx_t ctx = arg; | ||||
CTX_LOCK(ctx); | CTX_LOCK(ctx); | ||||
IFDI_LED_FUNC(ctx, onoff); | IFDI_LED_FUNC(ctx, onoff); | ||||
CTX_UNLOCK(ctx); | CTX_UNLOCK(ctx); | ||||
▲ Show 20 Lines • Show All 128 Lines • ▼ Show 20 Lines | if (scctx->isc_ntxd[i] < sctx->isc_ntxd_min[i]) { | ||||
scctx->isc_ntxd[i] = sctx->isc_ntxd_min[i]; | scctx->isc_ntxd[i] = sctx->isc_ntxd_min[i]; | ||||
} | } | ||||
if (scctx->isc_ntxd[i] > sctx->isc_ntxd_max[i]) { | if (scctx->isc_ntxd[i] > sctx->isc_ntxd_max[i]) { | ||||
device_printf(dev, "ntxd%d: %d greater than ntxd_max %d - resetting to max\n", | device_printf(dev, "ntxd%d: %d greater than ntxd_max %d - resetting to max\n", | ||||
i, scctx->isc_ntxd[i], sctx->isc_ntxd_max[i]); | i, scctx->isc_ntxd[i], sctx->isc_ntxd_max[i]); | ||||
scctx->isc_ntxd[i] = sctx->isc_ntxd_max[i]; | scctx->isc_ntxd[i] = sctx->isc_ntxd_max[i]; | ||||
} | } | ||||
} | } | ||||
CTX_LOCK(ctx); | |||||
if ((err = IFDI_ATTACH_PRE(ctx)) != 0) { | err = IFDI_ATTACH_PRE(ctx); | ||||
CTX_UNLOCK(ctx); | |||||
if (err) { | |||||
device_printf(dev, "IFDI_ATTACH_PRE failed %d\n", err); | device_printf(dev, "IFDI_ATTACH_PRE failed %d\n", err); | ||||
return (err); | return (err); | ||||
} | } | ||||
_iflib_pre_assert(scctx); | _iflib_pre_assert(scctx); | ||||
ctx->ifc_txrx = *scctx->isc_txrx; | ctx->ifc_txrx = *scctx->isc_txrx; | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
MPASS(scctx->isc_capenable); | MPASS(scctx->isc_capenable); | ||||
Show All 11 Lines | |||||
#ifdef ACPI_DMAR | #ifdef ACPI_DMAR | ||||
if (dmar_get_dma_tag(device_get_parent(dev), dev) != NULL) | if (dmar_get_dma_tag(device_get_parent(dev), dev) != NULL) | ||||
ctx->ifc_flags |= IFC_DMAR; | ctx->ifc_flags |= IFC_DMAR; | ||||
#elif !(defined(__i386__) || defined(__amd64__)) | #elif !(defined(__i386__) || defined(__amd64__)) | ||||
/* set unconditionally for !x86 */ | /* set unconditionally for !x86 */ | ||||
ctx->ifc_flags |= IFC_DMAR; | ctx->ifc_flags |= IFC_DMAR; | ||||
#endif | #endif | ||||
if (force_busdma) | |||||
ctx->ifc_flags |= IFC_DMAR; | |||||
msix_bar = scctx->isc_msix_bar; | msix_bar = scctx->isc_msix_bar; | ||||
main_txq = (sctx->isc_flags & IFLIB_HAS_TXCQ) ? 1 : 0; | main_txq = (sctx->isc_flags & IFLIB_HAS_TXCQ) ? 1 : 0; | ||||
main_rxq = (sctx->isc_flags & IFLIB_HAS_RXCQ) ? 1 : 0; | main_rxq = (sctx->isc_flags & IFLIB_HAS_RXCQ) ? 1 : 0; | ||||
/* XXX change for per-queue sizes */ | /* XXX change for per-queue sizes */ | ||||
device_printf(dev, "using %d tx descriptors and %d rx descriptors\n", | device_printf(dev, "using %d tx descriptors and %d rx descriptors\n", | ||||
scctx->isc_ntxd[main_txq], scctx->isc_nrxd[main_rxq]); | scctx->isc_ntxd[main_txq], scctx->isc_nrxd[main_rxq]); | ||||
for (i = 0; i < sctx->isc_nrxqs; i++) { | for (i = 0; i < sctx->isc_nrxqs; i++) { | ||||
if (!powerof2(scctx->isc_nrxd[i])) { | if (!powerof2(scctx->isc_nrxd[i])) { | ||||
/* round down instead? */ | /* round down instead? */ | ||||
device_printf(dev, "# rx descriptors must be a power of 2\n"); | device_printf(dev, "# rx descriptors must be a power of 2\n"); | ||||
err = EINVAL; | err = EINVAL; | ||||
goto fail; | goto fail; | ||||
} | } | ||||
} | } | ||||
for (i = 0; i < sctx->isc_ntxqs; i++) { | for (i = 0; i < sctx->isc_ntxqs; i++) { | ||||
if (!powerof2(scctx->isc_ntxd[i])) { | if (!powerof2(scctx->isc_ntxd[i])) { | ||||
device_printf(dev, | device_printf(dev, | ||||
"# tx descriptors must be a power of 2"); | "# tx descriptors must be a power of 2"); | ||||
Show All 22 Lines | #endif | ||||
ifp->if_hw_tsomax = scctx->isc_tx_tso_size_max; | ifp->if_hw_tsomax = scctx->isc_tx_tso_size_max; | ||||
ifp->if_hw_tsomaxsegsize = scctx->isc_tx_tso_segsize_max; | ifp->if_hw_tsomaxsegsize = scctx->isc_tx_tso_segsize_max; | ||||
if (scctx->isc_rss_table_size == 0) | if (scctx->isc_rss_table_size == 0) | ||||
scctx->isc_rss_table_size = 64; | scctx->isc_rss_table_size = 64; | ||||
scctx->isc_rss_table_mask = scctx->isc_rss_table_size-1; | scctx->isc_rss_table_mask = scctx->isc_rss_table_size-1; | ||||
GROUPTASK_INIT(&ctx->ifc_admin_task, 0, _task_fn_admin, ctx); | GROUPTASK_INIT(&ctx->ifc_admin_task, 0, _task_fn_admin, ctx); | ||||
/* XXX format name */ | /* XXX format name */ | ||||
taskqgroup_attach(qgroup_if_config_tqg, &ctx->ifc_admin_task, ctx, -1, "admin"); | taskqgroup_attach(qgroup_if_config, &ctx->ifc_admin_task, ctx, -1, "admin"); | ||||
/* | /* | ||||
** Now setup MSI or MSI/X, should | ** Now setup MSI or MSI/X, should | ||||
** return us the number of supported | ** return us the number of supported | ||||
** vectors. (Will be 1 for MSI) | ** vectors. (Will be 1 for MSI) | ||||
*/ | */ | ||||
if (sctx->isc_flags & IFLIB_SKIP_MSIX) { | if (sctx->isc_flags & IFLIB_SKIP_MSIX) { | ||||
msix = scctx->isc_vectors; | msix = scctx->isc_vectors; | ||||
} else if (scctx->isc_msix_bar != 0) | } else if (scctx->isc_msix_bar != 0) | ||||
▲ Show 20 Lines • Show All 42 Lines • ▼ Show 20 Lines | if (scctx->isc_intr == IFLIB_INTR_MSI) { | ||||
rid = 1; | rid = 1; | ||||
} | } | ||||
if ((err = iflib_legacy_setup(ctx, ctx->isc_legacy_intr, ctx->ifc_softc, &rid, "irq0")) != 0) { | if ((err = iflib_legacy_setup(ctx, ctx->isc_legacy_intr, ctx->ifc_softc, &rid, "irq0")) != 0) { | ||||
device_printf(dev, "iflib_legacy_setup failed %d\n", err); | device_printf(dev, "iflib_legacy_setup failed %d\n", err); | ||||
goto fail_intr_free; | goto fail_intr_free; | ||||
} | } | ||||
} | } | ||||
ether_ifattach(ctx->ifc_ifp, ctx->ifc_mac); | ether_ifattach(ctx->ifc_ifp, ctx->ifc_mac); | ||||
if ((err = IFDI_ATTACH_POST(ctx)) != 0) { | CTX_LOCK(ctx); | ||||
err = IFDI_ATTACH_POST(ctx); | |||||
CTX_UNLOCK(ctx); | |||||
if (err) { | |||||
device_printf(dev, "IFDI_ATTACH_POST failed %d\n", err); | device_printf(dev, "IFDI_ATTACH_POST failed %d\n", err); | ||||
goto fail_detach; | goto fail_detach; | ||||
} | } | ||||
if ((err = iflib_netmap_attach(ctx))) { | if ((err = iflib_netmap_attach(ctx))) { | ||||
device_printf(ctx->ifc_dev, "netmap attach failed: %d\n", err); | device_printf(ctx->ifc_dev, "netmap attach failed: %d\n", err); | ||||
goto fail_detach; | goto fail_detach; | ||||
} | } | ||||
*ctxp = ctx; | *ctxp = ctx; | ||||
if_setgetcounterfn(ctx->ifc_ifp, iflib_if_get_counter); | if_setgetcounterfn(ctx->ifc_ifp, iflib_if_get_counter); | ||||
iflib_add_device_sysctl_post(ctx); | iflib_add_device_sysctl_post(ctx); | ||||
iflib_ctx_insert(ctx); | |||||
ctx->ifc_flags |= IFC_INIT_DONE; | ctx->ifc_flags |= IFC_INIT_DONE; | ||||
return (0); | return (0); | ||||
fail_detach: | fail_detach: | ||||
ether_ifdetach(ctx->ifc_ifp); | ether_ifdetach(ctx->ifc_ifp); | ||||
fail_intr_free: | fail_intr_free: | ||||
if (scctx->isc_intr == IFLIB_INTR_MSIX || scctx->isc_intr == IFLIB_INTR_MSI) | if (scctx->isc_intr == IFLIB_INTR_MSIX || scctx->isc_intr == IFLIB_INTR_MSI) | ||||
pci_release_msi(ctx->ifc_dev); | pci_release_msi(ctx->ifc_dev); | ||||
fail_queues: | fail_queues: | ||||
/* XXX free queues */ | /* XXX free queues */ | ||||
fail: | fail: | ||||
CTX_LOCK(ctx); | |||||
IFDI_DETACH(ctx); | IFDI_DETACH(ctx); | ||||
CTX_UNLOCK(ctx); | |||||
return (err); | return (err); | ||||
} | } | ||||
int | int | ||||
iflib_device_attach(device_t dev) | iflib_device_attach(device_t dev) | ||||
{ | { | ||||
if_ctx_t ctx; | if_ctx_t ctx; | ||||
if_shared_ctx_t sctx; | if_shared_ctx_t sctx; | ||||
Show All 31 Lines | iflib_device_deregister(if_ctx_t ctx) | ||||
/* Unregister VLAN events */ | /* Unregister VLAN events */ | ||||
if (ctx->ifc_vlan_attach_event != NULL) | if (ctx->ifc_vlan_attach_event != NULL) | ||||
EVENTHANDLER_DEREGISTER(vlan_config, ctx->ifc_vlan_attach_event); | EVENTHANDLER_DEREGISTER(vlan_config, ctx->ifc_vlan_attach_event); | ||||
if (ctx->ifc_vlan_detach_event != NULL) | if (ctx->ifc_vlan_detach_event != NULL) | ||||
EVENTHANDLER_DEREGISTER(vlan_unconfig, ctx->ifc_vlan_detach_event); | EVENTHANDLER_DEREGISTER(vlan_unconfig, ctx->ifc_vlan_detach_event); | ||||
iflib_netmap_detach(ifp); | iflib_netmap_detach(ifp); | ||||
ether_ifdetach(ifp); | ether_ifdetach(ifp); | ||||
/* ether_ifdetach calls if_qflush - lock must be destroy afterwards*/ | |||||
CTX_LOCK_DESTROY(ctx); | |||||
if (ctx->ifc_led_dev != NULL) | if (ctx->ifc_led_dev != NULL) | ||||
led_destroy(ctx->ifc_led_dev); | led_destroy(ctx->ifc_led_dev); | ||||
/* XXX drain any dependent tasks */ | /* XXX drain any dependent tasks */ | ||||
tqg = qgroup_if_io_tqg; | tqg = qgroup_if_io; | ||||
for (txq = ctx->ifc_txqs, i = 0; i < NTXQSETS(ctx); i++, txq++) { | for (txq = ctx->ifc_txqs, i = 0; i < NTXQSETS(ctx); i++, txq++) { | ||||
callout_drain(&txq->ift_timer); | callout_drain(&txq->ift_timer); | ||||
if (txq->ift_task.gt_uniq != NULL) | if (txq->ift_task.gt_uniq != NULL) | ||||
taskqgroup_detach(tqg, &txq->ift_task); | taskqgroup_detach(tqg, &txq->ift_task); | ||||
} | } | ||||
for (i = 0, rxq = ctx->ifc_rxqs; i < NRXQSETS(ctx); i++, rxq++) { | for (i = 0, rxq = ctx->ifc_rxqs; i < NRXQSETS(ctx); i++, rxq++) { | ||||
if (rxq->ifr_task.gt_uniq != NULL) | if (rxq->ifr_task.gt_uniq != NULL) | ||||
taskqgroup_detach(tqg, &rxq->ifr_task); | taskqgroup_detach(tqg, &rxq->ifr_task); | ||||
for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) | for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) | ||||
free(fl->ifl_rx_bitmap, M_IFLIB); | free(fl->ifl_rx_bitmap, M_IFLIB); | ||||
} | } | ||||
tqg = qgroup_if_config_tqg; | tqg = qgroup_if_config; | ||||
if (ctx->ifc_admin_task.gt_uniq != NULL) | if (ctx->ifc_admin_task.gt_uniq != NULL) | ||||
taskqgroup_detach(tqg, &ctx->ifc_admin_task); | taskqgroup_detach(tqg, &ctx->ifc_admin_task); | ||||
if (ctx->ifc_vflr_task.gt_uniq != NULL) | if (ctx->ifc_vflr_task.gt_uniq != NULL) | ||||
taskqgroup_detach(tqg, &ctx->ifc_vflr_task); | taskqgroup_detach(tqg, &ctx->ifc_vflr_task); | ||||
CTX_LOCK(ctx); | |||||
IFDI_DETACH(ctx); | IFDI_DETACH(ctx); | ||||
CTX_UNLOCK(ctx); | |||||
CTX_LOCK_DESTROY(ctx); | |||||
device_set_softc(ctx->ifc_dev, NULL); | device_set_softc(ctx->ifc_dev, NULL); | ||||
if (ctx->ifc_softc_ctx.isc_intr != IFLIB_INTR_LEGACY) { | if (ctx->ifc_softc_ctx.isc_intr != IFLIB_INTR_LEGACY) { | ||||
pci_release_msi(dev); | pci_release_msi(dev); | ||||
} | } | ||||
if (ctx->ifc_softc_ctx.isc_intr != IFLIB_INTR_MSIX) { | if (ctx->ifc_softc_ctx.isc_intr != IFLIB_INTR_MSIX) { | ||||
iflib_irq_free(ctx, &ctx->ifc_legacy_irq); | iflib_irq_free(ctx, &ctx->ifc_legacy_irq); | ||||
} | } | ||||
if (ctx->ifc_msix_mem != NULL) { | if (ctx->ifc_msix_mem != NULL) { | ||||
bus_release_resource(ctx->ifc_dev, SYS_RES_MEMORY, | bus_release_resource(ctx->ifc_dev, SYS_RES_MEMORY, | ||||
ctx->ifc_softc_ctx.isc_msix_bar, ctx->ifc_msix_mem); | ctx->ifc_softc_ctx.isc_msix_bar, ctx->ifc_msix_mem); | ||||
ctx->ifc_msix_mem = NULL; | ctx->ifc_msix_mem = NULL; | ||||
} | } | ||||
bus_generic_detach(dev); | bus_generic_detach(dev); | ||||
if_free(ifp); | if_free(ifp); | ||||
iflib_tx_structures_free(ctx); | iflib_tx_structures_free(ctx); | ||||
iflib_rx_structures_free(ctx); | iflib_rx_structures_free(ctx); | ||||
if (ctx->ifc_flags & IFC_SC_ALLOCATED) | if (ctx->ifc_flags & IFC_SC_ALLOCATED) | ||||
free(ctx->ifc_softc, M_IFLIB); | free(ctx->ifc_softc, M_IFLIB); | ||||
iflib_ctx_remove(ctx); | |||||
free(ctx, M_IFLIB); | free(ctx, M_IFLIB); | ||||
return (0); | return (0); | ||||
} | } | ||||
int | int | ||||
iflib_device_detach(device_t dev) | iflib_device_detach(device_t dev) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 79 Lines • ▼ Show 20 Lines | |||||
} | } | ||||
/********************************************************************* | /********************************************************************* | ||||
* | * | ||||
* MODULE FUNCTION DEFINITIONS | * MODULE FUNCTION DEFINITIONS | ||||
* | * | ||||
**********************************************************************/ | **********************************************************************/ | ||||
/* | |||||
* - Start a fast taskqueue thread for each core | |||||
* - Start a taskqueue for control operations | |||||
*/ | |||||
static int | static int | ||||
iflib_module_init(void) | iflib_module_init(void) | ||||
{ | { | ||||
iflib_timer_int = hz / 2; | |||||
TUNABLE_INT_FETCH("net.iflib.timer_int", &iflib_timer_int); | |||||
LIST_INIT(&ctx_list); | |||||
mtx_init(&ctx_list_lock, "ctx list", NULL, MTX_DEF); | |||||
return (0); | return (0); | ||||
} | } | ||||
static int | static int | ||||
iflib_module_event_handler(module_t mod, int what, void *arg) | iflib_module_event_handler(module_t mod, int what, void *arg) | ||||
{ | { | ||||
int err; | int err; | ||||
▲ Show 20 Lines • Show All 427 Lines • ▼ Show 20 Lines | |||||
int | int | ||||
iflib_irq_alloc(if_ctx_t ctx, if_irq_t irq, int rid, | iflib_irq_alloc(if_ctx_t ctx, if_irq_t irq, int rid, | ||||
driver_filter_t filter, void *filter_arg, driver_intr_t handler, void *arg, char *name) | driver_filter_t filter, void *filter_arg, driver_intr_t handler, void *arg, char *name) | ||||
{ | { | ||||
return (_iflib_irq_alloc(ctx, irq, rid, filter, handler, arg, name)); | return (_iflib_irq_alloc(ctx, irq, rid, filter, handler, arg, name)); | ||||
} | } | ||||
#ifdef SMP | |||||
static int | static int | ||||
find_nth(if_ctx_t ctx, cpuset_t *cpus, int qid) | find_nth(if_ctx_t ctx, int qid) | ||||
{ | { | ||||
cpuset_t cpus; | |||||
int i, cpuid, eqid, count; | int i, cpuid, eqid, count; | ||||
CPU_COPY(&ctx->ifc_cpus, cpus); | CPU_COPY(&ctx->ifc_cpus, &cpus); | ||||
count = CPU_COUNT(&ctx->ifc_cpus); | count = CPU_COUNT(&ctx->ifc_cpus); | ||||
eqid = qid % count; | eqid = qid % count; | ||||
/* clear up to the qid'th bit */ | /* clear up to the qid'th bit */ | ||||
for (i = 0; i < eqid; i++) { | for (i = 0; i < eqid; i++) { | ||||
cpuid = CPU_FFS(cpus); | cpuid = CPU_FFS(&cpus); | ||||
MPASS(cpuid != 0); | MPASS(cpuid != 0); | ||||
CPU_CLR(cpuid-1, cpus); | CPU_CLR(cpuid-1, &cpus); | ||||
} | } | ||||
cpuid = CPU_FFS(cpus); | cpuid = CPU_FFS(&cpus); | ||||
MPASS(cpuid != 0); | MPASS(cpuid != 0); | ||||
return (cpuid-1); | return (cpuid-1); | ||||
} | } | ||||
static int | |||||
find_child_with_core(int cpu, struct cpu_group *grp) | |||||
{ | |||||
int i; | |||||
if (grp->cg_children == 0) | |||||
return -1; | |||||
MPASS(grp->cg_child); | |||||
for (i = 0; i < grp->cg_children; i++) { | |||||
if (CPU_ISSET(cpu, &grp->cg_child[i].cg_mask)) | |||||
return i; | |||||
} | |||||
return -1; | |||||
} | |||||
/* | |||||
* Find the nth thread on the specified core | |||||
*/ | |||||
static int | |||||
find_thread(int cpu, int thread_num) | |||||
{ | |||||
struct cpu_group *grp; | |||||
int i; | |||||
cpuset_t cs; | |||||
grp = smp_topo(); | |||||
if (grp == NULL) | |||||
return cpu; | |||||
i = 0; | |||||
while ((i = find_child_with_core(cpu, grp)) != -1) { | |||||
/* If the child only has one cpu, don't descend */ | |||||
if (grp->cg_child[i].cg_count <= 1) | |||||
break; | |||||
grp = &grp->cg_child[i]; | |||||
} | |||||
/* If they don't share at least an L2 cache, use the same CPU */ | |||||
if (grp->cg_level > CG_SHARE_L2 || grp->cg_level == CG_SHARE_NONE) | |||||
return cpu; | |||||
/* Now pick one */ | |||||
CPU_COPY(&grp->cg_mask, &cs); | |||||
for (i = thread_num % grp->cg_count; i > 0; i--) { | |||||
MPASS(CPU_FFS(&cs)); | |||||
CPU_CLR(CPU_FFS(&cs) - 1, &cs); | |||||
} | |||||
MPASS(CPU_FFS(&cs)); | |||||
return CPU_FFS(&cs) - 1; | |||||
} | |||||
static int | |||||
get_thread_num(if_ctx_t ctx, iflib_intr_type_t type, int qid) | |||||
{ | |||||
switch (type) { | |||||
case IFLIB_INTR_TX: | |||||
/* TX queues get threads on the same core as the corresponding RX queue */ | |||||
/* XXX handle multiple RX threads per core and more than two threads per core */ | |||||
return qid / CPU_COUNT(&ctx->ifc_cpus) + 1; | |||||
case IFLIB_INTR_RX: | |||||
case IFLIB_INTR_RXTX: | |||||
/* RX queues get the first thread on their core */ | |||||
return qid / CPU_COUNT(&ctx->ifc_cpus); | |||||
default: | |||||
return -1; | |||||
} | |||||
} | |||||
#else | |||||
#define get_thread_num(ctx, type, qid) 0 | |||||
#define find_thread(cpuid, tid) 0 | |||||
#define find_nth(ctx, gid) 0 | |||||
#endif | |||||
/* Just to avoid copy/paste */ | |||||
static inline int | |||||
iflib_irq_set_affinity(if_ctx_t ctx, int irq, iflib_intr_type_t type, int qid, | |||||
struct grouptask *gtask, struct taskqgroup *tqg, void *uniq, char *name) | |||||
{ | |||||
int cpuid; | |||||
int err, tid; | |||||
cpuid = find_nth(ctx, qid); | |||||
tid = get_thread_num(ctx, type, qid); | |||||
MPASS(tid >= 0); | |||||
cpuid = find_thread(cpuid, tid); | |||||
err = taskqgroup_attach_cpu(tqg, gtask, uniq, cpuid, irq, name); | |||||
if (err) { | |||||
device_printf(ctx->ifc_dev, "taskqgroup_attach_cpu failed %d\n", err); | |||||
return (err); | |||||
} | |||||
if (cpuid > ctx->ifc_cpuid_highest) | |||||
ctx->ifc_cpuid_highest = cpuid; | |||||
MPASS(gtask->gt_taskqueue != NULL); | |||||
return 0; | |||||
} | |||||
int | int | ||||
iflib_irq_alloc_generic(if_ctx_t ctx, if_irq_t irq, int rid, | iflib_irq_alloc_generic(if_ctx_t ctx, if_irq_t irq, int rid, | ||||
iflib_intr_type_t type, driver_filter_t *filter, | iflib_intr_type_t type, driver_filter_t *filter, | ||||
void *filter_arg, int qid, char *name) | void *filter_arg, int qid, char *name) | ||||
{ | { | ||||
struct grouptask *gtask; | struct grouptask *gtask; | ||||
struct taskqgroup *tqg; | struct taskqgroup *tqg; | ||||
iflib_filter_info_t info; | iflib_filter_info_t info; | ||||
cpuset_t cpus; | |||||
gtask_fn_t *fn; | gtask_fn_t *fn; | ||||
int tqrid, err, cpuid; | int tqrid, err; | ||||
driver_filter_t *intr_fast; | driver_filter_t *intr_fast; | ||||
void *q; | void *q; | ||||
info = &ctx->ifc_filter_info; | info = &ctx->ifc_filter_info; | ||||
tqrid = rid; | tqrid = rid; | ||||
switch (type) { | switch (type) { | ||||
/* XXX merge tx/rx for netmap? */ | /* XXX merge tx/rx for netmap? */ | ||||
case IFLIB_INTR_TX: | case IFLIB_INTR_TX: | ||||
q = &ctx->ifc_txqs[qid]; | q = &ctx->ifc_txqs[qid]; | ||||
info = &ctx->ifc_txqs[qid].ift_filter_info; | info = &ctx->ifc_txqs[qid].ift_filter_info; | ||||
gtask = &ctx->ifc_txqs[qid].ift_task; | gtask = &ctx->ifc_txqs[qid].ift_task; | ||||
tqg = qgroup_if_io_tqg; | tqg = qgroup_if_io; | ||||
fn = _task_fn_tx; | fn = _task_fn_tx; | ||||
intr_fast = iflib_fast_intr; | intr_fast = iflib_fast_intr; | ||||
GROUPTASK_INIT(gtask, 0, fn, q); | GROUPTASK_INIT(gtask, 0, fn, q); | ||||
break; | break; | ||||
case IFLIB_INTR_RX: | case IFLIB_INTR_RX: | ||||
q = &ctx->ifc_rxqs[qid]; | q = &ctx->ifc_rxqs[qid]; | ||||
info = &ctx->ifc_rxqs[qid].ifr_filter_info; | info = &ctx->ifc_rxqs[qid].ifr_filter_info; | ||||
gtask = &ctx->ifc_rxqs[qid].ifr_task; | gtask = &ctx->ifc_rxqs[qid].ifr_task; | ||||
tqg = qgroup_if_io_tqg; | tqg = qgroup_if_io; | ||||
fn = _task_fn_rx; | fn = _task_fn_rx; | ||||
intr_fast = iflib_fast_intr; | intr_fast = iflib_fast_intr_rx; | ||||
GROUPTASK_INIT(gtask, 0, fn, q); | GROUPTASK_INIT(gtask, 0, fn, q); | ||||
break; | break; | ||||
case IFLIB_INTR_RXTX: | case IFLIB_INTR_RXTX: | ||||
q = &ctx->ifc_rxqs[qid]; | q = &ctx->ifc_rxqs[qid]; | ||||
info = &ctx->ifc_rxqs[qid].ifr_filter_info; | info = &ctx->ifc_rxqs[qid].ifr_filter_info; | ||||
gtask = &ctx->ifc_rxqs[qid].ifr_task; | gtask = &ctx->ifc_rxqs[qid].ifr_task; | ||||
tqg = qgroup_if_io_tqg; | tqg = qgroup_if_io; | ||||
fn = _task_fn_rx; | fn = _task_fn_rx; | ||||
intr_fast = iflib_fast_intr_rxtx; | intr_fast = iflib_fast_intr_rxtx; | ||||
GROUPTASK_INIT(gtask, 0, fn, q); | GROUPTASK_INIT(gtask, 0, fn, q); | ||||
break; | break; | ||||
case IFLIB_INTR_ADMIN: | case IFLIB_INTR_ADMIN: | ||||
q = ctx; | q = ctx; | ||||
tqrid = -1; | tqrid = -1; | ||||
info = &ctx->ifc_filter_info; | info = &ctx->ifc_filter_info; | ||||
gtask = &ctx->ifc_admin_task; | gtask = &ctx->ifc_admin_task; | ||||
tqg = qgroup_if_config_tqg; | tqg = qgroup_if_config; | ||||
fn = _task_fn_admin; | fn = _task_fn_admin; | ||||
intr_fast = iflib_fast_intr_ctx; | intr_fast = iflib_fast_intr_ctx; | ||||
break; | break; | ||||
default: | default: | ||||
panic("unknown net intr type"); | panic("unknown net intr type"); | ||||
} | } | ||||
info->ifi_filter = filter; | info->ifi_filter = filter; | ||||
info->ifi_filter_arg = filter_arg; | info->ifi_filter_arg = filter_arg; | ||||
info->ifi_task = gtask; | info->ifi_task = gtask; | ||||
info->ifi_ctx = q; | info->ifi_ctx = q; | ||||
err = _iflib_irq_alloc(ctx, irq, rid, intr_fast, NULL, info, name); | err = _iflib_irq_alloc(ctx, irq, rid, intr_fast, NULL, info, name); | ||||
if (err != 0) { | if (err != 0) { | ||||
device_printf(ctx->ifc_dev, "_iflib_irq_alloc failed %d\n", err); | device_printf(ctx->ifc_dev, "_iflib_irq_alloc failed %d\n", err); | ||||
return (err); | return (err); | ||||
} | } | ||||
if (type == IFLIB_INTR_ADMIN) | if (type == IFLIB_INTR_ADMIN) | ||||
return (0); | return (0); | ||||
if (tqrid != -1) { | if (tqrid != -1) { | ||||
cpuid = find_nth(ctx, &cpus, qid); | err = iflib_irq_set_affinity(ctx, rman_get_start(irq->ii_res), type, qid, gtask, tqg, q, name); | ||||
taskqgroup_attach_cpu(tqg, gtask, q, cpuid, irq->ii_rid, name); | if (err) | ||||
return (err); | |||||
} else { | } else { | ||||
taskqgroup_attach(tqg, gtask, q, tqrid, name); | taskqgroup_attach(tqg, gtask, q, tqrid, name); | ||||
} | } | ||||
return (0); | return (0); | ||||
} | } | ||||
void | void | ||||
iflib_softirq_alloc_generic(if_ctx_t ctx, int rid, iflib_intr_type_t type, void *arg, int qid, char *name) | iflib_softirq_alloc_generic(if_ctx_t ctx, int rid, iflib_intr_type_t type, void *arg, int qid, char *name) | ||||
{ | { | ||||
struct grouptask *gtask; | struct grouptask *gtask; | ||||
struct taskqgroup *tqg; | struct taskqgroup *tqg; | ||||
gtask_fn_t *fn; | gtask_fn_t *fn; | ||||
void *q; | void *q; | ||||
int err; | |||||
switch (type) { | switch (type) { | ||||
case IFLIB_INTR_TX: | case IFLIB_INTR_TX: | ||||
q = &ctx->ifc_txqs[qid]; | q = &ctx->ifc_txqs[qid]; | ||||
gtask = &ctx->ifc_txqs[qid].ift_task; | gtask = &ctx->ifc_txqs[qid].ift_task; | ||||
tqg = qgroup_if_io_tqg; | tqg = qgroup_if_io; | ||||
fn = _task_fn_tx; | fn = _task_fn_tx; | ||||
break; | break; | ||||
case IFLIB_INTR_RX: | case IFLIB_INTR_RX: | ||||
q = &ctx->ifc_rxqs[qid]; | q = &ctx->ifc_rxqs[qid]; | ||||
gtask = &ctx->ifc_rxqs[qid].ifr_task; | gtask = &ctx->ifc_rxqs[qid].ifr_task; | ||||
tqg = qgroup_if_io_tqg; | tqg = qgroup_if_io; | ||||
fn = _task_fn_rx; | fn = _task_fn_rx; | ||||
break; | break; | ||||
case IFLIB_INTR_IOV: | case IFLIB_INTR_IOV: | ||||
q = ctx; | q = ctx; | ||||
gtask = &ctx->ifc_vflr_task; | gtask = &ctx->ifc_vflr_task; | ||||
tqg = qgroup_if_config_tqg; | tqg = qgroup_if_config; | ||||
rid = -1; | rid = -1; | ||||
fn = _task_fn_iov; | fn = _task_fn_iov; | ||||
break; | break; | ||||
default: | default: | ||||
panic("unknown net intr type"); | panic("unknown net intr type"); | ||||
} | } | ||||
GROUPTASK_INIT(gtask, 0, fn, q); | GROUPTASK_INIT(gtask, 0, fn, q); | ||||
if (rid != -1) { | |||||
err = iflib_irq_set_affinity(ctx, rid, type, qid, gtask, tqg, q, name); | |||||
if (err) | |||||
taskqgroup_attach(tqg, gtask, q, rid, name); | taskqgroup_attach(tqg, gtask, q, rid, name); | ||||
} | } | ||||
else { | |||||
taskqgroup_attach(tqg, gtask, q, rid, name); | |||||
} | |||||
} | |||||
void | void | ||||
iflib_irq_free(if_ctx_t ctx, if_irq_t irq) | iflib_irq_free(if_ctx_t ctx, if_irq_t irq) | ||||
{ | { | ||||
if (irq->ii_tag) | if (irq->ii_tag) | ||||
bus_teardown_intr(ctx->ifc_dev, irq->ii_res, irq->ii_tag); | bus_teardown_intr(ctx->ifc_dev, irq->ii_res, irq->ii_tag); | ||||
if (irq->ii_res) | if (irq->ii_res) | ||||
Show All 12 Lines | iflib_legacy_setup(if_ctx_t ctx, driver_filter_t filter, void *filter_arg, int *rid, char *name) | ||||
gtask_fn_t *fn; | gtask_fn_t *fn; | ||||
int tqrid; | int tqrid; | ||||
void *q; | void *q; | ||||
int err; | int err; | ||||
q = &ctx->ifc_rxqs[0]; | q = &ctx->ifc_rxqs[0]; | ||||
info = &rxq[0].ifr_filter_info; | info = &rxq[0].ifr_filter_info; | ||||
gtask = &rxq[0].ifr_task; | gtask = &rxq[0].ifr_task; | ||||
tqg = qgroup_if_io_tqg; | tqg = qgroup_if_io; | ||||
tqrid = irq->ii_rid = *rid; | tqrid = irq->ii_rid = *rid; | ||||
fn = _task_fn_rx; | fn = _task_fn_rx; | ||||
ctx->ifc_flags |= IFC_LEGACY; | ctx->ifc_flags |= IFC_LEGACY; | ||||
info->ifi_filter = filter; | info->ifi_filter = filter; | ||||
info->ifi_filter_arg = filter_arg; | info->ifi_filter_arg = filter_arg; | ||||
info->ifi_task = gtask; | info->ifi_task = gtask; | ||||
info->ifi_ctx = ctx; | info->ifi_ctx = ctx; | ||||
/* We allocate a single interrupt resource */ | /* We allocate a single interrupt resource */ | ||||
if ((err = _iflib_irq_alloc(ctx, irq, tqrid, iflib_fast_intr_ctx, NULL, info, name)) != 0) | if ((err = _iflib_irq_alloc(ctx, irq, tqrid, iflib_fast_intr_ctx, NULL, info, name)) != 0) | ||||
return (err); | return (err); | ||||
GROUPTASK_INIT(gtask, 0, fn, q); | GROUPTASK_INIT(gtask, 0, fn, q); | ||||
taskqgroup_attach(tqg, gtask, q, tqrid, name); | taskqgroup_attach(tqg, gtask, q, tqrid, name); | ||||
GROUPTASK_INIT(&txq->ift_task, 0, _task_fn_tx, txq); | GROUPTASK_INIT(&txq->ift_task, 0, _task_fn_tx, txq); | ||||
taskqgroup_attach(qgroup_if_io_tqg, &txq->ift_task, txq, tqrid, "tx"); | taskqgroup_attach(qgroup_if_io, &txq->ift_task, txq, tqrid, "tx"); | ||||
return (0); | return (0); | ||||
} | } | ||||
void | void | ||||
iflib_led_create(if_ctx_t ctx) | iflib_led_create(if_ctx_t ctx) | ||||
{ | { | ||||
ctx->ifc_led_dev = led_create(iflib_led_func, ctx, | ctx->ifc_led_dev = led_create(iflib_led_func, ctx, | ||||
Show All 16 Lines | |||||
void | void | ||||
iflib_admin_intr_deferred(if_ctx_t ctx) | iflib_admin_intr_deferred(if_ctx_t ctx) | ||||
{ | { | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
struct grouptask *gtask; | struct grouptask *gtask; | ||||
gtask = &ctx->ifc_admin_task; | gtask = &ctx->ifc_admin_task; | ||||
MPASS(gtask->gt_taskqueue != NULL); | MPASS(gtask != NULL && gtask->gt_taskqueue != NULL); | ||||
#endif | #endif | ||||
GROUPTASK_ENQUEUE(&ctx->ifc_admin_task); | GROUPTASK_ENQUEUE(&ctx->ifc_admin_task); | ||||
} | } | ||||
/* CONFIG context only */ | |||||
static void | |||||
iflib_handle_reset(if_ctx_t ctx, void *arg) | |||||
{ | |||||
CTX_LOCK(ctx); | |||||
ctx->ifc_flags |= IFC_DO_RESET; | |||||
iflib_admin_intr_deferred(ctx); | |||||
CTX_UNLOCK(ctx); | |||||
} | |||||
static void | |||||
iflib_admin_reset_deferred(if_ctx_t ctx) | |||||
{ | |||||
iflib_config_async_gtask_dispatch(ctx, iflib_handle_reset, "reset handler", NULL); | |||||
} | |||||
void | void | ||||
iflib_iov_intr_deferred(if_ctx_t ctx) | iflib_iov_intr_deferred(if_ctx_t ctx) | ||||
{ | { | ||||
GROUPTASK_ENQUEUE(&ctx->ifc_vflr_task); | GROUPTASK_ENQUEUE(&ctx->ifc_vflr_task); | ||||
} | } | ||||
void | void | ||||
iflib_io_tqg_attach(struct grouptask *gt, void *uniq, int cpu, char *name) | iflib_io_tqg_attach(struct grouptask *gt, void *uniq, int cpu, char *name) | ||||
{ | { | ||||
taskqgroup_attach_cpu(qgroup_if_io_tqg, gt, uniq, cpu, -1, name); | taskqgroup_attach_cpu(qgroup_if_io, gt, uniq, cpu, -1, name); | ||||
} | } | ||||
void | void | ||||
iflib_config_gtask_init(if_ctx_t ctx, struct grouptask *gtask, gtask_fn_t *fn, | iflib_config_gtask_init(if_ctx_t ctx, struct grouptask *gtask, gtask_fn_t *fn, | ||||
char *name) | char *name) | ||||
{ | { | ||||
GROUPTASK_INIT(gtask, 0, fn, ctx); | GROUPTASK_INIT(gtask, 0, fn, ctx); | ||||
taskqgroup_attach(qgroup_if_config_tqg, gtask, gtask, -1, name); | taskqgroup_attach(qgroup_if_config, gtask, gtask, -1, name); | ||||
} | } | ||||
static void | |||||
iflib_multi_set(if_ctx_t ctx, void *arg) | |||||
{ | |||||
CTX_LOCK(ctx); | |||||
IFDI_INTR_DISABLE(ctx); | |||||
IFDI_MULTI_SET(ctx); | |||||
IFDI_INTR_ENABLE(ctx); | |||||
CTX_UNLOCK(ctx); | |||||
} | |||||
static void | |||||
iflib_flags_set(if_ctx_t ctx, void *arg) | |||||
{ | |||||
int reinit, err; | |||||
if_t ifp = ctx->ifc_ifp; | |||||
err = reinit = 0; | |||||
CTX_LOCK(ctx); | |||||
if (if_getflags(ifp) & IFF_UP) { | |||||
if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { | |||||
if ((if_getflags(ifp) ^ ctx->ifc_if_flags) & | |||||
(IFF_PROMISC | IFF_ALLMULTI)) { | |||||
err = IFDI_PROMISC_SET(ctx, if_getflags(ifp)); | |||||
} | |||||
} else | |||||
reinit = 1; | |||||
} else if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { | |||||
iflib_stop(ctx); | |||||
} | |||||
ctx->ifc_if_flags = if_getflags(ifp); | |||||
if (reinit) | |||||
iflib_if_init_locked(ctx); | |||||
CTX_UNLOCK(ctx); | |||||
if (err) | |||||
log(LOG_WARNING, "IFDI_PROMISC_SET returned %d\n", err); | |||||
} | |||||
static void | |||||
async_gtask(void *ctx) | |||||
{ | |||||
struct async_task_arg *at_arg = ctx; | |||||
if_ctx_t if_ctx = at_arg->ata_ctx; | |||||
void *arg = at_arg->ata_arg; | |||||
at_arg->ata_fn(if_ctx, arg); | |||||
taskqgroup_detach(qgroup_if_config, at_arg->ata_gtask); | |||||
free(at_arg->ata_gtask, M_IFLIB); | |||||
} | |||||
static int | |||||
iflib_config_async_gtask_dispatch(if_ctx_t ctx, async_gtask_fn_t *fn, char *name, void *arg) | |||||
{ | |||||
struct grouptask *gtask; | |||||
struct async_task_arg *at_arg; | |||||
if ((gtask = malloc(sizeof(struct grouptask) + sizeof(struct async_task_arg), M_IFLIB, M_NOWAIT|M_ZERO)) == NULL) | |||||
return (ENOMEM); | |||||
at_arg = (struct async_task_arg *)(gtask + 1); | |||||
at_arg->ata_fn = fn; | |||||
at_arg->ata_ctx = ctx; | |||||
at_arg->ata_arg = arg; | |||||
at_arg->ata_gtask = gtask; | |||||
GROUPTASK_INIT(gtask, 0, async_gtask, at_arg); | |||||
taskqgroup_attach(qgroup_if_config, gtask, gtask, -1, name); | |||||
GROUPTASK_ENQUEUE(gtask); | |||||
return (0); | |||||
} | |||||
static int | |||||
async_if_ioctl(if_ctx_t ctx, u_long command, caddr_t data) | |||||
{ | |||||
int rc; | |||||
switch (command) { | |||||
case SIOCADDMULTI: | |||||
case SIOCDELMULTI: | |||||
rc = iflib_config_async_gtask_dispatch(ctx, iflib_multi_set, "async_if_multi", NULL); | |||||
break; | |||||
case SIOCSIFFLAGS: | |||||
rc = iflib_config_async_gtask_dispatch(ctx, iflib_flags_set, "async_if_flags", NULL); | |||||
break; | |||||
default: | |||||
panic("unknown command %lx", command); | |||||
} | |||||
return (rc); | |||||
} | |||||
void | void | ||||
iflib_config_gtask_deinit(struct grouptask *gtask) | iflib_config_gtask_deinit(struct grouptask *gtask) | ||||
{ | { | ||||
taskqgroup_detach(qgroup_if_config_tqg, gtask); | taskqgroup_detach(qgroup_if_config, gtask); | ||||
} | } | ||||
void | void | ||||
iflib_link_state_change(if_ctx_t ctx, int link_state, uint64_t baudrate) | iflib_link_state_change(if_ctx_t ctx, int link_state, uint64_t baudrate) | ||||
{ | { | ||||
if_t ifp = ctx->ifc_ifp; | if_t ifp = ctx->ifc_ifp; | ||||
iflib_txq_t txq = ctx->ifc_txqs; | iflib_txq_t txq = ctx->ifc_txqs; | ||||
▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Lines | iflib_add_int_delay_sysctl(if_ctx_t ctx, const char *name, | ||||
info->iidi_offset = offset; | info->iidi_offset = offset; | ||||
info->iidi_value = value; | info->iidi_value = value; | ||||
SYSCTL_ADD_PROC(device_get_sysctl_ctx(ctx->ifc_dev), | SYSCTL_ADD_PROC(device_get_sysctl_ctx(ctx->ifc_dev), | ||||
SYSCTL_CHILDREN(device_get_sysctl_tree(ctx->ifc_dev)), | SYSCTL_CHILDREN(device_get_sysctl_tree(ctx->ifc_dev)), | ||||
OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, | OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, | ||||
info, 0, iflib_sysctl_int_delay, "I", description); | info, 0, iflib_sysctl_int_delay, "I", description); | ||||
} | } | ||||
struct mtx * | struct sx * | ||||
iflib_ctx_lock_get(if_ctx_t ctx) | iflib_ctx_lock_get(if_ctx_t ctx) | ||||
{ | { | ||||
return (&ctx->ifc_mtx); | return (&ctx->ifc_sx); | ||||
} | } | ||||
static int | static int | ||||
iflib_msix_init(if_ctx_t ctx) | iflib_msix_init(if_ctx_t ctx) | ||||
{ | { | ||||
device_t dev = ctx->ifc_dev; | device_t dev = ctx->ifc_dev; | ||||
if_shared_ctx_t sctx = ctx->ifc_sctx; | if_shared_ctx_t sctx = ctx->ifc_sctx; | ||||
if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; | if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; | ||||
▲ Show 20 Lines • Show All 103 Lines • ▼ Show 20 Lines | #ifdef INVARIANTS | ||||
if (tx_queues != rx_queues) | if (tx_queues != rx_queues) | ||||
device_printf(dev, "queue equality override not set, capping rx_queues at %d and tx_queues at %d\n", | device_printf(dev, "queue equality override not set, capping rx_queues at %d and tx_queues at %d\n", | ||||
min(rx_queues, tx_queues), min(rx_queues, tx_queues)); | min(rx_queues, tx_queues), min(rx_queues, tx_queues)); | ||||
#endif | #endif | ||||
tx_queues = min(rx_queues, tx_queues); | tx_queues = min(rx_queues, tx_queues); | ||||
rx_queues = min(rx_queues, tx_queues); | rx_queues = min(rx_queues, tx_queues); | ||||
} | } | ||||
device_printf(dev, "using %d rx queues %d tx queues \n", rx_queues, tx_queues); | device_printf(dev, "trying %d rx queues %d tx queues \n", rx_queues, tx_queues); | ||||
vectors = rx_queues + admincnt; | vectors = tx_queues + rx_queues + admincnt; | ||||
if ((err = pci_alloc_msix(dev, &vectors)) == 0) { | if ((err = pci_alloc_msix(dev, &vectors)) == 0) { | ||||
device_printf(dev, | device_printf(dev, | ||||
"Using MSIX interrupts with %d vectors\n", vectors); | "Using MSIX interrupts with %d vectors\n", vectors); | ||||
scctx->isc_vectors = vectors; | scctx->isc_vectors = vectors; | ||||
if (vectors < tx_queues + rx_queues + admincnt) { | |||||
vectors -= admincnt; | |||||
if (vectors % 2 != 0) | |||||
vectors -= 1; | |||||
if (rx_queues > vectors / 2) | |||||
rx_queues = vectors / 2; | |||||
tx_queues = vectors - rx_queues; | |||||
} | |||||
scctx->isc_nrxqsets = rx_queues; | scctx->isc_nrxqsets = rx_queues; | ||||
scctx->isc_ntxqsets = tx_queues; | scctx->isc_ntxqsets = tx_queues; | ||||
scctx->isc_intr = IFLIB_INTR_MSIX; | scctx->isc_intr = IFLIB_INTR_MSIX; | ||||
return (vectors); | return (vectors); | ||||
} else { | } else { | ||||
device_printf(dev, "failed to allocate %d msix vectors, err: %d - using MSI\n", vectors, err); | device_printf(dev, "failed to allocate %d msix vectors, err: %d - using MSI\n", vectors, err); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 118 Lines • ▼ Show 20 Lines | SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "override_ntxqs", | ||||
CTLFLAG_RWTUN, &ctx->ifc_sysctl_ntxqs, 0, | CTLFLAG_RWTUN, &ctx->ifc_sysctl_ntxqs, 0, | ||||
"# of txqs to use, 0 => use default #"); | "# of txqs to use, 0 => use default #"); | ||||
SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "override_nrxqs", | SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "override_nrxqs", | ||||
CTLFLAG_RWTUN, &ctx->ifc_sysctl_nrxqs, 0, | CTLFLAG_RWTUN, &ctx->ifc_sysctl_nrxqs, 0, | ||||
"# of rxqs to use, 0 => use default #"); | "# of rxqs to use, 0 => use default #"); | ||||
SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "override_qs_enable", | SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "override_qs_enable", | ||||
CTLFLAG_RWTUN, &ctx->ifc_sysctl_qs_eq_override, 0, | CTLFLAG_RWTUN, &ctx->ifc_sysctl_qs_eq_override, 0, | ||||
"permit #txq != #rxq"); | "permit #txq != #rxq"); | ||||
SYSCTL_ADD_INT(ctx_list, oid_list, OID_AUTO, "disable_msix", | SYSCTL_ADD_INT(ctx_list, oid_list, OID_AUTO, "disable_msix", | ||||
CTLFLAG_RWTUN, &ctx->ifc_softc_ctx.isc_disable_msix, 0, | CTLFLAG_RWTUN, &ctx->ifc_softc_ctx.isc_disable_msix, 0, | ||||
"disable MSIX (default 0)"); | "disable MSIX (default 0)"); | ||||
SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "rx_budget", | |||||
CTLFLAG_RWTUN, &ctx->ifc_sysctl_rx_budget, 0, | |||||
"set the rx budget"); | |||||
/* XXX change for per-queue sizes */ | /* XXX change for per-queue sizes */ | ||||
SYSCTL_ADD_PROC(ctx_list, oid_list, OID_AUTO, "override_ntxds", | SYSCTL_ADD_PROC(ctx_list, oid_list, OID_AUTO, "override_ntxds", | ||||
CTLTYPE_STRING|CTLFLAG_RWTUN, ctx, IFLIB_NTXD_HANDLER, | CTLTYPE_STRING|CTLFLAG_RWTUN, ctx, IFLIB_NTXD_HANDLER, | ||||
mp_ndesc_handler, "A", | mp_ndesc_handler, "A", | ||||
"list of # of tx descriptors to use, 0 = use default #"); | "list of # of tx descriptors to use, 0 = use default #"); | ||||
SYSCTL_ADD_PROC(ctx_list, oid_list, OID_AUTO, "override_nrxds", | SYSCTL_ADD_PROC(ctx_list, oid_list, OID_AUTO, "override_nrxds", | ||||
CTLTYPE_STRING|CTLFLAG_RWTUN, ctx, IFLIB_NRXD_HANDLER, | CTLTYPE_STRING|CTLFLAG_RWTUN, ctx, IFLIB_NRXD_HANDLER, | ||||
mp_ndesc_handler, "A", | mp_ndesc_handler, "A", | ||||
"list of # of rx descriptors to use, 0 = use default #"); | "list of # of rx descriptors to use, 0 = use default #"); | ||||
SYSCTL_ADD_INT(ctx_list, oid_list, OID_AUTO, "watchdog_events", | |||||
CTLFLAG_RD, &ctx->ifc_watchdog_events, 0, | |||||
"Watchdog events seen since load"); | |||||
} | } | ||||
static void | static void | ||||
iflib_add_device_sysctl_post(if_ctx_t ctx) | iflib_add_device_sysctl_post(if_ctx_t ctx) | ||||
{ | { | ||||
if_shared_ctx_t sctx = ctx->ifc_sctx; | if_shared_ctx_t sctx = ctx->ifc_sctx; | ||||
if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; | if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; | ||||
device_t dev = iflib_get_dev(ctx); | device_t dev = iflib_get_dev(ctx); | ||||
▲ Show 20 Lines • Show All 176 Lines • Show Last 20 Lines |
Are these sysctls per-device? They probably should be, no? And there doesn't seem to be any code using these knobs. If that's WIP, no big deal, just curious.