Differential D19949 Diff 57096 stable/11/sys/net/iflib.c

Changeset View

Standalone View

stable/11/sys/net/iflib.c

/*-		/*-
* Copyright (c) 2014-2017, Matthew Macy <mmacy@nextbsd.org>		* Copyright (c) 2014-2018, Matthew Macy <mmacy@mattmacy.io>
* All rights reserved.		* All rights reserved.
*		*
* Redistribution and use in source and binary forms, with or without		* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:		* modification, are permitted provided that the following conditions are met:
*		*
* 1. Redistributions of source code must retain the above copyright notice,		* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.		* this list of conditions and the following disclaimer.
*		*
▲ Show 20 Lines • Show All 81 Lines • ▼ Show 20 Lines
#include <sys/memdesc.h>		#include <sys/memdesc.h>
#include <machine/bus.h>		#include <machine/bus.h>
#include <machine/md_var.h>		#include <machine/md_var.h>
#include <machine/specialreg.h>		#include <machine/specialreg.h>
#include <x86/include/busdma_impl.h>		#include <x86/include/busdma_impl.h>
#include <x86/iommu/busdma_dmar.h>		#include <x86/iommu/busdma_dmar.h>
#endif		#endif

		#ifdef PCI_IOV
		#include <dev/pci/pci_iov.h>
		#endif


#include <sys/bitstring.h>		#include <sys/bitstring.h>
/*		/*
* enable accounting of every mbuf as it comes in to and goes out of		* enable accounting of every mbuf as it comes in to and goes out of
* iflib's software descriptor references		* iflib's software descriptor references
*/		*/
#define MEMORY_LOGGING 0		#define MEMORY_LOGGING 0
/*		/*
* Enable mbuf vectors for compressing long mbuf chains		* Enable mbuf vectors for compressing long mbuf chains
Show All 37 Lines	typedef struct iflib_filter_info {
driver_filter_t *ifi_filter;		driver_filter_t *ifi_filter;
void *ifi_filter_arg;		void *ifi_filter_arg;
struct grouptask *ifi_task;		struct grouptask *ifi_task;
void *ifi_ctx;		void *ifi_ctx;
} *iflib_filter_info_t;		} *iflib_filter_info_t;

struct iflib_ctx {		struct iflib_ctx {
KOBJ_FIELDS;		KOBJ_FIELDS;
/*		/*
* Pointer to hardware driver's softc		* Pointer to hardware driver's softc
*/		*/
void *ifc_softc;		void *ifc_softc;
device_t ifc_dev;		device_t ifc_dev;
if_t ifc_ifp;		if_t ifc_ifp;

cpuset_t ifc_cpus;		cpuset_t ifc_cpus;
if_shared_ctx_t ifc_sctx;		if_shared_ctx_t ifc_sctx;
struct if_softc_ctx ifc_softc_ctx;		struct if_softc_ctx ifc_softc_ctx;

struct mtx ifc_mtx;		struct mtx ifc_ctx_mtx;
		struct mtx ifc_state_mtx;

uint16_t ifc_nhwtxqs;		uint16_t ifc_nhwtxqs;

iflib_txq_t ifc_txqs;		iflib_txq_t ifc_txqs;
iflib_rxq_t ifc_rxqs;		iflib_rxq_t ifc_rxqs;
uint32_t ifc_if_flags;		uint32_t ifc_if_flags;
uint32_t ifc_flags;		uint32_t ifc_flags;
uint32_t ifc_max_fl_buf_size;		uint32_t ifc_max_fl_buf_size;
int ifc_in_detach;

int ifc_link_state;		int ifc_link_state;
int ifc_link_irq;		int ifc_link_irq;
int ifc_watchdog_events;		int ifc_watchdog_events;
struct cdev *ifc_led_dev;		struct cdev *ifc_led_dev;
struct resource *ifc_msix_mem;		struct resource *ifc_msix_mem;

struct if_irq ifc_legacy_irq;		struct if_irq ifc_legacy_irq;
▲ Show 20 Lines • Show All 128 Lines • ▼ Show 20 Lines
#define IFC_LEGACY 0x001		#define IFC_LEGACY 0x001
#define IFC_QFLUSH 0x002		#define IFC_QFLUSH 0x002
#define IFC_MULTISEG 0x004		#define IFC_MULTISEG 0x004
#define IFC_DMAR 0x008		#define IFC_DMAR 0x008
#define IFC_SC_ALLOCATED 0x010		#define IFC_SC_ALLOCATED 0x010
#define IFC_INIT_DONE 0x020		#define IFC_INIT_DONE 0x020
#define IFC_PREFETCH 0x040		#define IFC_PREFETCH 0x040
#define IFC_DO_RESET 0x080		#define IFC_DO_RESET 0x080
#define IFC_CHECK_HUNG 0x100		#define IFC_DO_WATCHDOG 0x100
		#define IFC_CHECK_HUNG 0x200
		#define IFC_IN_DETACH 0x800


#define CSUM_OFFLOAD (CSUM_IP_TSO\|CSUM_IP6_TSO\|CSUM_IP\| \		#define CSUM_OFFLOAD (CSUM_IP_TSO\|CSUM_IP6_TSO\|CSUM_IP\| \
CSUM_IP_UDP\|CSUM_IP_TCP\|CSUM_IP_SCTP\| \		CSUM_IP_UDP\|CSUM_IP_TCP\|CSUM_IP_SCTP\| \
CSUM_IP6_UDP\|CSUM_IP6_TCP\|CSUM_IP6_SCTP)		CSUM_IP6_UDP\|CSUM_IP6_TCP\|CSUM_IP6_SCTP)
struct iflib_txq {		struct iflib_txq {
qidx_t ift_in_use;		qidx_t ift_in_use;
qidx_t ift_cidx;		qidx_t ift_cidx;
qidx_t ift_cidx_processed;		qidx_t ift_cidx_processed;
qidx_t ift_pidx;		qidx_t ift_pidx;
▲ Show 20 Lines • Show All 199 Lines • ▼ Show 20 Lines
/*		/*
* Only allow a single packet to take up most 1/nth of the tx ring		* Only allow a single packet to take up most 1/nth of the tx ring
*/		*/
#define MAX_SINGLE_PACKET_FRACTION 12		#define MAX_SINGLE_PACKET_FRACTION 12
#define IF_BAD_DMA (bus_addr_t)-1		#define IF_BAD_DMA (bus_addr_t)-1

#define CTX_ACTIVE(ctx) ((if_getdrvflags((ctx)->ifc_ifp) & IFF_DRV_RUNNING))		#define CTX_ACTIVE(ctx) ((if_getdrvflags((ctx)->ifc_ifp) & IFF_DRV_RUNNING))

#define CTX_LOCK_INIT(_sc, _name) mtx_init(&(_sc)->ifc_mtx, _name, "iflib ctx lock", MTX_DEF)		#define CTX_LOCK_INIT(_sc, _name) mtx_init(&(_sc)->ifc_ctx_mtx, _name, "iflib ctx lock", MTX_DEF)
		#define CTX_LOCK(ctx) mtx_lock(&(ctx)->ifc_ctx_mtx)
		#define CTX_UNLOCK(ctx) mtx_unlock(&(ctx)->ifc_ctx_mtx)
		#define CTX_LOCK_DESTROY(ctx) mtx_destroy(&(ctx)->ifc_ctx_mtx)

#define CTX_LOCK(ctx) mtx_lock(&(ctx)->ifc_mtx)
#define CTX_UNLOCK(ctx) mtx_unlock(&(ctx)->ifc_mtx)
#define CTX_LOCK_DESTROY(ctx) mtx_destroy(&(ctx)->ifc_mtx)

		#define STATE_LOCK_INIT(_sc, _name) mtx_init(&(_sc)->ifc_state_mtx, _name, "iflib state lock", MTX_DEF)
		#define STATE_LOCK(ctx) mtx_lock(&(ctx)->ifc_state_mtx)
		#define STATE_UNLOCK(ctx) mtx_unlock(&(ctx)->ifc_state_mtx)
		#define STATE_LOCK_DESTROY(ctx) mtx_destroy(&(ctx)->ifc_state_mtx)



#define CALLOUT_LOCK(txq) mtx_lock(&txq->ift_mtx)		#define CALLOUT_LOCK(txq) mtx_lock(&txq->ift_mtx)
#define CALLOUT_UNLOCK(txq) mtx_unlock(&txq->ift_mtx)		#define CALLOUT_UNLOCK(txq) mtx_unlock(&txq->ift_mtx)


/* Our boot-time initialization hook */		/* Our boot-time initialization hook */
static int iflib_module_event_handler(module_t, int, void *);		static int iflib_module_event_handler(module_t, int, void *);

static moduledata_t iflib_moduledata = {		static moduledata_t iflib_moduledata = {
"iflib",		"iflib",
iflib_module_event_handler,		iflib_module_event_handler,
NULL		NULL
};		};
▲ Show 20 Lines • Show All 161 Lines • ▼ Show 20 Lines
static int iflib_register(if_ctx_t);		static int iflib_register(if_ctx_t);
static void iflib_init_locked(if_ctx_t ctx);		static void iflib_init_locked(if_ctx_t ctx);
static void iflib_add_device_sysctl_pre(if_ctx_t ctx);		static void iflib_add_device_sysctl_pre(if_ctx_t ctx);
static void iflib_add_device_sysctl_post(if_ctx_t ctx);		static void iflib_add_device_sysctl_post(if_ctx_t ctx);
static void iflib_ifmp_purge(iflib_txq_t txq);		static void iflib_ifmp_purge(iflib_txq_t txq);
static void _iflib_pre_assert(if_softc_ctx_t scctx);		static void _iflib_pre_assert(if_softc_ctx_t scctx);
static void iflib_stop(if_ctx_t ctx);		static void iflib_stop(if_ctx_t ctx);
static void iflib_if_init_locked(if_ctx_t ctx);		static void iflib_if_init_locked(if_ctx_t ctx);
		static void iflib_free_intr_mem(if_ctx_t ctx);
#ifndef __NO_STRICT_ALIGNMENT		#ifndef __NO_STRICT_ALIGNMENT
static struct mbuf * iflib_fixup_rx(struct mbuf *m);		static struct mbuf * iflib_fixup_rx(struct mbuf *m);
#endif		#endif

#ifdef DEV_NETMAP		#ifdef DEV_NETMAP
#include <sys/selinfo.h>		#include <sys/selinfo.h>
#include <net/netmap.h>		#include <net/netmap.h>
#include <dev/netmap/netmap_kern.h>		#include <dev/netmap/netmap_kern.h>
▲ Show 20 Lines • Show All 1,212 Lines • ▼ Show 20 Lines	#endif

MPASS(fl->ifl_credits <= fl->ifl_size);		MPASS(fl->ifl_credits <= fl->ifl_size);
MPASS(reclaimable == delta);		MPASS(reclaimable == delta);

if (reclaimable > 0)		if (reclaimable > 0)
_iflib_fl_refill(ctx, fl, min(max, reclaimable));		_iflib_fl_refill(ctx, fl, min(max, reclaimable));
}		}

		uint8_t
		iflib_in_detach(if_ctx_t ctx)
		{
		bool in_detach;
		STATE_LOCK(ctx);
		in_detach = !!(ctx->ifc_flags & IFC_IN_DETACH);
		STATE_UNLOCK(ctx);
		return (in_detach);
		}

static void		static void
iflib_fl_bufs_free(iflib_fl_t fl)		iflib_fl_bufs_free(iflib_fl_t fl)
{		{
iflib_dma_info_t idi = fl->ifl_ifdi;		iflib_dma_info_t idi = fl->ifl_ifdi;
uint32_t i;		uint32_t i;

for (i = 0; i < fl->ifl_size; i++) {		for (i = 0; i < fl->ifl_size; i++) {
struct mbuf **sd_m = &fl->ifl_sds.ifsd_m[i];		struct mbuf **sd_m = &fl->ifl_sds.ifsd_m[i];
uint8_t *sd_flags = &fl->ifl_sds.ifsd_flags[i];		uint8_t *sd_flags = &fl->ifl_sds.ifsd_flags[i];
caddr_t *sd_cl = &fl->ifl_sds.ifsd_cl[i];		caddr_t *sd_cl = &fl->ifl_sds.ifsd_cl[i];

if (*sd_flags & RX_SW_DESC_INUSE) {		if (*sd_flags & RX_SW_DESC_INUSE) {
if (fl->ifl_sds.ifsd_map != NULL) {		if (fl->ifl_sds.ifsd_map != NULL) {
bus_dmamap_t sd_map = fl->ifl_sds.ifsd_map[i];		bus_dmamap_t sd_map = fl->ifl_sds.ifsd_map[i];
bus_dmamap_unload(fl->ifl_desc_tag, sd_map);		bus_dmamap_unload(fl->ifl_desc_tag, sd_map);
if (fl->ifl_rxq->ifr_ctx->ifc_in_detach)		// XXX: Should this get moved out?
		if (iflib_in_detach(fl->ifl_rxq->ifr_ctx))
bus_dmamap_destroy(fl->ifl_desc_tag, sd_map);		bus_dmamap_destroy(fl->ifl_desc_tag, sd_map);
}		}
if (*sd_m != NULL) {		if (*sd_m != NULL) {
m_init(*sd_m, M_NOWAIT, MT_DATA, 0);		m_init(*sd_m, M_NOWAIT, MT_DATA, 0);
uma_zfree(zone_mbuf, *sd_m);		uma_zfree(zone_mbuf, *sd_m);
}		}
if (*sd_cl != NULL)		if (*sd_cl != NULL)
uma_zfree(fl->ifl_zone, *sd_cl);		uma_zfree(fl->ifl_zone, *sd_cl);
▲ Show 20 Lines • Show All 144 Lines • ▼ Show 20 Lines	iflib_timer(void *arg)
/* handle any laggards */		/* handle any laggards */
if (txq->ift_db_pending)		if (txq->ift_db_pending)
GROUPTASK_ENQUEUE(&txq->ift_task);		GROUPTASK_ENQUEUE(&txq->ift_task);

sctx->isc_pause_frames = 0;		sctx->isc_pause_frames = 0;
if (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)		if (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)
callout_reset_on(&txq->ift_timer, hz/2, iflib_timer, txq, txq->ift_timer.c_cpu);		callout_reset_on(&txq->ift_timer, hz/2, iflib_timer, txq, txq->ift_timer.c_cpu);
return;		return;
hung:		hung:
CTX_LOCK(ctx);
if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
device_printf(ctx->ifc_dev, "TX(%d) desc avail = %d, pidx = %d\n",		device_printf(ctx->ifc_dev, "TX(%d) desc avail = %d, pidx = %d\n",
txq->ift_id, TXQ_AVAIL(txq), txq->ift_pidx);		txq->ift_id, TXQ_AVAIL(txq), txq->ift_pidx);
		STATE_LOCK(ctx);
IFDI_WATCHDOG_RESET(ctx);		if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
ctx->ifc_watchdog_events++;		ctx->ifc_flags \|= (IFC_DO_WATCHDOG\|IFC_DO_RESET);

ctx->ifc_flags \|= IFC_DO_RESET;
iflib_admin_intr_deferred(ctx);		iflib_admin_intr_deferred(ctx);
CTX_UNLOCK(ctx);		STATE_UNLOCK(ctx);
}		}

static void		static void
iflib_init_locked(if_ctx_t ctx)		iflib_init_locked(if_ctx_t ctx)
{		{
if_softc_ctx_t sctx = &ctx->ifc_softc_ctx;		if_softc_ctx_t sctx = &ctx->ifc_softc_ctx;
if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;		if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
if_t ifp = ctx->ifc_ifp;		if_t ifp = ctx->ifc_ifp;
▲ Show 20 Lines • Show All 501 Lines • ▼ Show 20 Lines	#endif
*/		*/
#if defined(INET6) \|\| defined(INET)		#if defined(INET6) \|\| defined(INET)
tcp_lro_flush_all(&rxq->ifr_lc);		tcp_lro_flush_all(&rxq->ifr_lc);
#endif		#endif
if (avail)		if (avail)
return true;		return true;
return (iflib_rxd_avail(ctx, rxq, *cidxp, 1));		return (iflib_rxd_avail(ctx, rxq, *cidxp, 1));
err:		err:
CTX_LOCK(ctx);		STATE_LOCK(ctx);
ctx->ifc_flags \|= IFC_DO_RESET;		ctx->ifc_flags \|= IFC_DO_RESET;
iflib_admin_intr_deferred(ctx);		iflib_admin_intr_deferred(ctx);
CTX_UNLOCK(ctx);		STATE_UNLOCK(ctx);
return (false);		return (false);
}		}

#define TXD_NOTIFY_COUNT(txq) (((txq)->ift_size / (txq)->ift_update_freq)-1)		#define TXD_NOTIFY_COUNT(txq) (((txq)->ift_size / (txq)->ift_update_freq)-1)
static inline qidx_t		static inline qidx_t
txq_max_db_deferred(iflib_txq_t txq, qidx_t in_use)		txq_max_db_deferred(iflib_txq_t txq, qidx_t in_use)
{		{
qidx_t notify_count = TXD_NOTIFY_COUNT(txq);		qidx_t notify_count = TXD_NOTIFY_COUNT(txq);
▲ Show 20 Lines • Show All 1,031 Lines • ▼ Show 20 Lines

static void		static void
_task_fn_admin(void *context)		_task_fn_admin(void *context)
{		{
if_ctx_t ctx = context;		if_ctx_t ctx = context;
if_softc_ctx_t sctx = &ctx->ifc_softc_ctx;		if_softc_ctx_t sctx = &ctx->ifc_softc_ctx;
iflib_txq_t txq;		iflib_txq_t txq;
int i;		int i;
		bool oactive, running, do_reset, do_watchdog, in_detach;

if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)) {		STATE_LOCK(ctx);
if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_OACTIVE)) {		running = (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING);
		oactive = (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_OACTIVE);
		do_reset = (ctx->ifc_flags & IFC_DO_RESET);
		do_watchdog = (ctx->ifc_flags & IFC_DO_WATCHDOG);
		in_detach = (ctx->ifc_flags & IFC_IN_DETACH);
		ctx->ifc_flags &= ~(IFC_DO_RESET\|IFC_DO_WATCHDOG);
		STATE_UNLOCK(ctx);

		if ((!running & !oactive) &&
		!(ctx->ifc_sctx->isc_flags & IFLIB_ADMIN_ALWAYS_RUN))
		if (in_detach)
return;		return;
}
}

CTX_LOCK(ctx);		CTX_LOCK(ctx);
for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) {		for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) {
CALLOUT_LOCK(txq);		CALLOUT_LOCK(txq);
callout_stop(&txq->ift_timer);		callout_stop(&txq->ift_timer);
CALLOUT_UNLOCK(txq);		CALLOUT_UNLOCK(txq);
}		}
		if (do_watchdog) {
		ctx->ifc_watchdog_events++;
		IFDI_WATCHDOG_RESET(ctx);
		}
IFDI_UPDATE_ADMIN_STATUS(ctx);		IFDI_UPDATE_ADMIN_STATUS(ctx);
for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++)		for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++)
callout_reset_on(&txq->ift_timer, hz/2, iflib_timer, txq, txq->ift_timer.c_cpu);		callout_reset_on(&txq->ift_timer, hz/2, iflib_timer, txq, txq->ift_timer.c_cpu);
IFDI_LINK_INTR_ENABLE(ctx);		IFDI_LINK_INTR_ENABLE(ctx);
if (ctx->ifc_flags & IFC_DO_RESET) {		if (do_reset)
ctx->ifc_flags &= ~IFC_DO_RESET;
iflib_if_init_locked(ctx);		iflib_if_init_locked(ctx);
}
CTX_UNLOCK(ctx);		CTX_UNLOCK(ctx);

if (LINK_ACTIVE(ctx) == 0)		if (LINK_ACTIVE(ctx) == 0)
return;		return;
for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++)		for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++)
iflib_txq_check_drain(txq, IFLIB_RESTART_BUDGET);		iflib_txq_check_drain(txq, IFLIB_RESTART_BUDGET);
}		}


static void		static void
_task_fn_iov(void *context)		_task_fn_iov(void *context)
{		{
if_ctx_t ctx = context;		if_ctx_t ctx = context;

if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING))		if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING) &&
		!(ctx->ifc_sctx->isc_flags & IFLIB_ADMIN_ALWAYS_RUN))
return;		return;

CTX_LOCK(ctx);		CTX_LOCK(ctx);
IFDI_VFLR_HANDLE(ctx);		IFDI_VFLR_HANDLE(ctx);
CTX_UNLOCK(ctx);		CTX_UNLOCK(ctx);
}		}

static int		static int
▲ Show 20 Lines • Show All 112 Lines • ▼ Show 20 Lines

static void		static void
iflib_if_qflush(if_t ifp)		iflib_if_qflush(if_t ifp)
{		{
if_ctx_t ctx = if_getsoftc(ifp);		if_ctx_t ctx = if_getsoftc(ifp);
iflib_txq_t txq = ctx->ifc_txqs;		iflib_txq_t txq = ctx->ifc_txqs;
int i;		int i;

CTX_LOCK(ctx);		STATE_LOCK(ctx);
ctx->ifc_flags \|= IFC_QFLUSH;		ctx->ifc_flags \|= IFC_QFLUSH;
CTX_UNLOCK(ctx);		STATE_UNLOCK(ctx);
for (i = 0; i < NTXQSETS(ctx); i++, txq++)		for (i = 0; i < NTXQSETS(ctx); i++, txq++)
while (!(ifmp_ring_is_idle(txq->ift_br) \|\| ifmp_ring_is_stalled(txq->ift_br)))		while (!(ifmp_ring_is_idle(txq->ift_br) \|\| ifmp_ring_is_stalled(txq->ift_br)))
iflib_txq_check_drain(txq, 0);		iflib_txq_check_drain(txq, 0);
CTX_LOCK(ctx);		STATE_LOCK(ctx);
ctx->ifc_flags &= ~IFC_QFLUSH;		ctx->ifc_flags &= ~IFC_QFLUSH;
CTX_UNLOCK(ctx);		STATE_UNLOCK(ctx);

if_qflush(ifp);		if_qflush(ifp);
}		}


#define IFCAP_FLAGS (IFCAP_TXCSUM_IPV6 \| IFCAP_RXCSUM_IPV6 \| IFCAP_HWCSUM \| IFCAP_LRO \| \		#define IFCAP_FLAGS (IFCAP_TXCSUM_IPV6 \| IFCAP_RXCSUM_IPV6 \| IFCAP_HWCSUM \| IFCAP_LRO \| \
IFCAP_TSO4 \| IFCAP_TSO6 \| IFCAP_VLAN_HWTAGGING \| IFCAP_HWSTATS \| \		IFCAP_TSO4 \| IFCAP_TSO6 \| IFCAP_VLAN_HWTAGGING \| IFCAP_HWSTATS \| \
IFCAP_VLAN_MTU \| IFCAP_VLAN_HWFILTER \| IFCAP_VLAN_HWTSO)		IFCAP_VLAN_MTU \| IFCAP_VLAN_HWFILTER \| IFCAP_VLAN_HWTSO)
Show All 40 Lines	if (ifr->ifr_mtu == if_getmtu(ifp)) {
CTX_UNLOCK(ctx);		CTX_UNLOCK(ctx);
break;		break;
}		}
bits = if_getdrvflags(ifp);		bits = if_getdrvflags(ifp);
/* stop the driver and free any clusters before proceeding */		/* stop the driver and free any clusters before proceeding */
iflib_stop(ctx);		iflib_stop(ctx);

if ((err = IFDI_MTU_SET(ctx, ifr->ifr_mtu)) == 0) {		if ((err = IFDI_MTU_SET(ctx, ifr->ifr_mtu)) == 0) {
		STATE_LOCK(ctx);
if (ifr->ifr_mtu > ctx->ifc_max_fl_buf_size)		if (ifr->ifr_mtu > ctx->ifc_max_fl_buf_size)
ctx->ifc_flags \|= IFC_MULTISEG;		ctx->ifc_flags \|= IFC_MULTISEG;
else		else
ctx->ifc_flags &= ~IFC_MULTISEG;		ctx->ifc_flags &= ~IFC_MULTISEG;
		STATE_UNLOCK(ctx);
err = if_setmtu(ifp, ifr->ifr_mtu);		err = if_setmtu(ifp, ifr->ifr_mtu);
}		}
iflib_init_locked(ctx);		iflib_init_locked(ctx);
		STATE_LOCK(ctx);
if_setdrvflags(ifp, bits);		if_setdrvflags(ifp, bits);
		STATE_UNLOCK(ctx);
CTX_UNLOCK(ctx);		CTX_UNLOCK(ctx);
break;		break;
case SIOCSIFFLAGS:		case SIOCSIFFLAGS:
CTX_LOCK(ctx);		CTX_LOCK(ctx);
if (if_getflags(ifp) & IFF_UP) {		if (if_getflags(ifp) & IFF_UP) {
if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {		if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
if ((if_getflags(ifp) ^ ctx->ifc_if_flags) &		if ((if_getflags(ifp) ^ ctx->ifc_if_flags) &
(IFF_PROMISC \| IFF_ALLMULTI)) {		(IFF_PROMISC \| IFF_ALLMULTI)) {
▲ Show 20 Lines • Show All 67 Lines • ▼ Show 20 Lines	#endif
/*		/*
* want to ensure that traffic has stopped before we change any of the flags		* want to ensure that traffic has stopped before we change any of the flags
*/		*/
if (setmask) {		if (setmask) {
CTX_LOCK(ctx);		CTX_LOCK(ctx);
bits = if_getdrvflags(ifp);		bits = if_getdrvflags(ifp);
if (bits & IFF_DRV_RUNNING)		if (bits & IFF_DRV_RUNNING)
iflib_stop(ctx);		iflib_stop(ctx);
		STATE_LOCK(ctx);
if_togglecapenable(ifp, setmask);		if_togglecapenable(ifp, setmask);
		STATE_UNLOCK(ctx);
if (bits & IFF_DRV_RUNNING)		if (bits & IFF_DRV_RUNNING)
iflib_init_locked(ctx);		iflib_init_locked(ctx);
		STATE_LOCK(ctx);
if_setdrvflags(ifp, bits);		if_setdrvflags(ifp, bits);
		STATE_UNLOCK(ctx);
CTX_UNLOCK(ctx);		CTX_UNLOCK(ctx);
}		}
break;		break;
}		}
case SIOCGPRIVATE_0:		case SIOCGPRIVATE_0:
case SIOCSDRVSPEC:		case SIOCSDRVSPEC:
case SIOCGDRVSPEC:		case SIOCGDRVSPEC:
CTX_LOCK(ctx);		CTX_LOCK(ctx);
▲ Show 20 Lines • Show All 397 Lines • ▼ Show 20 Lines	iflib_device_deregister(if_ctx_t ctx)
iflib_rxq_t rxq;		iflib_rxq_t rxq;
device_t dev = ctx->ifc_dev;		device_t dev = ctx->ifc_dev;
int i, j;		int i, j;
struct taskqgroup *tqg;		struct taskqgroup *tqg;
iflib_fl_t fl;		iflib_fl_t fl;

/* Make sure VLANS are not using driver */		/* Make sure VLANS are not using driver */
if (if_vlantrunkinuse(ifp)) {		if (if_vlantrunkinuse(ifp)) {
device_printf(dev,"Vlan in use, detach first\n");		device_printf(dev, "Vlan in use, detach first\n");
return (EBUSY);		return (EBUSY);
}		}
		#ifdef PCI_IOV
		if (!CTX_IS_VF(ctx) && pci_iov_detach(dev) != 0) {
		device_printf(dev, "SR-IOV in use; detach first.\n");
		return (EBUSY);
		}
		#endif

		STATE_LOCK(ctx);
		ctx->ifc_flags \|= IFC_IN_DETACH;
		STATE_UNLOCK(ctx);

CTX_LOCK(ctx);		CTX_LOCK(ctx);
ctx->ifc_in_detach = 1;
iflib_stop(ctx);		iflib_stop(ctx);
CTX_UNLOCK(ctx);		CTX_UNLOCK(ctx);

/* Unregister VLAN events */		/* Unregister VLAN events */
if (ctx->ifc_vlan_attach_event != NULL)		if (ctx->ifc_vlan_attach_event != NULL)
EVENTHANDLER_DEREGISTER(vlan_config, ctx->ifc_vlan_attach_event);		EVENTHANDLER_DEREGISTER(vlan_config, ctx->ifc_vlan_attach_event);
if (ctx->ifc_vlan_detach_event != NULL)		if (ctx->ifc_vlan_detach_event != NULL)
EVENTHANDLER_DEREGISTER(vlan_unconfig, ctx->ifc_vlan_detach_event);		EVENTHANDLER_DEREGISTER(vlan_unconfig, ctx->ifc_vlan_detach_event);

iflib_netmap_detach(ifp);		iflib_netmap_detach(ifp);
ether_ifdetach(ifp);		ether_ifdetach(ifp);
/* ether_ifdetach calls if_qflush - lock must be destroy afterwards*/
CTX_LOCK_DESTROY(ctx);
if (ctx->ifc_led_dev != NULL)		if (ctx->ifc_led_dev != NULL)
led_destroy(ctx->ifc_led_dev);		led_destroy(ctx->ifc_led_dev);
/* XXX drain any dependent tasks */		/* XXX drain any dependent tasks */
tqg = qgroup_if_io_tqg;		tqg = qgroup_if_io_tqg;
for (txq = ctx->ifc_txqs, i = 0; i < NTXQSETS(ctx); i++, txq++) {		for (txq = ctx->ifc_txqs, i = 0; i < NTXQSETS(ctx); i++, txq++) {
callout_drain(&txq->ift_timer);		callout_drain(&txq->ift_timer);
if (txq->ift_task.gt_uniq != NULL)		if (txq->ift_task.gt_uniq != NULL)
taskqgroup_detach(tqg, &txq->ift_task);		taskqgroup_detach(tqg, &txq->ift_task);
}		}
for (i = 0, rxq = ctx->ifc_rxqs; i < NRXQSETS(ctx); i++, rxq++) {		for (i = 0, rxq = ctx->ifc_rxqs; i < NRXQSETS(ctx); i++, rxq++) {
if (rxq->ifr_task.gt_uniq != NULL)		if (rxq->ifr_task.gt_uniq != NULL)
taskqgroup_detach(tqg, &rxq->ifr_task);		taskqgroup_detach(tqg, &rxq->ifr_task);

for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++)		for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++)
free(fl->ifl_rx_bitmap, M_IFLIB);		free(fl->ifl_rx_bitmap, M_IFLIB);

}		}
tqg = qgroup_if_config_tqg;		tqg = qgroup_if_config_tqg;
if (ctx->ifc_admin_task.gt_uniq != NULL)		if (ctx->ifc_admin_task.gt_uniq != NULL)
taskqgroup_detach(tqg, &ctx->ifc_admin_task);		taskqgroup_detach(tqg, &ctx->ifc_admin_task);
if (ctx->ifc_vflr_task.gt_uniq != NULL)		if (ctx->ifc_vflr_task.gt_uniq != NULL)
taskqgroup_detach(tqg, &ctx->ifc_vflr_task);		taskqgroup_detach(tqg, &ctx->ifc_vflr_task);
		CTX_LOCK(ctx);
IFDI_DETACH(ctx);		IFDI_DETACH(ctx);
		CTX_UNLOCK(ctx);

		/* ether_ifdetach calls if_qflush - lock must be destroy afterwards*/
		CTX_LOCK_DESTROY(ctx);
device_set_softc(ctx->ifc_dev, NULL);		device_set_softc(ctx->ifc_dev, NULL);
if (ctx->ifc_softc_ctx.isc_intr != IFLIB_INTR_LEGACY) {		iflib_free_intr_mem(ctx);
pci_release_msi(dev);
}
if (ctx->ifc_softc_ctx.isc_intr != IFLIB_INTR_MSIX) {
iflib_irq_free(ctx, &ctx->ifc_legacy_irq);
}
if (ctx->ifc_msix_mem != NULL) {
bus_release_resource(ctx->ifc_dev, SYS_RES_MEMORY,
ctx->ifc_softc_ctx.isc_msix_bar, ctx->ifc_msix_mem);
ctx->ifc_msix_mem = NULL;
}

bus_generic_detach(dev);		bus_generic_detach(dev);
if_free(ifp);		if_free(ifp);

iflib_tx_structures_free(ctx);		iflib_tx_structures_free(ctx);
iflib_rx_structures_free(ctx);		iflib_rx_structures_free(ctx);
if (ctx->ifc_flags & IFC_SC_ALLOCATED)		if (ctx->ifc_flags & IFC_SC_ALLOCATED)
free(ctx->ifc_softc, M_IFLIB);		free(ctx->ifc_softc, M_IFLIB);
		STATE_LOCK_DESTROY(ctx);
free(ctx, M_IFLIB);		free(ctx, M_IFLIB);
return (0);		return (0);
}		}

		static void
		iflib_free_intr_mem(if_ctx_t ctx)
		{

		if (ctx->ifc_softc_ctx.isc_intr != IFLIB_INTR_MSIX) {
		iflib_irq_free(ctx, &ctx->ifc_legacy_irq);
		}
		if (ctx->ifc_softc_ctx.isc_intr != IFLIB_INTR_LEGACY) {
		pci_release_msi(ctx->ifc_dev);
		}
		if (ctx->ifc_msix_mem != NULL) {
		bus_release_resource(ctx->ifc_dev, SYS_RES_MEMORY,
		rman_get_rid(ctx->ifc_msix_mem), ctx->ifc_msix_mem);
		ctx->ifc_msix_mem = NULL;
		}
		}

int		int
iflib_device_detach(device_t dev)		iflib_device_detach(device_t dev)
{		{
if_ctx_t ctx = device_get_softc(dev);		if_ctx_t ctx = device_get_softc(dev);

return (iflib_device_deregister(ctx));		return (iflib_device_deregister(ctx));
}		}

▲ Show 20 Lines • Show All 154 Lines • ▼ Show 20 Lines	iflib_register(if_ctx_t ctx)
driver_t *driver = sctx->isc_driver;		driver_t *driver = sctx->isc_driver;
device_t dev = ctx->ifc_dev;		device_t dev = ctx->ifc_dev;
if_t ifp;		if_t ifp;

_iflib_assert(sctx);		_iflib_assert(sctx);

CTX_LOCK_INIT(ctx, device_get_nameunit(ctx->ifc_dev));		CTX_LOCK_INIT(ctx, device_get_nameunit(ctx->ifc_dev));

ifp = ctx->ifc_ifp = if_gethandle(IFT_ETHER);		ifp = ctx->ifc_ifp = if_alloc(IFT_ETHER);
if (ifp == NULL) {		if (ifp == NULL) {
device_printf(dev, "can not allocate ifnet structure\n");		device_printf(dev, "can not allocate ifnet structure\n");
return (ENOMEM);		return (ENOMEM);
}		}

/*		/*
* Initialize our context's device specific methods		* Initialize our context's device specific methods
*/		*/
▲ Show 20 Lines • Show All 161 Lines • ▼ Show 20 Lines	for (rxconf = i = 0; i < nrxqsets; i++, rxconf++, rxq++) {
}		}
rxq->ifr_fl = fl;		rxq->ifr_fl = fl;
for (j = 0; j < nfree_lists; j++) {		for (j = 0; j < nfree_lists; j++) {
fl[j].ifl_rxq = rxq;		fl[j].ifl_rxq = rxq;
fl[j].ifl_id = j;		fl[j].ifl_id = j;
fl[j].ifl_ifdi = &rxq->ifr_ifdi[j + rxq->ifr_fl_offset];		fl[j].ifl_ifdi = &rxq->ifr_ifdi[j + rxq->ifr_fl_offset];
fl[j].ifl_rxd_size = scctx->isc_rxd_size[j];		fl[j].ifl_rxd_size = scctx->isc_rxd_size[j];
}		}
/* Allocate receive buffers for the ring*/		/* Allocate receive buffers for the ring*/
if (iflib_rxsd_alloc(rxq)) {		if (iflib_rxsd_alloc(rxq)) {
device_printf(dev,		device_printf(dev,
"Critical Failure setting up receive buffers\n");		"Critical Failure setting up receive buffers\n");
err = ENOMEM;		err = ENOMEM;
goto err_rx_desc;		goto err_rx_desc;
}		}

for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++)		for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++)
▲ Show 20 Lines • Show All 138 Lines • ▼ Show 20 Lines
static void		static void
iflib_rx_structures_free(if_ctx_t ctx)		iflib_rx_structures_free(if_ctx_t ctx)
{		{
iflib_rxq_t rxq = ctx->ifc_rxqs;		iflib_rxq_t rxq = ctx->ifc_rxqs;

for (int i = 0; i < ctx->ifc_softc_ctx.isc_nrxqsets; i++, rxq++) {		for (int i = 0; i < ctx->ifc_softc_ctx.isc_nrxqsets; i++, rxq++) {
iflib_rx_sds_free(rxq);		iflib_rx_sds_free(rxq);
}		}
		free(ctx->ifc_rxqs, M_IFLIB);
		ctx->ifc_rxqs = NULL;
}		}

static int		static int
iflib_qset_structures_setup(if_ctx_t ctx)		iflib_qset_structures_setup(if_ctx_t ctx)
{		{
int err;		int err;

/*		/*
▲ Show 20 Lines • Show All 243 Lines • ▼ Show 20 Lines	iflib_irq_alloc_generic(if_ctx_t ctx, if_irq_t irq, int rid,
} else {		} else {
taskqgroup_attach(tqg, gtask, q, rman_get_start(irq->ii_res), name);		taskqgroup_attach(tqg, gtask, q, rman_get_start(irq->ii_res), name);
}		}

return (0);		return (0);
}		}

void		void
iflib_softirq_alloc_generic(if_ctx_t ctx, if_irq_t irq, iflib_intr_type_t type, void arg, int qid, char name)		iflib_softirq_alloc_generic(if_ctx_t ctx, if_irq_t irq, iflib_intr_type_t type, void arg, int qid, char name)
{		{
struct grouptask *gtask;		struct grouptask *gtask;
struct taskqgroup *tqg;		struct taskqgroup *tqg;
gtask_fn_t *fn;		gtask_fn_t *fn;
void *q;		void *q;
int irq_num = -1;		int irq_num = -1;
int err;		int err;

▲ Show 20 Lines • Show All 148 Lines • ▼ Show 20 Lines

void		void
iflib_link_state_change(if_ctx_t ctx, int link_state, uint64_t baudrate)		iflib_link_state_change(if_ctx_t ctx, int link_state, uint64_t baudrate)
{		{
if_t ifp = ctx->ifc_ifp;		if_t ifp = ctx->ifc_ifp;
iflib_txq_t txq = ctx->ifc_txqs;		iflib_txq_t txq = ctx->ifc_txqs;

if_setbaudrate(ifp, baudrate);		if_setbaudrate(ifp, baudrate);
if (baudrate >= IF_Gbps(10))		if (baudrate >= IF_Gbps(10)) {
		STATE_LOCK(ctx);
ctx->ifc_flags \|= IFC_PREFETCH;		ctx->ifc_flags \|= IFC_PREFETCH;
		STATE_UNLOCK(ctx);
		}
/* If link down, disable watchdog */		/* If link down, disable watchdog */
if ((ctx->ifc_link_state == LINK_STATE_UP) && (link_state == LINK_STATE_DOWN)) {		if ((ctx->ifc_link_state == LINK_STATE_UP) && (link_state == LINK_STATE_DOWN)) {
for (int i = 0; i < ctx->ifc_softc_ctx.isc_ntxqsets; i++, txq++)		for (int i = 0; i < ctx->ifc_softc_ctx.isc_ntxqsets; i++, txq++)
txq->ift_qstatus = IFLIB_QUEUE_IDLE;		txq->ift_qstatus = IFLIB_QUEUE_IDLE;
}		}
ctx->ifc_link_state = link_state;		ctx->ifc_link_state = link_state;
if_link_state_change(ifp, link_state);		if_link_state_change(ifp, link_state);
}		}
▲ Show 20 Lines • Show All 42 Lines • ▼ Show 20 Lines	SYSCTL_ADD_PROC(device_get_sysctl_ctx(ctx->ifc_dev),
OID_AUTO, name, CTLTYPE_INT\|CTLFLAG_RW,		OID_AUTO, name, CTLTYPE_INT\|CTLFLAG_RW,
info, 0, iflib_sysctl_int_delay, "I", description);		info, 0, iflib_sysctl_int_delay, "I", description);
}		}

struct mtx *		struct mtx *
iflib_ctx_lock_get(if_ctx_t ctx)		iflib_ctx_lock_get(if_ctx_t ctx)
{		{

return (&ctx->ifc_mtx);		return (&ctx->ifc_ctx_mtx);
}		}

static int		static int
iflib_msix_init(if_ctx_t ctx)		iflib_msix_init(if_ctx_t ctx)
{		{
device_t dev = ctx->ifc_dev;		device_t dev = ctx->ifc_dev;
if_shared_ctx_t sctx = ctx->ifc_sctx;		if_shared_ctx_t sctx = ctx->ifc_sctx;
if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;		if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
▲ Show 20 Lines • Show All 110 Lines • ▼ Show 20 Lines	else
tx_queues = mp_ncpus;		tx_queues = mp_ncpus;

if (tx_queues > scctx->isc_ntxqsets)		if (tx_queues > scctx->isc_ntxqsets)
tx_queues = scctx->isc_ntxqsets;		tx_queues = scctx->isc_ntxqsets;

if (ctx->ifc_sysctl_qs_eq_override == 0) {		if (ctx->ifc_sysctl_qs_eq_override == 0) {
#ifdef INVARIANTS		#ifdef INVARIANTS
if (tx_queues != rx_queues)		if (tx_queues != rx_queues)
device_printf(dev, "queue equality override not set, capping rx_queues at %d and tx_queues at %d\n",		device_printf(dev,
		"queue equality override not set, capping rx_queues at %d and tx_queues at %d\n",
min(rx_queues, tx_queues), min(rx_queues, tx_queues));		min(rx_queues, tx_queues), min(rx_queues, tx_queues));
#endif		#endif
tx_queues = min(rx_queues, tx_queues);		tx_queues = min(rx_queues, tx_queues);
rx_queues = min(rx_queues, tx_queues);		rx_queues = min(rx_queues, tx_queues);
}		}

device_printf(dev, "using %d rx queues %d tx queues \n", rx_queues, tx_queues);		device_printf(dev, "using %d rx queues %d tx queues \n", rx_queues, tx_queues);

vectors = rx_queues + admincnt;		vectors = rx_queues + admincnt;
if ((err = pci_alloc_msix(dev, &vectors)) == 0) {		if ((err = pci_alloc_msix(dev, &vectors)) == 0) {
device_printf(dev,		device_printf(dev, "Using MSI-X interrupts with %d vectors\n",
"Using MSIX interrupts with %d vectors\n", vectors);		vectors);
scctx->isc_vectors = vectors;		scctx->isc_vectors = vectors;
scctx->isc_nrxqsets = rx_queues;		scctx->isc_nrxqsets = rx_queues;
scctx->isc_ntxqsets = tx_queues;		scctx->isc_ntxqsets = tx_queues;
scctx->isc_intr = IFLIB_INTR_MSIX;		scctx->isc_intr = IFLIB_INTR_MSIX;

return (vectors);		return (vectors);
} else {		} else {
device_printf(dev, "failed to allocate %d msix vectors, err: %d - using MSI\n", vectors, err);		device_printf(dev,
		"failed to allocate %d msix vectors, err: %d - using MSI\n", vectors, err);
}		}
msi:		msi:
vectors = pci_msi_count(dev);		vectors = pci_msi_count(dev);
scctx->isc_nrxqsets = 1;		scctx->isc_nrxqsets = 1;
scctx->isc_ntxqsets = 1;		scctx->isc_ntxqsets = 1;
scctx->isc_vectors = vectors;		scctx->isc_vectors = vectors;
if (vectors == 1 && pci_alloc_msi(dev, &vectors) == 0) {		if (vectors == 1 && pci_alloc_msi(dev, &vectors) == 0) {
device_printf(dev,"Using an MSI interrupt\n");		device_printf(dev,"Using an MSI interrupt\n");
▲ Show 20 Lines • Show All 283 Lines • ▼ Show 20 Lines	#if MEMORY_LOGGING
SYSCTL_ADD_QUAD(ctx_list, fl_list, OID_AUTO, "fl_cl_dequeued",		SYSCTL_ADD_QUAD(ctx_list, fl_list, OID_AUTO, "fl_cl_dequeued",
CTLFLAG_RD,		CTLFLAG_RD,
&fl->ifl_cl_dequeued, "clusters freed");		&fl->ifl_cl_dequeued, "clusters freed");
#endif		#endif

}		}
}		}

		}

		void
		iflib_request_reset(if_ctx_t ctx)
		{

		STATE_LOCK(ctx);
		ctx->ifc_flags \|= IFC_DO_RESET;
		STATE_UNLOCK(ctx);
}		}

#ifndef __NO_STRICT_ALIGNMENT		#ifndef __NO_STRICT_ALIGNMENT
static struct mbuf *		static struct mbuf *
iflib_fixup_rx(struct mbuf *m)		iflib_fixup_rx(struct mbuf *m)
{		{
struct mbuf *n;		struct mbuf *n;

Show All 20 Lines