Index: sys/amd64/conf/GENERIC =================================================================== --- sys/amd64/conf/GENERIC +++ sys/amd64/conf/GENERIC @@ -233,8 +233,8 @@ device ix # Intel PRO/10GbE PCIE PF Ethernet device ixv # Intel PRO/10GbE PCIE VF Ethernet device ixl # Intel XL710 40Gbe PCIE Ethernet -options IXL_IW # Enable iWARP Client Interface in ixl(4) -device ixlv # Intel XL710 40Gbe VF PCIE Ethernet +#options IXL_IW # Enable iWARP Client Interface in ixl(4) +#device ixlv # Intel XL710 40Gbe VF PCIE Ethernet device le # AMD Am7900 LANCE and Am79C9xx PCnet device ti # Alteon Networks Tigon I/II gigabit Ethernet device txp # 3Com 3cR990 (``Typhoon'') Index: sys/conf/files.amd64 =================================================================== --- sys/conf/files.amd64 +++ sys/conf/files.amd64 @@ -247,35 +247,35 @@ dev/ipmi/ipmi_ssif.c optional ipmi smbus dev/ipmi/ipmi_pci.c optional ipmi pci dev/ipmi/ipmi_linux.c optional ipmi compat_linux32 -dev/ixl/if_ixl.c optional ixl pci \ +dev/ixl/if_ixl.c optional ixl \ compile-with "${NORMAL_C} -I$S/dev/ixl" -dev/ixl/ixl_pf_main.c optional ixl pci \ +dev/ixl/ixl_pf_main.c optional ixl \ compile-with "${NORMAL_C} -I$S/dev/ixl" -dev/ixl/ixl_pf_qmgr.c optional ixl pci \ +dev/ixl/ixl_pf_qmgr.c optional ixl \ compile-with "${NORMAL_C} -I$S/dev/ixl" -dev/ixl/ixl_pf_iov.c optional ixl pci \ +dev/ixl/ixl_pf_iov.c optional ixl \ compile-with "${NORMAL_C} -I$S/dev/ixl" -dev/ixl/ixl_pf_i2c.c optional ixl pci \ +dev/ixl/ixl_pf_i2c.c optional ixl \ compile-with "${NORMAL_C} -I$S/dev/ixl" -dev/ixl/ixl_iw.c optional ixl pci \ +dev/ixl/ixl_iw.c optional ixl \ compile-with "${NORMAL_C} -I$S/dev/ixl" -dev/ixl/if_ixlv.c optional ixlv pci \ +#dev/ixl/if_ixlv.c optional ixlv \ +# compile-with "${NORMAL_C} -I$S/dev/ixl" +#dev/ixl/ixlvc.c optional ixlv \ +# compile-with "${NORMAL_C} -I$S/dev/ixl" +dev/ixl/ixl_txrx.c optional ixl | ixlv \ compile-with "${NORMAL_C} -I$S/dev/ixl" -dev/ixl/ixlvc.c optional ixlv pci \ +dev/ixl/i40e_osdep.c optional ixl | ixlv \ compile-with "${NORMAL_C} -I$S/dev/ixl" -dev/ixl/ixl_txrx.c optional ixl pci | ixlv pci \ +dev/ixl/i40e_lan_hmc.c optional ixl | ixlv \ compile-with "${NORMAL_C} -I$S/dev/ixl" -dev/ixl/i40e_osdep.c optional ixl pci | ixlv pci \ +dev/ixl/i40e_hmc.c optional ixl | ixlv \ compile-with "${NORMAL_C} -I$S/dev/ixl" -dev/ixl/i40e_lan_hmc.c optional ixl pci | ixlv pci \ +dev/ixl/i40e_common.c optional ixl | ixlv \ compile-with "${NORMAL_C} -I$S/dev/ixl" -dev/ixl/i40e_hmc.c optional ixl pci | ixlv pci \ +dev/ixl/i40e_nvm.c optional ixl | ixlv \ compile-with "${NORMAL_C} -I$S/dev/ixl" -dev/ixl/i40e_common.c optional ixl pci | ixlv pci \ - compile-with "${NORMAL_C} -I$S/dev/ixl" -dev/ixl/i40e_nvm.c optional ixl pci | ixlv pci \ - compile-with "${NORMAL_C} -I$S/dev/ixl" -dev/ixl/i40e_adminq.c optional ixl pci | ixlv pci \ +dev/ixl/i40e_adminq.c optional ixl | ixlv \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/fdc/fdc.c optional fdc dev/fdc/fdc_acpi.c optional fdc Index: sys/dev/ixl/i40e_adminq.c =================================================================== --- sys/dev/ixl/i40e_adminq.c +++ sys/dev/ixl/i40e_adminq.c @@ -548,7 +548,10 @@ * i40e_resume_aq - resume AQ processing from 0 * @hw: pointer to the hardware structure **/ +#if 0 static void i40e_resume_aq(struct i40e_hw *hw) +#endif +void i40e_resume_aq(struct i40e_hw *hw) { /* Registers are reset after PF reset */ hw->aq.asq.next_to_use = 0; Index: sys/dev/ixl/i40e_osdep.h =================================================================== --- sys/dev/ixl/i40e_osdep.h +++ sys/dev/ixl/i40e_osdep.h @@ -146,15 +146,6 @@ #define prefetch(x) #endif -struct i40e_osdep { - bus_space_tag_t mem_bus_space_tag; - bus_space_handle_t mem_bus_space_handle; - bus_size_t mem_bus_space_size; - uint32_t flush_reg; - int i2c_intfc_num; - device_t dev; -}; - struct i40e_dma_mem { void *va; u64 pa; @@ -164,6 +155,19 @@ bus_size_t size; int nseg; int flags; + int type; +}; + +struct i40e_osdep { + // TODO: DEBUG -- have the HMC re-use this memory instead of destroying/creating a new one + struct i40e_dma_mem lan_hmc_mem; + + bus_space_tag_t mem_bus_space_tag; + bus_space_handle_t mem_bus_space_handle; + bus_size_t mem_bus_space_size; + uint32_t flush_reg; + int i2c_intfc_num; + device_t dev; }; struct i40e_virt_mem { Index: sys/dev/ixl/i40e_osdep.c =================================================================== --- sys/dev/ixl/i40e_osdep.c +++ sys/dev/ixl/i40e_osdep.c @@ -64,11 +64,27 @@ i40e_status i40e_allocate_dma_mem(struct i40e_hw *hw, struct i40e_dma_mem *mem, - enum i40e_memory_type type __unused, u64 size, u32 alignment) + enum i40e_memory_type type, u64 size, u32 alignment) { + struct i40e_osdep *osdep = ((struct i40e_osdep *)hw->back); device_t dev = ((struct i40e_osdep *)hw->back)->dev; int err; + device_printf(dev, "%s: type %d, size %lu\n", __func__, type, size); + + // DEBUG -- try out this special case + if (type == i40e_mem_bp_jumbo) { + device_printf(dev, "%s: - re-using memory for mem_bp_jumbo\n", __func__); + + bcopy(&osdep->lan_hmc_mem, mem, sizeof(*mem)); + + device_printf(dev, "%s: - mem->size is %lu\n", __func__, mem->size); + bzero(mem->va, mem->size); + + bus_dmamap_sync(mem->tag, mem->map, + BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); + return (I40E_SUCCESS); + } err = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ alignment, 0, /* alignment, bounds */ @@ -109,6 +125,7 @@ } mem->nseg = 1; mem->size = size; + mem->type = type; bus_dmamap_sync(mem->tag, mem->map, BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); return (0); @@ -125,6 +142,14 @@ i40e_status i40e_free_dma_mem(struct i40e_hw *hw, struct i40e_dma_mem *mem) { + device_t dev = ((struct i40e_osdep *)hw->back)->dev; + + if (mem->type == i40e_mem_bp_jumbo) { + device_printf(dev, "%s: not freeing memory for mem_bp_jumbo\n", __func__); + // bus_dmamap_unload(mem->tag, mem->map); + return (0); + } + bus_dmamap_sync(mem->tag, mem->map, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(mem->tag, mem->map); Index: sys/dev/ixl/i40e_prototype.h =================================================================== --- sys/dev/ixl/i40e_prototype.h +++ sys/dev/ixl/i40e_prototype.h @@ -45,7 +45,8 @@ * has happened and will assist in the early SW and FW * setup. */ - +// DEBUG +void i40e_resume_aq(struct i40e_hw *hw); /* adminq functions */ enum i40e_status_code i40e_init_adminq(struct i40e_hw *hw); enum i40e_status_code i40e_shutdown_adminq(struct i40e_hw *hw); Index: sys/dev/ixl/if_ixl.c =================================================================== --- sys/dev/ixl/if_ixl.c +++ sys/dev/ixl/if_ixl.c @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2013-2015, Intel Corporation + Copyright (c) 2013-2017, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -47,59 +47,76 @@ /********************************************************************* * Driver version *********************************************************************/ -char ixl_driver_version[] = "1.7.12-k"; +char ixl_driver_version[] = "1.7.12-iflib-k"; /********************************************************************* * PCI Device ID Table * * Used by probe to select devices to load on - * Last field stores an index into ixl_strings - * Last entry must be all 0s * - * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } + * ( Vendor ID, Device ID, Branding String ) *********************************************************************/ -static ixl_vendor_info_t ixl_vendor_info_array[] = -{ - {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_XL710, 0, 0, 0}, - {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_B, 0, 0, 0}, - {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_C, 0, 0, 0}, - {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_A, 0, 0, 0}, - {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_B, 0, 0, 0}, - {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_C, 0, 0, 0}, - {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T, 0, 0, 0}, - {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T4, 0, 0, 0}, - {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_X722, 0, 0, 0}, - {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_X722, 0, 0, 0}, - {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_X722, 0, 0, 0}, - {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_1G_BASE_T_X722, 0, 0, 0}, - {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T_X722, 0, 0, 0}, - {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_I_X722, 0, 0, 0}, - {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_25G_B, 0, 0, 0}, - {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_25G_SFP28, 0, 0, 0}, +static pci_vendor_info_t ixl_vendor_info_array[] = +{ + PVID(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_XL710, "Intel(R) Ethernet Connection 700 Series Driver"), + PVID(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_B, "Intel(R) Ethernet Connection 700 Series Driver"), + PVID(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_C, "Intel(R) Ethernet Connection 700 Series Driver"), + PVID(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_A, "Intel(R) Ethernet Connection 700 Series Driver"), + PVID(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_B, "Intel(R) Ethernet Connection 700 Series Driver"), + PVID(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_C, "Intel(R) Ethernet Connection 700 Series Driver"), + PVID(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T, "Intel(R) Ethernet Connection 700 Series Driver"), + PVID(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T4, "Intel(R) Ethernet Connection 700 Series Driver"), + PVID(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_X722, "Intel(R) Ethernet Connection 700 Series Driver"), + PVID(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_X722, "Intel(R) Ethernet Connection 700 Series Driver"), + PVID(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_X722, "Intel(R) Ethernet Connection 700 Series Driver"), + PVID(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_1G_BASE_T_X722, "Intel(R) Ethernet Connection 700 Series Driver"), + PVID(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T_X722, "Intel(R) Ethernet Connection 700 Series Driver"), + PVID(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_I_X722, "Intel(R) Ethernet Connection 700 Series Driver"), + PVID(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_25G_B, "Intel(R) Ethernet Connection 700 Series Driver"), + PVID(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_25G_SFP28, "Intel(R) Ethernet Connection 700 Series Driver"), /* required last entry */ - {0, 0, 0, 0, 0} -}; - -/********************************************************************* - * Table of branding strings - *********************************************************************/ - -static char *ixl_strings[] = { - "Intel(R) Ethernet Connection XL710/X722 Driver" + PVID_END }; - /********************************************************************* * Function prototypes *********************************************************************/ -static int ixl_probe(device_t); -static int ixl_attach(device_t); -static int ixl_detach(device_t); -static int ixl_shutdown(device_t); -static int ixl_save_pf_tunables(struct ixl_pf *); -static int ixl_attach_get_link_status(struct ixl_pf *); +/*** IFLIB interface ***/ +static void *ixl_register(device_t dev); +static int ixl_if_attach_pre(if_ctx_t ctx); +static int ixl_if_attach_post(if_ctx_t ctx); +static int ixl_if_detach(if_ctx_t ctx); +static int ixl_if_shutdown(if_ctx_t ctx); +static int ixl_if_suspend(if_ctx_t ctx); +static int ixl_if_resume(if_ctx_t ctx); +static int ixl_if_msix_intr_assign(if_ctx_t ctx, int msix); +static void ixl_if_enable_intr(if_ctx_t ctx); +static void ixl_if_disable_intr(if_ctx_t ctx); +static int ixl_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid); +static int ixl_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid); +static int ixl_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int ntxqs, int ntxqsets); +static int ixl_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nqs, int nqsets); +static void ixl_if_queues_free(if_ctx_t ctx); +static void ixl_if_update_admin_status(if_ctx_t ctx); +static void ixl_if_multi_set(if_ctx_t ctx); +static int ixl_if_mtu_set(if_ctx_t ctx, uint32_t mtu); +static void ixl_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr); +static int ixl_if_media_change(if_ctx_t ctx); +static int ixl_if_promisc_set(if_ctx_t ctx, int flags); +static void ixl_if_timer(if_ctx_t ctx, uint16_t qid); +static void ixl_if_vlan_register(if_ctx_t ctx, u16 vtag); +static void ixl_if_vlan_unregister(if_ctx_t ctx, u16 vtag); +static uint64_t ixl_if_get_counter(if_ctx_t ctx, ift_counter cnt); +// static void ixl_if_link_intr_enable(if_ctx_t ctx); + +/*** Other ***/ +static int ixl_mc_filter_apply(void *arg, struct ifmultiaddr *ifma, int); +static int ixl_save_pf_tunables(struct ixl_pf *); +static int ixl_attach_get_link_status(struct ixl_pf *); +static int ixl_allocate_pci_resources(struct ixl_pf *); + /********************************************************************* * FreeBSD Device Interface Entry Points @@ -107,16 +124,17 @@ static device_method_t ixl_methods[] = { /* Device interface */ - DEVMETHOD(device_probe, ixl_probe), - DEVMETHOD(device_attach, ixl_attach), - DEVMETHOD(device_detach, ixl_detach), - DEVMETHOD(device_shutdown, ixl_shutdown), + DEVMETHOD(device_register, ixl_register), + DEVMETHOD(device_probe, iflib_device_probe), + DEVMETHOD(device_attach, iflib_device_attach), + DEVMETHOD(device_detach, iflib_device_detach), + DEVMETHOD(device_shutdown, iflib_device_shutdown), #ifdef PCI_IOV DEVMETHOD(pci_iov_init, ixl_iov_init), DEVMETHOD(pci_iov_uninit, ixl_iov_uninit), DEVMETHOD(pci_iov_add_vf, ixl_add_vf), #endif - {0, 0} + DEVMETHOD_END }; static driver_t ixl_driver = { @@ -126,17 +144,48 @@ devclass_t ixl_devclass; DRIVER_MODULE(ixl, pci, ixl_driver, ixl_devclass, 0, 0); -MODULE_VERSION(ixl, 1); - MODULE_DEPEND(ixl, pci, 1, 1, 1); MODULE_DEPEND(ixl, ether, 1, 1, 1); -#if defined(DEV_NETMAP) && __FreeBSD_version >= 1100000 -MODULE_DEPEND(ixl, netmap, 1, 1, 1); -#endif /* DEV_NETMAP */ +MODULE_DEPEND(ixl, iflib, 1, 1, 1); + +static device_method_t ixl_if_methods[] = { + DEVMETHOD(ifdi_attach_pre, ixl_if_attach_pre), + DEVMETHOD(ifdi_attach_post, ixl_if_attach_post), + DEVMETHOD(ifdi_detach, ixl_if_detach), + DEVMETHOD(ifdi_shutdown, ixl_if_shutdown), + DEVMETHOD(ifdi_suspend, ixl_if_suspend), + DEVMETHOD(ifdi_resume, ixl_if_resume), + DEVMETHOD(ifdi_init, ixl_if_init), + DEVMETHOD(ifdi_stop, ixl_if_stop), + DEVMETHOD(ifdi_msix_intr_assign, ixl_if_msix_intr_assign), + DEVMETHOD(ifdi_intr_enable, ixl_if_enable_intr), + DEVMETHOD(ifdi_intr_disable, ixl_if_disable_intr), + //DEVMETHOD(ifdi_link_intr_enable, ixl_if_link_intr_enable), + DEVMETHOD(ifdi_rx_queue_intr_enable, ixl_if_rx_queue_intr_enable), + DEVMETHOD(ifdi_tx_queue_intr_enable, ixl_if_tx_queue_intr_enable), + DEVMETHOD(ifdi_tx_queues_alloc, ixl_if_tx_queues_alloc), + DEVMETHOD(ifdi_rx_queues_alloc, ixl_if_rx_queues_alloc), + DEVMETHOD(ifdi_queues_free, ixl_if_queues_free), + DEVMETHOD(ifdi_update_admin_status, ixl_if_update_admin_status), + DEVMETHOD(ifdi_multi_set, ixl_if_multi_set), + DEVMETHOD(ifdi_mtu_set, ixl_if_mtu_set), + DEVMETHOD(ifdi_media_status, ixl_if_media_status), + DEVMETHOD(ifdi_media_change, ixl_if_media_change), + DEVMETHOD(ifdi_promisc_set, ixl_if_promisc_set), + DEVMETHOD(ifdi_timer, ixl_if_timer), + DEVMETHOD(ifdi_vlan_register, ixl_if_vlan_register), + DEVMETHOD(ifdi_vlan_unregister, ixl_if_vlan_unregister), + DEVMETHOD(ifdi_get_counter, ixl_if_get_counter), + DEVMETHOD_END +}; -/* +static driver_t ixl_if_driver = { + "ixl_if", ixl_if_methods, sizeof(struct ixl_pf) +}; + +/***************************************************************************** ** TUNEABLE PARAMETERS: -*/ +*****************************************************************************/ static SYSCTL_NODE(_hw, OID_AUTO, ixl, CTLFLAG_RD, 0, "IXL driver parameters"); @@ -159,6 +208,15 @@ SYSCTL_INT(_hw_ixl, OID_AUTO, ring_size, CTLFLAG_RDTUN, &ixl_ring_size, 0, "Descriptor Ring Size"); +/* +** Number of descriptors per ring: +** - TX and RX are the same size +*/ +static int ixl_enable_hwb = 0; +TUNABLE_INT("hw.ixl.enable_hwb", &ixl_enable_hwb); +SYSCTL_INT(_hw_ixl, OID_AUTO, enable_hwb, CTLFLAG_RDTUN, + &ixl_enable_hwb, 0, "Enable Head Writeback for TX Descriptors"); + /* ** This can be set manually, if left as 0 the ** number of queues will be calculated based @@ -195,211 +253,169 @@ ** - true/false for dynamic adjustment ** - default values for static ITR */ -static int ixl_dynamic_rx_itr = 1; +static int ixl_dynamic_rx_itr = 0; TUNABLE_INT("hw.ixl.dynamic_rx_itr", &ixl_dynamic_rx_itr); SYSCTL_INT(_hw_ixl, OID_AUTO, dynamic_rx_itr, CTLFLAG_RDTUN, &ixl_dynamic_rx_itr, 0, "Dynamic RX Interrupt Rate"); -static int ixl_dynamic_tx_itr = 1; -TUNABLE_INT("hw.ixl.dynamic_tx_itr", &ixl_dynamic_tx_itr); -SYSCTL_INT(_hw_ixl, OID_AUTO, dynamic_tx_itr, CTLFLAG_RDTUN, - &ixl_dynamic_tx_itr, 0, "Dynamic TX Interrupt Rate"); - static int ixl_rx_itr = IXL_ITR_8K; TUNABLE_INT("hw.ixl.rx_itr", &ixl_rx_itr); SYSCTL_INT(_hw_ixl, OID_AUTO, rx_itr, CTLFLAG_RDTUN, &ixl_rx_itr, 0, "RX Interrupt Rate"); -static int ixl_tx_itr = IXL_ITR_4K; -TUNABLE_INT("hw.ixl.tx_itr", &ixl_tx_itr); -SYSCTL_INT(_hw_ixl, OID_AUTO, tx_itr, CTLFLAG_RDTUN, - &ixl_tx_itr, 0, "TX Interrupt Rate"); - #ifdef IXL_IW int ixl_enable_iwarp = 0; TUNABLE_INT("hw.ixl.enable_iwarp", &ixl_enable_iwarp); #endif -#ifdef DEV_NETMAP -#define NETMAP_IXL_MAIN /* only bring in one part of the netmap code */ -#include -#endif /* DEV_NETMAP */ - -/********************************************************************* - * Device identification routine - * - * ixl_probe determines if the driver should be loaded on - * the hardware based on PCI vendor/device id of the device. - * - * return BUS_PROBE_DEFAULT on success, positive on failure - *********************************************************************/ - -static int -ixl_probe(device_t dev) -{ - ixl_vendor_info_t *ent; - - u16 pci_vendor_id, pci_device_id; - u16 pci_subvendor_id, pci_subdevice_id; - char device_name[256]; +extern struct if_txrx ixl_txrx; + +static struct if_shared_ctx ixl_sctx_init = { + .isc_magic = IFLIB_MAGIC, + .isc_q_align = PAGE_SIZE,/* max(DBA_ALIGN, PAGE_SIZE) */ + .isc_tx_maxsize = IXL_TSO_SIZE, + + .isc_tx_maxsegsize = PAGE_SIZE, + + // TODO: Review the rx_maxsize and rx_maxsegsize params + // Where are they used in iflib? + .isc_rx_maxsize = 16384, + .isc_rx_nsegments = 1, // XXX: This is probably 5 + .isc_rx_maxsegsize = 16384, + // TODO: What is isc_nfl for? + .isc_nfl = 1, + .isc_ntxqs = 1, + .isc_nrxqs = 1, + + .isc_admin_intrcnt = 1, + .isc_vendor_info = ixl_vendor_info_array, + .isc_driver_version = ixl_driver_version, + .isc_driver = &ixl_if_driver, + + .isc_nrxd_min = {IXL_MIN_RING}, + .isc_ntxd_min = {IXL_MIN_RING}, + .isc_nrxd_max = {IXL_MAX_RING}, + .isc_ntxd_max = {IXL_MAX_RING}, + .isc_nrxd_default = {IXL_DEFAULT_RING}, + .isc_ntxd_default = {IXL_DEFAULT_RING}, +}; -#if 0 - INIT_DEBUGOUT("ixl_probe: begin"); -#endif - pci_vendor_id = pci_get_vendor(dev); - if (pci_vendor_id != I40E_INTEL_VENDOR_ID) - return (ENXIO); +if_shared_ctx_t ixl_sctx = &ixl_sctx_init; - pci_device_id = pci_get_device(dev); - pci_subvendor_id = pci_get_subvendor(dev); - pci_subdevice_id = pci_get_subdevice(dev); - - ent = ixl_vendor_info_array; - while (ent->vendor_id != 0) { - if ((pci_vendor_id == ent->vendor_id) && - (pci_device_id == ent->device_id) && - - ((pci_subvendor_id == ent->subvendor_id) || - (ent->subvendor_id == 0)) && - - ((pci_subdevice_id == ent->subdevice_id) || - (ent->subdevice_id == 0))) { - sprintf(device_name, "%s, Version - %s", - ixl_strings[ent->index], - ixl_driver_version); - device_set_desc_copy(dev, device_name); - return (BUS_PROBE_DEFAULT); - } - ent++; - } - return (ENXIO); -} +/*** Functions ***/ -static int -ixl_attach_get_link_status(struct ixl_pf *pf) +static void * +ixl_register(device_t dev) { - struct i40e_hw *hw = &pf->hw; - device_t dev = pf->dev; - int error = 0; - - if (((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver < 33)) || - (hw->aq.fw_maj_ver < 4)) { - i40e_msec_delay(75); - error = i40e_aq_set_link_restart_an(hw, TRUE, NULL); - if (error) { - device_printf(dev, "link restart failed, aq_err=%d\n", - pf->hw.aq.asq_last_status); - return error; - } - } - - /* Determine link state */ - hw->phy.get_link_info = TRUE; - i40e_get_link_status(hw, &pf->link_up); - return (0); + return (ixl_sctx); } -/* - * Sanity check and save off tunable values. - */ -static int -ixl_save_pf_tunables(struct ixl_pf *pf) +int +ixl_allocate_pci_resources(struct ixl_pf *pf) { - device_t dev = pf->dev; - - /* Save tunable information */ - pf->enable_msix = ixl_enable_msix; - pf->max_queues = ixl_max_queues; - pf->enable_tx_fc_filter = ixl_enable_tx_fc_filter; - pf->dynamic_rx_itr = ixl_dynamic_rx_itr; - pf->dynamic_tx_itr = ixl_dynamic_tx_itr; - pf->dbg_mask = ixl_core_debug_mask; - pf->hw.debug_mask = ixl_shared_debug_mask; + int rid; + struct i40e_hw *hw = &pf->hw; + device_t dev = iflib_get_dev(pf->vsi.ctx); - if (ixl_ring_size < IXL_MIN_RING - || ixl_ring_size > IXL_MAX_RING - || ixl_ring_size % IXL_RING_INCREMENT != 0) { - device_printf(dev, "Invalid ring_size value of %d set!\n", - ixl_ring_size); - device_printf(dev, "ring_size must be between %d and %d, " - "inclusive, and must be a multiple of %d\n", - IXL_MIN_RING, IXL_MAX_RING, IXL_RING_INCREMENT); - device_printf(dev, "Using default value of %d instead\n", - IXL_DEFAULT_RING); - pf->ringsz = IXL_DEFAULT_RING; - } else - pf->ringsz = ixl_ring_size; + /* Map BAR0 */ + rid = PCIR_BAR(0); + pf->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, + &rid, RF_ACTIVE); - if (ixl_tx_itr < 0 || ixl_tx_itr > IXL_MAX_ITR) { - device_printf(dev, "Invalid tx_itr value of %d set!\n", - ixl_tx_itr); - device_printf(dev, "tx_itr must be between %d and %d, " - "inclusive\n", - 0, IXL_MAX_ITR); - device_printf(dev, "Using default value of %d instead\n", - IXL_ITR_4K); - pf->tx_itr = IXL_ITR_4K; - } else - pf->tx_itr = ixl_tx_itr; + if (!(pf->pci_mem)) { + device_printf(dev, "Unable to allocate bus resource: PCI memory\n"); + return (ENXIO); + } - if (ixl_rx_itr < 0 || ixl_rx_itr > IXL_MAX_ITR) { - device_printf(dev, "Invalid rx_itr value of %d set!\n", - ixl_rx_itr); - device_printf(dev, "rx_itr must be between %d and %d, " - "inclusive\n", - 0, IXL_MAX_ITR); - device_printf(dev, "Using default value of %d instead\n", - IXL_ITR_8K); - pf->rx_itr = IXL_ITR_8K; - } else - pf->rx_itr = ixl_rx_itr; + /* Save off the PCI information */ + hw->vendor_id = pci_get_vendor(dev); + hw->device_id = pci_get_device(dev); + hw->revision_id = pci_read_config(dev, PCIR_REVID, 1); + hw->subsystem_vendor_id = + pci_read_config(dev, PCIR_SUBVEND_0, 2); + hw->subsystem_device_id = + pci_read_config(dev, PCIR_SUBDEV_0, 2); + + hw->bus.device = pci_get_slot(dev); + hw->bus.func = pci_get_function(dev); + + /* Save off register access information */ + pf->osdep.mem_bus_space_tag = + rman_get_bustag(pf->pci_mem); + pf->osdep.mem_bus_space_handle = + rman_get_bushandle(pf->pci_mem); + pf->osdep.mem_bus_space_size = rman_get_size(pf->pci_mem); + pf->osdep.flush_reg = I40E_GLGEN_STAT; + pf->osdep.dev = dev; + + pf->hw.hw_addr = (u8 *) &pf->osdep.mem_bus_space_handle; + pf->hw.back = &pf->osdep; return (0); } -/********************************************************************* - * Device initialization routine - * - * The attach entry point is called when the driver is being loaded. - * This routine identifies the type of hardware, allocates all resources - * and initializes the hardware. - * - * return 0 on success, positive on failure - *********************************************************************/ - static int -ixl_attach(device_t dev) +ixl_if_attach_pre(if_ctx_t ctx) { - struct ixl_pf *pf; - struct i40e_hw *hw; - struct ixl_vsi *vsi; + device_t dev; + struct ixl_pf *pf; + struct i40e_hw *hw; + struct ixl_vsi *vsi; + if_softc_ctx_t scctx; + struct i40e_filter_control_settings filter; enum i40e_status_code status; - int error = 0; + int error = 0; - INIT_DEBUGOUT("ixl_attach: begin"); + INIT_DEBUGOUT("ixl_if_attach_pre: begin"); - /* Allocate, clear, and link in our primary soft structure */ - pf = device_get_softc(dev); - pf->dev = pf->osdep.dev = dev; - hw = &pf->hw; + dev = iflib_get_dev(ctx); + pf = iflib_get_softc(ctx); + hw = &pf->hw; /* ** Note this assumes we have a single embedded VSI, ** this could be enhanced later to allocate multiple */ vsi = &pf->vsi; - vsi->dev = pf->dev; + vsi->back = pf; + vsi->hw = &pf->hw; + vsi->id = 0; + vsi->num_vlans = 0; + vsi->ctx = ctx; + vsi->media = iflib_get_media(ctx); + vsi->shared = scctx = iflib_get_softc_ctx(ctx); + pf->dev = dev; + + /* + * These are the same across all current ixl models + */ + vsi->shared->isc_tx_nsegments = IXL_MAX_TX_SEGS; + vsi->shared->isc_msix_bar = PCIR_BAR(IXL_MSIX_BAR); + + vsi->shared->isc_tx_tso_segments_max = IXL_MAX_TSO_SEGS; + vsi->shared->isc_tx_tso_size_max = IXL_TSO_SIZE; + vsi->shared->isc_tx_tso_segsize_max = PAGE_SIZE; /* Save tunable values */ error = ixl_save_pf_tunables(pf); if (error) return (error); - /* Core Lock Init*/ - IXL_PF_LOCK_INIT(pf, device_get_nameunit(dev)); - - /* Set up the timer callout */ - callout_init_mtx(&pf->timer, &pf->pf_mtx, 0); + /* + * TODO: Excoriate mmacy for not documenting what needs to be set in the iflib stuff + * in attach_pre() + * Or, in general... + */ + /* + scctx->isc_txqsizes[0] = roundup2(scctx->isc_ntxd[0] + * sizeof(struct i40e_tx_desc) + sizeof(u32), DBA_ALIGN); + */ + scctx->isc_txqsizes[0] = roundup2(scctx->isc_ntxd[0] + * sizeof(struct i40e_tx_desc), DBA_ALIGN); + scctx->isc_rxqsizes[0] = roundup2(scctx->isc_nrxd[0] + * sizeof(union i40e_32byte_rx_desc), DBA_ALIGN); + /* XXX: No idea what this does */ + scctx->isc_max_txqsets = scctx->isc_max_rxqsets = 32; /* Do PCI setup - map BAR0, etc */ if (ixl_allocate_pci_resources(pf)) { @@ -427,18 +443,13 @@ goto err_out; } - /* - * Allocate interrupts and figure out number of queues to use - * for PF interface - */ - pf->msix = ixl_init_msix(pf); - - /* Set up the admin queue */ + /* Set admin queue parameters */ hw->aq.num_arq_entries = IXL_AQ_LEN; hw->aq.num_asq_entries = IXL_AQ_LEN; hw->aq.arq_buf_size = IXL_AQ_BUF_SZ; hw->aq.asq_buf_size = IXL_AQ_BUF_SZ; + /* Set up the admin queue */ status = i40e_init_adminq(hw); if (status != 0 && status != I40E_ERR_FIRMWARE_API_VERSION) { device_printf(dev, "Unable to initialize Admin Queue, error %s\n", @@ -474,9 +485,20 @@ /* Get capabilities from the device */ error = ixl_get_hw_capabilities(pf); if (error) { - device_printf(dev, "HW capabilities failure!\n"); + device_printf(dev, "get_hw_capabilities failed: %d\n", + error); + goto err_get_cap; + } + + /* DEBUG/TODO: Set up re-usable DMA block for lan hmc */ + status = i40e_allocate_dma_mem(hw, &((pf->osdep).lan_hmc_mem), 0, 122880, + I40E_HMC_PD_BP_BUF_ALIGNMENT); + if (status) { + device_printf(dev, "dma pre-alloc for LAN HMC failed: %s\n", + i40e_stat_str(hw, status)); goto err_get_cap; } + pf->osdep.lan_hmc_mem.type = i40e_mem_bp_jumbo; /* Set up host memory cache */ status = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, @@ -486,7 +508,6 @@ i40e_stat_str(hw, status)); goto err_get_cap; } - status = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); if (status) { device_printf(dev, "configure_lan_hmc failed: %s\n", @@ -494,23 +515,6 @@ goto err_mac_hmc; } - /* Init queue allocation manager */ - error = ixl_pf_qmgr_init(&pf->qmgr, hw->func_caps.num_tx_qp); - if (error) { - device_printf(dev, "Failed to init queue manager for PF queues, error %d\n", - error); - goto err_mac_hmc; - } - /* reserve a contiguous allocation for the PF's VSI */ - error = ixl_pf_qmgr_alloc_contiguous(&pf->qmgr, vsi->num_queues, &pf->qtag); - if (error) { - device_printf(dev, "Failed to reserve queues for PF LAN VSI, error %d\n", - error); - goto err_mac_hmc; - } - device_printf(dev, "Allocating %d queues for PF LAN VSI; %d queues active\n", - pf->qtag.num_allocated, pf->qtag.num_active); - /* Disable LLDP from the firmware for certain NVM versions */ if (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 3)) || (pf->hw.aq.fw_maj_ver < 4)) @@ -524,17 +528,58 @@ goto err_mac_hmc; } bcopy(hw->mac.addr, hw->mac.perm_addr, ETHER_ADDR_LEN); + iflib_set_mac(ctx, hw->mac.addr); i40e_get_port_mac_addr(hw, hw->mac.port_addr); + /* Set up the device filtering */ + bzero(&filter, sizeof(filter)); + filter.enable_ethtype = TRUE; + filter.enable_macvlan = TRUE; + filter.enable_fdir = FALSE; + filter.hash_lut_size = I40E_HASH_LUT_SIZE_512; + if (i40e_set_filter_control(hw, &filter)) + device_printf(dev, "i40e_set_filter_control() failed\n"); + /* Initialize mac filter list for VSI */ SLIST_INIT(&vsi->ftl); - /* Set up SW VSI and allocate queue memory and rings */ - if (ixl_setup_stations(pf)) { - device_printf(dev, "setup stations failed!\n"); - error = ENOMEM; - goto err_mac_hmc; - } + /* Fill out more iflib parameters */ + scctx->isc_txrx = &ixl_txrx; + vsi->shared->isc_rss_table_size = pf->hw.func_caps.rss_table_size; + scctx->isc_tx_csum_flags = CSUM_OFFLOAD; + scctx->isc_capenable = IXL_CAPS; + + INIT_DEBUGOUT("ixl_if_attach_pre: end"); + return (0); + +// TODO: Review what needs to be cleaned up when this fails +err_mac_hmc: + i40e_shutdown_lan_hmc(hw); +err_get_cap: + i40e_shutdown_adminq(hw); +err_out: + ixl_free_pci_resources(pf); + ixl_free_mac_filters(vsi); + return (error); +} + +static int +ixl_if_attach_post(if_ctx_t ctx) +{ + device_t dev; + struct ixl_pf *pf; + struct i40e_hw *hw; + struct ixl_vsi *vsi; + int error = 0; + enum i40e_status_code status; + + INIT_DEBUGOUT("ixl_if_attach_post: begin"); + + dev = iflib_get_dev(ctx); + vsi = iflib_get_softc(ctx); + vsi->ifp = iflib_get_ifp(ctx); + pf = (struct ixl_pf *)vsi->back; + hw = &pf->hw; /* Setup OS network interface / ifnet */ if (ixl_setup_interface(dev, vsi)) { @@ -556,6 +601,27 @@ goto err_late; } + /* Init queue allocation manager */ + /* XXX: This init can go in pre or post; allocation must be in post */ + error = ixl_pf_qmgr_init(&pf->qmgr, hw->func_caps.num_tx_qp); + if (error) { + device_printf(dev, "Failed to init queue manager for PF queues, error %d\n", + error); + goto err_mac_hmc; + } + /* reserve a contiguous allocation for the PF's VSI */ + /* TODO: Could be refined? */ + error = ixl_pf_qmgr_alloc_contiguous(&pf->qmgr, + max(vsi->num_tx_queues, vsi->num_rx_queues), &pf->qtag); + if (error) { + device_printf(dev, "Failed to reserve queues for PF LAN VSI, error %d\n", + error); + goto err_mac_hmc; + } + device_printf(dev, "Allocating %d queues for PF LAN VSI; %d queues active\n", + pf->qtag.num_allocated, pf->qtag.num_active); + + /* Limit PHY interrupts to link, autoneg, and modules failure */ status = i40e_aq_set_phy_int_mask(hw, IXL_DEFAULT_PHY_INT_MASK, NULL); @@ -566,56 +632,13 @@ goto err_late; } - /* Get the bus configuration and set the shared code's config */ + /* Get the bus configuration and set the shared code */ ixl_get_bus_info(pf); - /* - * In MSI-X mode, initialize the Admin Queue interrupt, - * so userland tools can communicate with the adapter regardless of - * the ifnet interface's status. - */ - if (pf->msix > 1) { - error = ixl_setup_adminq_msix(pf); - if (error) { - device_printf(dev, "ixl_setup_adminq_msix() error: %d\n", - error); - goto err_late; - } - error = ixl_setup_adminq_tq(pf); - if (error) { - device_printf(dev, "ixl_setup_adminq_tq() error: %d\n", - error); - goto err_late; - } + /* Keep admin queue interrupts active while driver is loaded */ + if (vsi->shared->isc_intr == IFLIB_INTR_MSIX) { ixl_configure_intr0_msix(pf); ixl_enable_intr0(hw); - - error = ixl_setup_queue_msix(vsi); - if (error) - device_printf(dev, "ixl_setup_queue_msix() error: %d\n", - error); - error = ixl_setup_queue_tqs(vsi); - if (error) - device_printf(dev, "ixl_setup_queue_tqs() error: %d\n", - error); - } else { - error = ixl_setup_legacy(pf); - - error = ixl_setup_adminq_tq(pf); - if (error) { - device_printf(dev, "ixl_setup_adminq_tq() error: %d\n", - error); - goto err_late; - } - - error = ixl_setup_queue_tqs(vsi); - if (error) - device_printf(dev, "ixl_setup_queue_tqs() error: %d\n", - error); - } - - if (error) { - device_printf(dev, "interrupt setup error: %d\n", error); } /* Set initial advertised speed sysctl value */ @@ -623,25 +646,14 @@ /* Initialize statistics & add sysctls */ ixl_add_device_sysctls(pf); - ixl_pf_reset_stats(pf); ixl_update_stats_counters(pf); ixl_add_hw_stats(pf); - /* Register for VLAN events */ - vsi->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, - ixl_register_vlan, vsi, EVENTHANDLER_PRI_FIRST); - vsi->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, - ixl_unregister_vlan, vsi, EVENTHANDLER_PRI_FIRST); - #ifdef PCI_IOV ixl_initialize_sriov(pf); #endif - -#ifdef DEV_NETMAP - ixl_netmap_attach(vsi); -#endif /* DEV_NETMAP */ - + // TODO: Probably broken #ifdef IXL_IW if (hw->func_caps.iwarp && ixl_enable_iwarp) { pf->iw_enabled = (pf->iw_msix > 0) ? true : false; @@ -662,125 +674,1022 @@ } #endif - INIT_DEBUGOUT("ixl_attach: end"); + INIT_DBG_DEV(dev, "end"); return (0); +// TODO: Review what needs to be cleaned up when this fails err_late: - if (vsi->ifp != NULL) { - ether_ifdetach(vsi->ifp); - if_free(vsi->ifp); - } err_mac_hmc: i40e_shutdown_lan_hmc(hw); -err_get_cap: i40e_shutdown_adminq(hw); -err_out: ixl_free_pci_resources(pf); - ixl_free_vsi(vsi); - IXL_PF_LOCK_DESTROY(pf); + ixl_free_mac_filters(vsi); + INIT_DBG_DEV(dev, "end: error %d", error); return (error); } -/********************************************************************* - * Device removal routine - * - * The detach entry point is called when the driver is being removed. - * This routine stops the adapter and deallocates all the resources - * that were allocated for driver operation. - * - * return 0 on success, positive on failure - *********************************************************************/ - static int -ixl_detach(device_t dev) +ixl_if_detach(if_ctx_t ctx) { - struct ixl_pf *pf = device_get_softc(dev); + struct ixl_vsi *vsi = iflib_get_softc(ctx); + struct ixl_pf *pf = vsi->back; struct i40e_hw *hw = &pf->hw; - struct ixl_vsi *vsi = &pf->vsi; - enum i40e_status_code status; -#if defined(PCI_IOV) || defined(IXL_IW) + device_t dev = pf->dev; + i40e_status status; +#ifdef PCI_IOV int error; #endif - INIT_DEBUGOUT("ixl_detach: begin"); + INIT_DBG_DEV(dev, "begin"); - /* Make sure VLANS are not using driver */ - if (vsi->ifp->if_vlantrunk != NULL) { - device_printf(dev, "Vlan in use, detach first\n"); - return (EBUSY); +#ifdef IXL_IW + if (ixl_enable_iwarp && pf->iw_enabled) { + error = ixl_iw_pf_detach(pf); + if (error == EBUSY) { + device_printf(dev, "iwarp in use; stop it first.\n"); + return (error); + } } - +#endif #ifdef PCI_IOV - error = pci_iov_detach(dev); + error = pci_iov_detach(iflib_get_dev(ctx)); if (error != 0) { - device_printf(dev, "SR-IOV in use; detach first.\n"); + device_printf(iflib_get_dev(ctx), "SR-IOV in use; detach first.\n"); return (error); } #endif - ether_ifdetach(vsi->ifp); - if (vsi->ifp->if_drv_flags & IFF_DRV_RUNNING) - ixl_stop(pf); - /* Shutdown LAN HMC */ - status = i40e_shutdown_lan_hmc(hw); - if (status) - device_printf(dev, - "Shutdown LAN HMC failed with code %d\n", status); + if (hw->hmc.hmc_obj) { + status = i40e_shutdown_lan_hmc(hw); + if (status) + device_printf(dev, + "i40e_shutdown_lan_hmc() failed with status %s\n", + i40e_stat_str(hw, status)); + } + + // DEBUG/HACK/TODO + pf->osdep.lan_hmc_mem.type = 0; + i40e_free_dma_mem(hw, &(pf->osdep.lan_hmc_mem)); - /* Teardown LAN queue resources */ - ixl_teardown_queue_msix(vsi); - ixl_free_queue_tqs(vsi); /* Shutdown admin queue */ ixl_disable_intr0(hw); - ixl_teardown_adminq_msix(pf); - ixl_free_adminq_tq(pf); status = i40e_shutdown_adminq(hw); if (status) device_printf(dev, - "Shutdown Admin queue failed with code %d\n", status); - - /* Unregister VLAN events */ - if (vsi->vlan_attach != NULL) - EVENTHANDLER_DEREGISTER(vlan_config, vsi->vlan_attach); - if (vsi->vlan_detach != NULL) - EVENTHANDLER_DEREGISTER(vlan_unconfig, vsi->vlan_detach); - - callout_drain(&pf->timer); - -#ifdef IXL_IW - if (ixl_enable_iwarp && pf->iw_enabled) { - error = ixl_iw_pf_detach(pf); - if (error == EBUSY) { - device_printf(dev, "iwarp in use; stop it first.\n"); - return (error); - } - } -#endif + "i40e_shutdown_adminq() failed with status %s\n", + i40e_stat_str(hw, status)); -#ifdef DEV_NETMAP - netmap_detach(vsi->ifp); -#endif /* DEV_NETMAP */ ixl_pf_qmgr_destroy(&pf->qmgr); ixl_free_pci_resources(pf); - bus_generic_detach(dev); - if_free(vsi->ifp); - ixl_free_vsi(vsi); - IXL_PF_LOCK_DESTROY(pf); + ixl_free_mac_filters(vsi); + INIT_DBG_DEV(dev, "end"); return (0); } -/********************************************************************* - * - * Shutdown entry point - * - **********************************************************************/ - +/* TODO: Do shutdown-specific stuff here */ static int -ixl_shutdown(device_t dev) +ixl_if_shutdown(if_ctx_t ctx) { - struct ixl_pf *pf = device_get_softc(dev); - ixl_stop(pf); + int error = 0; + + INIT_DEBUGOUT("ixl_if_shutdown: begin"); + + /* TODO: Call ixl_if_stop()? */ + + /* TODO: Then setup low power mode */ + + return (error); +} + +static int +ixl_if_suspend(if_ctx_t ctx) +{ + int error = 0; + + INIT_DEBUGOUT("ixl_if_suspend: begin"); + + /* TODO: Call ixl_if_stop()? */ + + /* TODO: Then setup low power mode */ + + return (error); +} + +static int +ixl_if_resume(if_ctx_t ctx) +{ + struct ifnet *ifp = iflib_get_ifp(ctx); + + INIT_DEBUGOUT("ixl_if_resume: begin"); + + /* Read & clear wake-up registers */ + + /* Required after D3->D0 transition */ + if (ifp->if_flags & IFF_UP) + ixl_if_init(ctx); + + return (0); +} + +/* Set Report Status queue fields to 0 */ +static void +ixl_init_tx_rsqs(struct ixl_vsi *vsi) +{ + if_softc_ctx_t scctx = vsi->shared; + struct ixl_tx_queue *tx_que; + int i, j; + + for (i = 0, tx_que = vsi->tx_queues; i < vsi->num_tx_queues; i++, tx_que++) { + struct tx_ring *txr = &tx_que->txr; + + txr->tx_rs_cidx = txr->tx_rs_pidx = txr->tx_cidx_processed = 0; + + for (j = 0; j < scctx->isc_ntxd[0]; j++) + txr->tx_rsq[j] = QIDX_INVALID; + } +} + +void +ixl_if_init(if_ctx_t ctx) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + struct ixl_pf *pf = vsi->back; + struct i40e_hw *hw = &pf->hw; + device_t dev = iflib_get_dev(ctx); + u8 tmpaddr[ETHER_ADDR_LEN]; + int ret; + + /* + * If the aq is dead here, it probably means something outside of the driver + * did something to the adapter, like a PF reset. + * So rebuild the driver's state here if that occurs. + */ + if (!i40e_check_asq_alive(&pf->hw)) { + device_printf(dev, "Admin Queue is down; resetting...\n"); + ixl_teardown_hw_structs(pf); + ixl_reset(pf); + } + + /* Get the latest mac address... User might use a LAA */ + bcopy(IF_LLADDR(vsi->ifp), tmpaddr, + I40E_ETH_LENGTH_OF_ADDRESS); + if (!cmp_etheraddr(hw->mac.addr, tmpaddr) && + (i40e_validate_mac_addr(tmpaddr) == I40E_SUCCESS)) { + ixl_del_filter(vsi, hw->mac.addr, IXL_VLAN_ANY); + bcopy(tmpaddr, hw->mac.addr, + I40E_ETH_LENGTH_OF_ADDRESS); + ret = i40e_aq_mac_address_write(hw, + I40E_AQC_WRITE_TYPE_LAA_ONLY, + hw->mac.addr, NULL); + if (ret) { + device_printf(dev, "LLA address" + "change failed!!\n"); + return; + } + } + + iflib_set_mac(ctx, hw->mac.addr); + ixl_add_filter(vsi, hw->mac.addr, IXL_VLAN_ANY); + + /* Prepare the VSI: rings, hmc contexts, etc... */ + if (ixl_initialize_vsi(vsi)) { + device_printf(dev, "initialize vsi failed!!\n"); + return; + } + + // TODO: Call iflib setup multicast filters here? + // It's called in ixgbe in D5213 + ixl_if_multi_set(ctx); + + /* Set up RSS */ + ixl_config_rss(pf); + + /* Add protocol filters to list */ + ixl_init_filters(vsi); + + /* Setup vlan's if needed */ + ixl_setup_vlan_filters(vsi); + + /* Set up MSI/X routing and the ITR settings */ + if (pf->enable_msix) { + ixl_configure_queue_intr_msix(pf); + ixl_configure_itr(pf); + } else + ixl_configure_legacy(pf); + + ixl_init_tx_rsqs(vsi); + + ixl_enable_rings(vsi); + + ixl_reconfigure_filters(vsi); + +#ifdef IXL_IW + if (ixl_enable_iwarp && pf->iw_enabled) { + ret = ixl_iw_pf_init(pf); + if (ret) + device_printf(dev, + "initialize iwarp failed, code %d\n", ret); + } +#endif +} + +void +ixl_if_stop(if_ctx_t ctx) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + + INIT_DEBUGOUT("ixl_if_stop: begin\n"); + + ixl_disable_rings_intr(vsi); + ixl_disable_rings(vsi); +} + +static int +ixl_if_msix_intr_assign(if_ctx_t ctx, int msix) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + struct ixl_pf *pf = vsi->back; + struct ixl_rx_queue *rx_que = vsi->rx_queues; + struct ixl_tx_queue *tx_que = vsi->tx_queues; + int err, i, rid, vector = 0; + char buf[16]; + + /* Admin Que is vector 0*/ + rid = vector + 1; + err = iflib_irq_alloc_generic(ctx, &vsi->irq, rid, IFLIB_INTR_ADMIN, + ixl_msix_adminq, pf, 0, "aq"); + if (err) { + iflib_irq_free(ctx, &vsi->irq); + device_printf(iflib_get_dev(ctx), "Failed to register Admin que handler"); + return (err); + } + pf->admvec = vector; + // TODO: Re-enable this at some point + // iflib_softirq_alloc_generic(ctx, rid, IFLIB_INTR_IOV, pf, 0, "ixl_iov"); + + /* Now set up the stations */ + for (i = 0, vector = 1; i < vsi->num_rx_queues; i++, vector++, rx_que++) { + rid = vector + 1; + + snprintf(buf, sizeof(buf), "rxq%d", i); + err = iflib_irq_alloc_generic(ctx, &rx_que->que_irq, rid, IFLIB_INTR_RXTX, + ixl_msix_que, rx_que, rx_que->rxr.me, buf); + /* XXX: Does the driver work as expected if there are fewer num_rx_queues than + * what's expected in the iflib context? */ + if (err) { + device_printf(iflib_get_dev(ctx), "Failed to allocate q int %d err: %d", i, err); + vsi->num_rx_queues = i + 1; + goto fail; + } + rx_que->msix = vector; + } + + for (i = 0, vector = 1; i < vsi->num_tx_queues; i++, vector++, tx_que++) { + rid = vector + 1; + + snprintf(buf, sizeof(buf), "txq%d", i); + iflib_softirq_alloc_generic(ctx, rid, IFLIB_INTR_TX, tx_que, tx_que->txr.me, buf); + + /* TODO: Maybe call a strategy function for this to figure out which + * interrupts to map Tx queues to. I don't know if there's an immediately + * better way than this other than a user-supplied map, though. */ + tx_que->msix = (vector % vsi->num_rx_queues); + } + + return (0); +fail: + iflib_irq_free(ctx, &vsi->irq); + rx_que = vsi->rx_queues; + for (int i = 0; i < vsi->num_rx_queues; i++, rx_que++) + iflib_irq_free(ctx, &rx_que->que_irq); + return (err); +} + +/* + * Enable all interrupts + * TODO: Let it enable all interrupts? + * + * Called in: + * iflib_init_locked, after ixl_if_init() + */ +static void +ixl_if_enable_intr(if_ctx_t ctx) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + struct i40e_hw *hw = vsi->hw; + struct ixl_rx_queue *que = vsi->rx_queues; + + // TODO: Allow this to be enabled here? + ixl_enable_intr0(hw); + /* Enable queue interrupts */ + for (int i = 0; i < vsi->num_rx_queues; i++, que++) + /* TODO: Queue index parameter is probably wrong */ + ixl_enable_queue(hw, que->rxr.me); +} + +/* Disable all interrupts */ +static void +ixl_if_disable_intr(if_ctx_t ctx) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + struct i40e_hw *hw = vsi->hw; + struct ixl_rx_queue *que = vsi->rx_queues; + + // TODO: Don't disable admin queue interrupt / cause here + // ixl_disable_intr0(hw); + /* Disable queue interrupts */ + for (int i = 0; i < vsi->num_rx_queues; i++, que++) + /* TODO: Queue index parameter is probably wrong */ + ixl_disable_queue(hw, que->rxr.me); +} + +static int +ixl_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + struct i40e_hw *hw = vsi->hw; + struct ixl_rx_queue *rx_que = &vsi->rx_queues[rxqid]; + + ixl_enable_queue(hw, rx_que->msix - 1); + return (0); +} + +static int +ixl_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + struct i40e_hw *hw = vsi->hw; + struct ixl_tx_queue *tx_que = &vsi->tx_queues[txqid]; + + ixl_enable_queue(hw, tx_que->msix - 1); + return (0); +} + +static int +ixl_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int ntxqs, int ntxqsets) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + if_softc_ctx_t scctx = vsi->shared; + struct ixl_tx_queue *que; + int i, j, error = 0; + + MPASS(vsi->num_tx_queues > 0); + MPASS(ntxqs == 1); + MPASS(vsi->num_tx_queues == ntxqsets); + + /* Allocate queue structure memory */ + if (!(vsi->tx_queues = + (struct ixl_tx_queue *) malloc(sizeof(struct ixl_tx_queue) *ntxqsets, M_IXL, M_NOWAIT | M_ZERO))) { + device_printf(iflib_get_dev(ctx), "Unable to allocate TX ring memory\n"); + return (ENOMEM); + } + + for (i = 0, que = vsi->tx_queues; i < ntxqsets; i++, que++) { + struct tx_ring *txr = &que->txr; + + txr->me = i; + que->vsi = vsi; + + /* Allocate report status array */ + if (!(txr->tx_rsq = malloc(sizeof(qidx_t) * scctx->isc_ntxd[0], M_IXL, M_NOWAIT))) { + device_printf(iflib_get_dev(ctx), "failed to allocate tx_rsq memory\n"); + error = ENOMEM; + goto fail; + } + /* Init report status array */ + for (j = 0; j < scctx->isc_ntxd[0]; j++) + txr->tx_rsq[j] = QIDX_INVALID; + /* get the virtual and physical address of the hardware queues */ + txr->tail = I40E_QTX_TAIL(txr->me); + txr->tx_base = (struct i40e_tx_desc *)vaddrs[i * ntxqs]; + txr->tx_paddr = paddrs[i * ntxqs]; + txr->que = que; + } + + // TODO: Remove when finished + device_printf(iflib_get_dev(ctx), "allocated for %d txqs\n", vsi->num_tx_queues); + return (0); +fail: + ixl_if_queues_free(ctx); + return (error); +} + +static int +ixl_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nrxqs, int nrxqsets) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + struct ixl_rx_queue *que; + int i, error = 0; + + MPASS(vsi->num_rx_queues > 0); + MPASS(nrxqs == 1); + MPASS(vsi->num_rx_queues == nrxqsets); + + /* Allocate queue structure memory */ + if (!(vsi->rx_queues = + (struct ixl_rx_queue *) malloc(sizeof(struct ixl_rx_queue) * + nrxqsets, M_IXL, M_NOWAIT | M_ZERO))) { + device_printf(iflib_get_dev(ctx), "Unable to allocate RX ring memory\n"); + error = ENOMEM; + goto fail; + } + + for (i = 0, que = vsi->rx_queues; i < nrxqsets; i++, que++) { + struct rx_ring *rxr = &que->rxr; + + rxr->me = i; + que->vsi = vsi; + + /* get the virtual and physical address of the hardware queues */ + rxr->tail = I40E_QRX_TAIL(rxr->me); + rxr->rx_base = (union i40e_rx_desc *)vaddrs[i * nrxqs]; + rxr->rx_paddr = paddrs[i * nrxqs]; + rxr->que = que; + } + + // TODO: Remove when finished + device_printf(iflib_get_dev(ctx), "allocated for %d rxqs\n", vsi->num_rx_queues); + return (0); +fail: + ixl_if_queues_free(ctx); + return (error); +} + +static void +ixl_if_queues_free(if_ctx_t ctx) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + struct ixl_tx_queue *que; + int i; + + for (i = 0, que = vsi->tx_queues; i < vsi->num_tx_queues; i++, que++) { + struct tx_ring *txr = &que->txr; + if (txr->tx_rsq != NULL) { + free(txr->tx_rsq, M_IXL); + txr->tx_rsq = NULL; + } + } + + if (vsi->tx_queues != NULL) { + free(vsi->tx_queues, M_IXL); + vsi->tx_queues = NULL; + } + if (vsi->rx_queues != NULL) { + free(vsi->rx_queues, M_IXL); + vsi->rx_queues = NULL; + } +} + +void +ixl_update_link_status(if_ctx_t ctx) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + struct ixl_pf *pf = vsi->back; + u64 baudrate; + + if (pf->link_up) { + if (vsi->link_active == FALSE) { + vsi->link_active = TRUE; + baudrate = ixl_max_aq_speed_to_value(pf->link_speed); + iflib_link_state_change(ctx, LINK_STATE_UP, baudrate); + ixl_link_up_msg(pf); + // ixl_ping_all_vfs(adapter); + + } + } else { /* Link down */ + if (vsi->link_active == TRUE) { + vsi->link_active = FALSE; + iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); + // ixl_ping_all_vfs(adapter); + } + } +} + +/* TODO: Temporary name? */ +static int +ixl_process_adminq(struct ixl_pf *pf, u16 *pending) +{ + enum i40e_status_code status = I40E_SUCCESS; + struct i40e_arq_event_info event; + struct i40e_hw *hw = &pf->hw; + u16 opcode; + u32 loop = 0, reg; + + event.buf_len = IXL_AQ_BUF_SZ; + event.msg_buf = malloc(event.buf_len, M_IXL, M_NOWAIT | M_ZERO); + if (!event.msg_buf) { + device_printf(pf->dev, "%s: Unable to allocate memory for Admin" + " Queue event!\n", __func__); + return (ENOMEM); + } + + /* clean and process any events */ + do { + status = i40e_clean_arq_element(hw, &event, pending); + if (status) + break; + opcode = LE16_TO_CPU(event.desc.opcode); + ixl_dbg(pf, IXL_DBG_AQ, + "Admin Queue event: %#06x\n", opcode); + switch (opcode) { + case i40e_aqc_opc_get_link_status: + ixl_link_event(pf, &event); + break; + case i40e_aqc_opc_send_msg_to_pf: +#ifdef PCI_IOV + ixl_handle_vf_msg(pf, &event); +#endif + break; + case i40e_aqc_opc_event_lan_overflow: + break; + default: + printf("AdminQ unknown event %x\n", opcode); + break; + } + } while (*pending && (loop++ < IXL_ADM_LIMIT)); + + free(event.msg_buf, M_IXL); + + /* Re-enable admin queue interrupt cause */ + reg = rd32(hw, I40E_PFINT_ICR0_ENA); + reg |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK; + wr32(hw, I40E_PFINT_ICR0_ENA, reg); + + return (status); +} + +static void +ixl_if_update_admin_status(if_ctx_t ctx) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + struct ixl_pf *pf = vsi->back; + struct i40e_hw *hw = &pf->hw; + u16 pending; + + // TODO: Refactor reset handling + if (pf->state & IXL_PF_STATE_ADAPTER_RESETTING) { + ixl_handle_empr_reset(pf); + iflib_init_locked(ctx); + } + + if (pf->state & IXL_PF_STATE_CORE_RESET_REQ) { + device_printf(pf->dev, "Doing CORE reset...\n"); + iflib_stop(ctx); + ixl_teardown_hw_structs(pf); + wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK); + atomic_set_int(&pf->state, IXL_PF_STATE_ADAPTER_RESETTING); + // ixl_handle_empr_reset(pf); + // iflib_init_locked(ctx); + return; + } + + if (pf->state & IXL_PF_STATE_GLOB_RESET_REQ) { + device_printf(pf->dev, "Doing GLOB reset...\n"); + iflib_stop(ctx); + ixl_teardown_hw_structs(pf); + wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_GLOBR_MASK); + atomic_set_int(&pf->state, IXL_PF_STATE_ADAPTER_RESETTING); + // ixl_handle_empr_reset(pf); + // iflib_init_locked(ctx); + return; + } + + if (pf->state & IXL_PF_STATE_EMP_RESET_REQ) { + /* This register is read-only to drivers */ + if (!(rd32(hw, 0x000B818C) & 0x1)) { + device_printf(pf->dev, "SW not allowed to initiate EMPR\n"); + atomic_clear_int(&pf->state, IXL_PF_STATE_EMP_RESET_REQ); + } else { + device_printf(pf->dev, "Doing EMP reset...\n"); + iflib_stop(ctx); + ixl_teardown_hw_structs(pf); + wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_EMPFWR_MASK); + atomic_set_int(&pf->state, IXL_PF_STATE_ADAPTER_RESETTING); + // ixl_handle_empr_reset(pf); + // iflib_init_locked(ctx); + return; + } + } + + if (pf->state & IXL_PF_STATE_MDD_PENDING) + ixl_handle_mdd_event(pf); + + if (pf->state & IXL_PF_STATE_PF_RESET_REQ) { + device_printf(pf->dev, "Doing PF reset...\n"); + iflib_stop(ctx); + ixl_teardown_hw_structs(pf); + ixl_reset(pf); + device_printf(pf->dev, "PF reset done.\n"); + // TODO: Do init if previously up! + iflib_init_locked(ctx); + } + + if (pf->state & IXL_PF_STATE_VF_RESET_REQ) + ixl_handle_vflr(pf, 0); + + ixl_process_adminq(pf, &pending); + ixl_update_link_status(ctx); + + /* + * If there are still messages to process, reschedule ourselves. + * Otherwise, re-enable our interrupt and go to sleep. + */ + if (pending > 0) + iflib_admin_intr_deferred(ctx); + else + ixl_enable_intr0(hw); +} + +static void +ixl_if_multi_set(if_ctx_t ctx) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + struct i40e_hw *hw = vsi->hw; + int mcnt = 0, flags; + + IOCTL_DEBUGOUT("ixl_if_multi_set: begin"); + + mcnt = if_multiaddr_count(iflib_get_ifp(ctx), MAX_MULTICAST_ADDR); + /* delete existing MC filters */ + ixl_del_multi(vsi); + + if (__predict_false(mcnt == MAX_MULTICAST_ADDR)) { + i40e_aq_set_vsi_multicast_promiscuous(hw, + vsi->seid, TRUE, NULL); + return; + } + /* (re-)install filters for all mcast addresses */ + mcnt = if_multi_apply(iflib_get_ifp(ctx), ixl_mc_filter_apply, vsi); + + if (mcnt > 0) { + flags = (IXL_FILTER_ADD | IXL_FILTER_USED | IXL_FILTER_MC); + ixl_add_hw_filters(vsi, flags, mcnt); + } + + IOCTL_DEBUGOUT("ixl_if_multi_set: end"); +} + +static int +ixl_if_mtu_set(if_ctx_t ctx, uint32_t mtu) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + + IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); + if (mtu > IXL_MAX_FRAME - ETHER_HDR_LEN - ETHER_CRC_LEN - + ETHER_VLAN_ENCAP_LEN) + return (EINVAL); + + vsi->shared->isc_max_frame_size = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN + + ETHER_VLAN_ENCAP_LEN; + + return (0); +} + +static void +ixl_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + struct ixl_pf *pf = (struct ixl_pf *)vsi->back; + struct i40e_hw *hw = &pf->hw; + + INIT_DEBUGOUT("ixl_media_status: begin"); + + hw->phy.get_link_info = TRUE; + i40e_get_link_status(hw, &pf->link_up); + + ifmr->ifm_status = IFM_AVALID; + ifmr->ifm_active = IFM_ETHER; + + if (!pf->link_up) { + return; + } + + ifmr->ifm_status |= IFM_ACTIVE; + /* Hardware is always full-duplex */ + ifmr->ifm_active |= IFM_FDX; + + switch (hw->phy.link_info.phy_type) { + /* 100 M */ + case I40E_PHY_TYPE_100BASE_TX: + ifmr->ifm_active |= IFM_100_TX; + break; + /* 1 G */ + case I40E_PHY_TYPE_1000BASE_T: + ifmr->ifm_active |= IFM_1000_T; + break; + case I40E_PHY_TYPE_1000BASE_SX: + ifmr->ifm_active |= IFM_1000_SX; + break; + case I40E_PHY_TYPE_1000BASE_LX: + ifmr->ifm_active |= IFM_1000_LX; + break; + case I40E_PHY_TYPE_1000BASE_T_OPTICAL: + ifmr->ifm_active |= IFM_OTHER; + break; + /* 10 G */ + case I40E_PHY_TYPE_10GBASE_SFPP_CU: + ifmr->ifm_active |= IFM_10G_TWINAX; + break; + case I40E_PHY_TYPE_10GBASE_SR: + ifmr->ifm_active |= IFM_10G_SR; + break; + case I40E_PHY_TYPE_10GBASE_LR: + ifmr->ifm_active |= IFM_10G_LR; + break; + case I40E_PHY_TYPE_10GBASE_T: + ifmr->ifm_active |= IFM_10G_T; + break; + case I40E_PHY_TYPE_XAUI: + case I40E_PHY_TYPE_XFI: + case I40E_PHY_TYPE_10GBASE_AOC: + ifmr->ifm_active |= IFM_OTHER; + break; + /* 25 G */ + case I40E_PHY_TYPE_25GBASE_KR: + ifmr->ifm_active |= IFM_25G_KR; + break; + case I40E_PHY_TYPE_25GBASE_CR: + ifmr->ifm_active |= IFM_25G_CR; + break; + case I40E_PHY_TYPE_25GBASE_SR: + ifmr->ifm_active |= IFM_25G_SR; + break; + case I40E_PHY_TYPE_25GBASE_LR: + ifmr->ifm_active |= IFM_UNKNOWN; + break; + /* 40 G */ + case I40E_PHY_TYPE_40GBASE_CR4: + case I40E_PHY_TYPE_40GBASE_CR4_CU: + ifmr->ifm_active |= IFM_40G_CR4; + break; + case I40E_PHY_TYPE_40GBASE_SR4: + ifmr->ifm_active |= IFM_40G_SR4; + break; + case I40E_PHY_TYPE_40GBASE_LR4: + ifmr->ifm_active |= IFM_40G_LR4; + break; + case I40E_PHY_TYPE_XLAUI: + ifmr->ifm_active |= IFM_OTHER; + break; + case I40E_PHY_TYPE_1000BASE_KX: + ifmr->ifm_active |= IFM_1000_KX; + break; + case I40E_PHY_TYPE_SGMII: + ifmr->ifm_active |= IFM_1000_SGMII; + break; + /* ERJ: What's the difference between these? */ + case I40E_PHY_TYPE_10GBASE_CR1_CU: + case I40E_PHY_TYPE_10GBASE_CR1: + ifmr->ifm_active |= IFM_10G_CR1; + break; + case I40E_PHY_TYPE_10GBASE_KX4: + ifmr->ifm_active |= IFM_10G_KX4; + break; + case I40E_PHY_TYPE_10GBASE_KR: + ifmr->ifm_active |= IFM_10G_KR; + break; + case I40E_PHY_TYPE_SFI: + ifmr->ifm_active |= IFM_10G_SFI; + break; + /* Our single 20G media type */ + case I40E_PHY_TYPE_20GBASE_KR2: + ifmr->ifm_active |= IFM_20G_KR2; + break; + case I40E_PHY_TYPE_40GBASE_KR4: + ifmr->ifm_active |= IFM_40G_KR4; + break; + case I40E_PHY_TYPE_XLPPI: + case I40E_PHY_TYPE_40GBASE_AOC: + ifmr->ifm_active |= IFM_40G_XLPPI; + break; + /* Unknown to driver */ + default: + ifmr->ifm_active |= IFM_UNKNOWN; + break; + } + /* Report flow control status as well */ + if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_TX) + ifmr->ifm_active |= IFM_ETH_TXPAUSE; + if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_RX) + ifmr->ifm_active |= IFM_ETH_RXPAUSE; + +} + +static int +ixl_if_media_change(if_ctx_t ctx) +{ + struct ifmedia *ifm = iflib_get_media(ctx); + + INIT_DEBUGOUT("ixl_media_change: begin"); + + if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) + return (EINVAL); + + if_printf(iflib_get_ifp(ctx), "Media change is not supported.\n"); + return (ENODEV); +} + +static int +ixl_if_promisc_set(if_ctx_t ctx, int flags) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + struct ifnet *ifp = iflib_get_ifp(ctx); + struct i40e_hw *hw = vsi->hw; + int err; + bool uni = FALSE, multi = FALSE; + + if (flags & IFF_ALLMULTI || + if_multiaddr_count(ifp, MAX_MULTICAST_ADDR) == MAX_MULTICAST_ADDR) + multi = TRUE; + if (flags & IFF_PROMISC) + uni = TRUE; + + err = i40e_aq_set_vsi_unicast_promiscuous(hw, + vsi->seid, uni, NULL, false); + if (err) + return (err); + err = i40e_aq_set_vsi_multicast_promiscuous(hw, + vsi->seid, multi, NULL); + return (err); +} + +static void +ixl_if_timer(if_ctx_t ctx, uint16_t qid) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + struct ixl_pf *pf = vsi->back; + //struct i40e_hw *hw = &pf->hw; + //struct ixl_tx_queue *que = &vsi->tx_queues[qid]; + #if 0 + u32 mask; + + /* + ** Check status of the queues + */ + mask = (I40E_PFINT_DYN_CTLN_INTENA_MASK | + I40E_PFINT_DYN_CTLN_SWINT_TRIG_MASK); + + /* If queue param has outstanding work, trigger sw irq */ + // TODO: TX queues in iflib don't use HW interrupts; does this do anything? + if (que->busy) + wr32(hw, I40E_PFINT_DYN_CTLN(que->txr.me), mask); +#endif + + if (qid != 0) + return; + + /* Fire off the adminq task */ + iflib_admin_intr_deferred(ctx); + + /* Update stats */ + ixl_update_stats_counters(pf); +} + +static void +ixl_if_vlan_register(if_ctx_t ctx, u16 vtag) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + struct i40e_hw *hw = vsi->hw; + + if ((vtag == 0) || (vtag > 4095)) /* Invalid */ + return; + + ++vsi->num_vlans; + ixl_add_filter(vsi, hw->mac.addr, vtag); +} + +static void +ixl_if_vlan_unregister(if_ctx_t ctx, u16 vtag) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + struct i40e_hw *hw = vsi->hw; + + if ((vtag == 0) || (vtag > 4095)) /* Invalid */ + return; + + --vsi->num_vlans; + ixl_del_filter(vsi, hw->mac.addr, vtag); +} + +static uint64_t +ixl_if_get_counter(if_ctx_t ctx, ift_counter cnt) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + if_t ifp = iflib_get_ifp(ctx); + + switch (cnt) { + case IFCOUNTER_IPACKETS: + return (vsi->ipackets); + case IFCOUNTER_IERRORS: + return (vsi->ierrors); + case IFCOUNTER_OPACKETS: + return (vsi->opackets); + case IFCOUNTER_OERRORS: + return (vsi->oerrors); + case IFCOUNTER_COLLISIONS: + /* Collisions are by standard impossible in 40G/10G Ethernet */ + return (0); + case IFCOUNTER_IBYTES: + return (vsi->ibytes); + case IFCOUNTER_OBYTES: + return (vsi->obytes); + case IFCOUNTER_IMCASTS: + return (vsi->imcasts); + case IFCOUNTER_OMCASTS: + return (vsi->omcasts); + case IFCOUNTER_IQDROPS: + return (vsi->iqdrops); + case IFCOUNTER_OQDROPS: + return (vsi->oqdrops); + case IFCOUNTER_NOPROTO: + return (vsi->noproto); + default: + return (if_get_counter_default(ifp, cnt)); + } +} + +static int +ixl_mc_filter_apply(void *arg, struct ifmultiaddr *ifma, int count __unused) +{ + struct ixl_vsi *vsi = arg; + + if (ifma->ifma_addr->sa_family != AF_LINK) + return (0); + ixl_add_mc_filter(vsi, + (u8*)LLADDR((struct sockaddr_dl *) ifma->ifma_addr)); + return (1); +} + +static int +ixl_save_pf_tunables(struct ixl_pf *pf) +{ + device_t dev = pf->dev; + + /* Save tunable information */ + pf->enable_msix = ixl_enable_msix; + pf->max_queues = ixl_max_queues; + pf->enable_tx_fc_filter = ixl_enable_tx_fc_filter; + pf->dynamic_rx_itr = ixl_dynamic_rx_itr; + pf->rx_itr = ixl_rx_itr; + pf->dbg_mask = ixl_core_debug_mask; + pf->hw.debug_mask = ixl_shared_debug_mask; + pf->enable_hwb = ixl_enable_hwb; + + /* TODO: Ring size tunable probably needs to be removed */ + /* But maybe the iflib value should be checked here, too */ + if (ixl_ring_size < IXL_MIN_RING + || ixl_ring_size > IXL_MAX_RING + || ixl_ring_size % IXL_RING_INCREMENT != 0) { + device_printf(dev, "Invalid ring_size value of %d set!\n", + ixl_ring_size); + device_printf(dev, "ring_size must be between %d and %d, " + "inclusive, and must be a multiple of %d\n", + IXL_MIN_RING, IXL_MAX_RING, IXL_RING_INCREMENT); + device_printf(dev, "Using default value of %d instead\n", + IXL_DEFAULT_RING); + pf->ringsz = IXL_DEFAULT_RING; + } else + pf->ringsz = ixl_ring_size; + + /* No TX ITR (for HW at least) */ + + if (ixl_rx_itr < 0 || ixl_rx_itr > IXL_MAX_ITR) { + device_printf(dev, "Invalid rx_itr value of %d set!\n", + ixl_rx_itr); + device_printf(dev, "rx_itr must be between %d and %d, " + "inclusive\n", + 0, IXL_MAX_ITR); + device_printf(dev, "Using default value of %d instead\n", + IXL_ITR_8K); + pf->rx_itr = IXL_ITR_8K; + } else + pf->rx_itr = ixl_rx_itr; + + return (0); +} + +static int +ixl_attach_get_link_status(struct ixl_pf *pf) +{ + struct i40e_hw *hw = &pf->hw; + device_t dev = pf->dev; + int error = 0; + + if (((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver < 33)) || + (hw->aq.fw_maj_ver < 4)) { + i40e_msec_delay(75); + error = i40e_aq_set_link_restart_an(hw, TRUE, NULL); + if (error) { + device_printf(dev, "link restart failed, aq_err=%d\n", + pf->hw.aq.asq_last_status); + return error; + } + } + + /* Determine link state */ + hw->phy.get_link_info = TRUE; + i40e_get_link_status(hw, &pf->link_up); return (0); } Index: sys/dev/ixl/if_ixlv.c =================================================================== --- sys/dev/ixl/if_ixlv.c +++ sys/dev/ixl/if_ixlv.c @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2013-2015, Intel Corporation + Copyright (c) 2013-2017, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -44,92 +44,85 @@ * PCI Device ID Table * * Used by probe to select devices to load on - * Last field stores an index into ixlv_strings - * Last entry must be all 0s * - * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } + * ( Vendor ID, Device ID, Branding String ) *********************************************************************/ -static ixl_vendor_info_t ixlv_vendor_info_array[] = +static pci_vendor_info_t ixlv_vendor_info_array[] = { - {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_VF, 0, 0, 0}, - {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_X722_VF, 0, 0, 0}, - {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_X722_A0_VF, 0, 0, 0}, + PVID(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_VF, "Intel(R) Ethernet Connection 700 Series VF Driver"), + PVID(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_X722_VF, "Intel(R) Ethernet Connection 700 Series VF Driver"), /* required last entry */ - {0, 0, 0, 0, 0} + PVID_END }; /********************************************************************* - * Table of branding strings - *********************************************************************/ - -static char *ixlv_strings[] = { - "Intel(R) Ethernet Connection XL710/X722 VF Driver" -}; - - -/********************************************************************* * Function prototypes *********************************************************************/ -static int ixlv_probe(device_t); -static int ixlv_attach(device_t); -static int ixlv_detach(device_t); -static int ixlv_shutdown(device_t); -static void ixlv_init_locked(struct ixlv_sc *); +static void *ixlv_register(device_t dev); +static int ixlv_if_attach_pre(if_ctx_t ctx); +static int ixlv_if_attach_post(if_ctx_t ctx); +static int ixlv_if_detach(if_ctx_t ctx); +static int ixlv_if_shutdown(if_ctx_t ctx); +static int ixlv_if_suspend(if_ctx_t ctx); +static int ixlv_if_resume(if_ctx_t ctx); +static int ixlv_if_msix_intr_assign(if_ctx_t ctx, int msix); +static void ixlv_if_enable_intr(if_ctx_t ctx); +static void ixlv_if_disable_intr(if_ctx_t ctx); +static int ixlv_if_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid); +static int ixlv_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int ntxqs, int ntxqsets); +static int ixlv_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nqs, int nqsets); +static void ixlv_if_queues_free(if_ctx_t ctx); +static void ixlv_if_update_admin_status(if_ctx_t ctx); +static void ixlv_if_multi_set(if_ctx_t ctx); +static int ixlv_if_mtu_set(if_ctx_t ctx, uint32_t mtu); +static void ixlv_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr); +static int ixlv_if_media_change(if_ctx_t ctx); +static int ixlv_if_promisc_set(if_ctx_t ctx, int flags); +static void ixlv_if_timer(if_ctx_t ctx, uint16_t qid); +static void ixlv_if_vlan_register(if_ctx_t ctx, u16 vtag); +static void ixlv_if_vlan_unregister(if_ctx_t ctx, u16 vtag); +static uint64_t ixlv_if_get_counter(if_ctx_t ctx, ift_counter cnt); +static void ixlv_if_stop(if_ctx_t ctx); + static int ixlv_allocate_pci_resources(struct ixlv_sc *); +static int ixlv_reset_complete(struct i40e_hw *); +static int ixlv_setup_vc(struct ixlv_sc *); +static int ixlv_reset(struct ixlv_sc *); +static int ixlv_vf_config(struct ixlv_sc *); +static void ixlv_init_filters(struct ixlv_sc *); static void ixlv_free_pci_resources(struct ixlv_sc *); -static int ixlv_assign_msix(struct ixlv_sc *); -static int ixlv_init_msix(struct ixlv_sc *); -static int ixlv_init_taskqueue(struct ixlv_sc *); -static int ixlv_setup_queues(struct ixlv_sc *); +static void ixlv_free_filters(struct ixlv_sc *); +static void ixlv_setup_interface(device_t, struct ixl_vsi *); +static void ixlv_add_sysctls(struct ixlv_sc *); +static void ixlv_enable_adminq_irq(struct i40e_hw *); +static void ixlv_disable_adminq_irq(struct i40e_hw *); +static void ixlv_enable_queue_irq(struct i40e_hw *, int); +static void ixlv_disable_queue_irq(struct i40e_hw *, int); + static void ixlv_config_rss(struct ixlv_sc *); static void ixlv_stop(struct ixlv_sc *); -static void ixlv_add_multi(struct ixl_vsi *); -static void ixlv_del_multi(struct ixl_vsi *); -static void ixlv_free_queues(struct ixl_vsi *); -static int ixlv_setup_interface(device_t, struct ixlv_sc *); -static int ixlv_teardown_adminq_msix(struct ixlv_sc *); - -static int ixlv_media_change(struct ifnet *); -static void ixlv_media_status(struct ifnet *, struct ifmediareq *); - -static void ixlv_local_timer(void *); +//static void ixlv_add_multi(struct ixl_vsi *); +//static void ixlv_del_multi(struct ixl_vsi *); static int ixlv_add_mac_filter(struct ixlv_sc *, u8 *, u16); static int ixlv_del_mac_filter(struct ixlv_sc *sc, u8 *macaddr); -static void ixlv_init_filters(struct ixlv_sc *); -static void ixlv_free_filters(struct ixlv_sc *); -static void ixlv_msix_que(void *); -static void ixlv_msix_adminq(void *); -static void ixlv_do_adminq(void *, int); +static int ixlv_msix_que(void *); +static int ixlv_msix_adminq(void *); +//static void ixlv_do_adminq(void *, int); static void ixlv_do_adminq_locked(struct ixlv_sc *sc); -static void ixlv_handle_que(void *, int); -static int ixlv_reset(struct ixlv_sc *); -static int ixlv_reset_complete(struct i40e_hw *); -static void ixlv_set_queue_rx_itr(struct ixl_queue *); -static void ixlv_set_queue_tx_itr(struct ixl_queue *); +//static void ixlv_handle_que(void *, int); +// TODO: Call this! +//static void ixlv_set_queue_rx_itr(struct ixl_rx_queue *); static void ixl_init_cmd_complete(struct ixl_vc_cmd *, void *, enum i40e_status_code); static void ixlv_configure_itr(struct ixlv_sc *); -static void ixlv_enable_adminq_irq(struct i40e_hw *); -static void ixlv_disable_adminq_irq(struct i40e_hw *); -static void ixlv_enable_queue_irq(struct i40e_hw *, int); -static void ixlv_disable_queue_irq(struct i40e_hw *, int); - static void ixlv_setup_vlan_filters(struct ixlv_sc *); -static void ixlv_register_vlan(void *, struct ifnet *, u16); -static void ixlv_unregister_vlan(void *, struct ifnet *, u16); - -static void ixlv_init_hw(struct ixlv_sc *); -static int ixlv_setup_vc(struct ixlv_sc *); -static int ixlv_vf_config(struct ixlv_sc *); -static void ixlv_cap_txcsum_tso(struct ixl_vsi *, - struct ifnet *, int); +// static void ixlv_init_hw(struct ixlv_sc *); -static void ixlv_add_sysctls(struct ixlv_sc *); #ifdef IXL_DEBUG static int ixlv_sysctl_qtx_tail_handler(SYSCTL_HANDLER_ARGS); static int ixlv_sysctl_qrx_tail_handler(SYSCTL_HANDLER_ARGS); @@ -141,11 +134,12 @@ static device_method_t ixlv_methods[] = { /* Device interface */ - DEVMETHOD(device_probe, ixlv_probe), - DEVMETHOD(device_attach, ixlv_attach), - DEVMETHOD(device_detach, ixlv_detach), - DEVMETHOD(device_shutdown, ixlv_shutdown), - {0, 0} + DEVMETHOD(device_register, ixlv_register), + DEVMETHOD(device_probe, iflib_device_probe), + DEVMETHOD(device_attach, iflib_device_attach), + DEVMETHOD(device_detach, iflib_device_detach), + DEVMETHOD(device_shutdown, iflib_device_shutdown), + DEVMETHOD_END }; static driver_t ixlv_driver = { @@ -157,10 +151,45 @@ MODULE_DEPEND(ixlv, pci, 1, 1, 1); MODULE_DEPEND(ixlv, ether, 1, 1, 1); +MODULE_DEPEND(ixlv, iflib, 1, 1, 1); + +static device_method_t ixlv_if_methods[] = { + DEVMETHOD(ifdi_attach_pre, ixlv_if_attach_pre), + DEVMETHOD(ifdi_attach_post, ixlv_if_attach_post), + DEVMETHOD(ifdi_detach, ixlv_if_detach), + DEVMETHOD(ifdi_shutdown, ixlv_if_shutdown), + DEVMETHOD(ifdi_suspend, ixlv_if_suspend), + DEVMETHOD(ifdi_resume, ixlv_if_resume), + DEVMETHOD(ifdi_init, ixlv_if_init), + DEVMETHOD(ifdi_stop, ixlv_if_stop), + DEVMETHOD(ifdi_msix_intr_assign, ixlv_if_msix_intr_assign), + DEVMETHOD(ifdi_intr_enable, ixlv_if_enable_intr), + DEVMETHOD(ifdi_intr_disable, ixlv_if_disable_intr), + DEVMETHOD(ifdi_queue_intr_enable, ixlv_if_queue_intr_enable), + DEVMETHOD(ifdi_tx_queues_alloc, ixlv_if_tx_queues_alloc), + DEVMETHOD(ifdi_rx_queues_alloc, ixlv_if_rx_queues_alloc), + DEVMETHOD(ifdi_queues_free, ixlv_if_queues_free), + DEVMETHOD(ifdi_update_admin_status, ixlv_if_update_admin_status), + DEVMETHOD(ifdi_multi_set, ixlv_if_multi_set), + DEVMETHOD(ifdi_mtu_set, ixlv_if_mtu_set), + // DEVMETHOD(ifdi_crcstrip_set, ixlv_if_crcstrip_set), + DEVMETHOD(ifdi_media_status, ixlv_if_media_status), + DEVMETHOD(ifdi_media_change, ixlv_if_media_change), + DEVMETHOD(ifdi_promisc_set, ixlv_if_promisc_set), + DEVMETHOD(ifdi_timer, ixlv_if_timer), + DEVMETHOD(ifdi_vlan_register, ixlv_if_vlan_register), + DEVMETHOD(ifdi_vlan_unregister, ixlv_if_vlan_unregister), + DEVMETHOD(ifdi_get_counter, ixlv_if_get_counter), + DEVMETHOD_END +}; -/* +static driver_t ixlv_if_driver = { + "ixlv_if", ixlv_if_methods, sizeof(struct ixlv_sc) +}; + +/***************************************************************************** ** TUNEABLE PARAMETERS: -*/ +*****************************************************************************/ static SYSCTL_NODE(_hw, OID_AUTO, ixlv, CTLFLAG_RD, 0, "IXLV driver parameters"); @@ -169,10 +198,10 @@ ** Number of descriptors per ring: ** - TX and RX are the same size */ -static int ixlv_ringsz = IXL_DEFAULT_RING; -TUNABLE_INT("hw.ixlv.ringsz", &ixlv_ringsz); +static int ixlv_ring_size = IXL_DEFAULT_RING; +TUNABLE_INT("hw.ixlv.ring_size", &ixlv_ring_size); SYSCTL_INT(_hw_ixlv, OID_AUTO, ring_size, CTLFLAG_RDTUN, - &ixlv_ringsz, 0, "Descriptor Ring Size"); + &ixlv_ring_size, 0, "Descriptor Ring Size"); /* Set to zero to auto calculate */ int ixlv_max_queues = 0; @@ -181,17 +210,6 @@ &ixlv_max_queues, 0, "Number of Queues"); /* -** Number of entries in Tx queue buf_ring. -** Increasing this will reduce the number of -** errors when transmitting fragmented UDP -** packets. -*/ -static int ixlv_txbrsz = DEFAULT_TXBRSZ; -TUNABLE_INT("hw.ixlv.txbrsz", &ixlv_txbrsz); -SYSCTL_INT(_hw_ixlv, OID_AUTO, txbr_size, CTLFLAG_RDTUN, - &ixlv_txbrsz, 0, "TX Buf Ring Size"); - -/* ** Controls for Interrupt Throttling ** - true/false for dynamic adjustment ** - default values for static ITR @@ -201,111 +219,107 @@ SYSCTL_INT(_hw_ixlv, OID_AUTO, dynamic_rx_itr, CTLFLAG_RDTUN, &ixlv_dynamic_rx_itr, 0, "Dynamic RX Interrupt Rate"); -int ixlv_dynamic_tx_itr = 0; -TUNABLE_INT("hw.ixlv.dynamic_tx_itr", &ixlv_dynamic_tx_itr); -SYSCTL_INT(_hw_ixlv, OID_AUTO, dynamic_tx_itr, CTLFLAG_RDTUN, - &ixlv_dynamic_tx_itr, 0, "Dynamic TX Interrupt Rate"); - int ixlv_rx_itr = IXL_ITR_8K; TUNABLE_INT("hw.ixlv.rx_itr", &ixlv_rx_itr); SYSCTL_INT(_hw_ixlv, OID_AUTO, rx_itr, CTLFLAG_RDTUN, &ixlv_rx_itr, 0, "RX Interrupt Rate"); -int ixlv_tx_itr = IXL_ITR_4K; -TUNABLE_INT("hw.ixlv.tx_itr", &ixlv_tx_itr); -SYSCTL_INT(_hw_ixlv, OID_AUTO, tx_itr, CTLFLAG_RDTUN, - &ixlv_tx_itr, 0, "TX Interrupt Rate"); - -/********************************************************************* - * Device identification routine - * - * ixlv_probe determines if the driver should be loaded on - * the hardware based on PCI vendor/device id of the device. - * - * return BUS_PROBE_DEFAULT on success, positive on failure - *********************************************************************/ - -static int -ixlv_probe(device_t dev) -{ - ixl_vendor_info_t *ent; - - u16 pci_vendor_id, pci_device_id; - u16 pci_subvendor_id, pci_subdevice_id; - char device_name[256]; +extern struct if_txrx ixl_txrx; + +static struct if_shared_ctx ixlv_sctx_init = { + .isc_magic = IFLIB_MAGIC, + .isc_q_align = PAGE_SIZE,/* max(DBA_ALIGN, PAGE_SIZE) */ + .isc_tx_maxsize = IXL_TSO_SIZE, + + .isc_tx_maxsegsize = PAGE_SIZE, + + // TODO: Review the rx_maxsize and rx_maxsegsize params + // Where are they used in iflib? + .isc_rx_maxsize = 16384, + .isc_rx_nsegments = 1, + .isc_rx_maxsegsize = 16384, + // TODO: What is isc_nfl for? + .isc_nfl = 1, + .isc_ntxqs = 1, + .isc_nrxqs = 1, + + .isc_admin_intrcnt = 1, + .isc_vendor_info = ixlv_vendor_info_array, + .isc_driver_version = ixlv_driver_version, + .isc_driver = &ixlv_if_driver, + + .isc_nrxd_min = {IXL_MIN_RING}, + .isc_ntxd_min = {IXL_MIN_RING}, + .isc_nrxd_max = {IXL_MAX_RING}, + .isc_ntxd_max = {IXL_MAX_RING}, + .isc_nrxd_default = {IXL_DEFAULT_RING}, + .isc_ntxd_default = {IXL_DEFAULT_RING}, +}; -#if 0 - INIT_DEBUGOUT("ixlv_probe: begin"); -#endif +if_shared_ctx_t ixlv_sctx = &ixlv_sctx_init; - pci_vendor_id = pci_get_vendor(dev); - if (pci_vendor_id != I40E_INTEL_VENDOR_ID) - return (ENXIO); +/*** Functions ***/ - pci_device_id = pci_get_device(dev); - pci_subvendor_id = pci_get_subvendor(dev); - pci_subdevice_id = pci_get_subdevice(dev); - - ent = ixlv_vendor_info_array; - while (ent->vendor_id != 0) { - if ((pci_vendor_id == ent->vendor_id) && - (pci_device_id == ent->device_id) && - - ((pci_subvendor_id == ent->subvendor_id) || - (ent->subvendor_id == 0)) && - - ((pci_subdevice_id == ent->subdevice_id) || - (ent->subdevice_id == 0))) { - sprintf(device_name, "%s, Version - %s", - ixlv_strings[ent->index], - ixlv_driver_version); - device_set_desc_copy(dev, device_name); - return (BUS_PROBE_DEFAULT); - } - ent++; - } - return (ENXIO); +static void * +ixlv_register(device_t dev) +{ + return (ixlv_sctx); } -/********************************************************************* - * Device initialization routine - * - * The attach entry point is called when the driver is being loaded. - * This routine identifies the type of hardware, allocates all resources - * and initializes the hardware. - * - * return 0 on success, positive on failure - *********************************************************************/ - static int -ixlv_attach(device_t dev) +ixlv_if_attach_pre(if_ctx_t ctx) { + device_t dev; struct ixlv_sc *sc; struct i40e_hw *hw; struct ixl_vsi *vsi; - int error = 0; + if_softc_ctx_t scctx; + int error = 0; INIT_DBG_DEV(dev, "begin"); - /* Allocate, clear, and link in our primary soft structure */ - sc = device_get_softc(dev); - sc->dev = sc->osdep.dev = dev; + dev = iflib_get_dev(ctx); + sc = iflib_get_softc(ctx); + hw = &sc->hw; + /* + ** Note this assumes we have a single embedded VSI, + ** this could be enhanced later to allocate multiple + */ vsi = &sc->vsi; - vsi->dev = dev; + vsi->back = sc; + vsi->hw = &sc->hw; + // vsi->id = 0; + vsi->num_vlans = 0; + vsi->ctx = ctx; + vsi->media = iflib_get_media(ctx); + vsi->shared = scctx = iflib_get_softc_ctx(ctx); + sc->dev = dev; - /* Initialize hw struct */ - ixlv_init_hw(sc); + /* + * These are the same across all current ixl models + */ + vsi->shared->isc_tx_nsegments = IXL_MAX_TX_SEGS; + vsi->shared->isc_msix_bar = PCIR_BAR(IXL_MSIX_BAR); - /* Allocate filter lists */ - ixlv_init_filters(sc); + vsi->shared->isc_tx_tso_segments_max = IXL_MAX_TSO_SEGS; + vsi->shared->isc_tx_tso_size_max = IXL_TSO_SIZE; + vsi->shared->isc_tx_tso_segsize_max = PAGE_SIZE; - /* Core Lock Init */ - mtx_init(&sc->mtx, device_get_nameunit(dev), - "IXL SC Lock", MTX_DEF); +#if 0 // TODO: Implement + /* Save tunable values */ + error = ixl_save_pf_tunables(pf); + if (error) + return (error); +#endif - /* Set up the timer callout */ - callout_init_mtx(&sc->timer, &sc->mtx, 0); + scctx->isc_txqsizes[0] = roundup2(scctx->isc_ntxd[0] + * sizeof(struct i40e_tx_desc) + sizeof(u32), DBA_ALIGN); + scctx->isc_rxqsizes[0] = roundup2(scctx->isc_nrxd[0] + * sizeof(union i40e_32byte_rx_desc), DBA_ALIGN); + /* XXX: No idea what this does */ + /* TODO: This value may depend on resources received */ + scctx->isc_max_txqsets = scctx->isc_max_rxqsets = 16; /* Do PCI setup - map BAR0, etc */ if (ixlv_allocate_pci_resources(sc)) { @@ -317,6 +331,7 @@ INIT_DBG_DEV(dev, "Allocated PCI resources and MSIX vectors"); + /* XXX: This is called by init_shared_code in the PF driver */ error = i40e_set_mac_type(hw); if (error) { device_printf(dev, "%s: set_mac_type failed: %d\n", @@ -333,6 +348,7 @@ INIT_DBG_DEV(dev, "VF Device is ready for configuration"); + /* Sets up Admin Queue */ error = ixlv_setup_vc(sc); if (error) { device_printf(dev, "%s: Error setting up PF comms, %d\n", @@ -380,6 +396,7 @@ error = EIO; goto err_res_buf; } + vsi->id = sc->vsi_res->vsi_id; INIT_DBG_DEV(dev, "Resource Acquisition complete"); @@ -391,57 +408,60 @@ addr[0] |= 0x02; bcopy(addr, hw->mac.addr, sizeof(addr)); } + bcopy(hw->mac.addr, hw->mac.perm_addr, ETHER_ADDR_LEN); + iflib_set_mac(ctx, hw->mac.addr); - /* Now that the number of queues for this VF is known, set up interrupts */ - sc->msix = ixlv_init_msix(sc); - /* We fail without MSIX support */ - if (sc->msix == 0) { - error = ENXIO; - goto err_res_buf; - } + /* Allocate filter lists */ + ixlv_init_filters(sc); - vsi->id = sc->vsi_res->vsi_id; - vsi->back = (void *)sc; - sc->link_up = TRUE; + /* Fill out more iflib parameters */ + scctx->isc_txrx = &ixl_txrx; + // TODO: Probably needs changing + vsi->shared->isc_rss_table_size = sc->hw.func_caps.rss_table_size; + scctx->isc_tx_csum_flags = CSUM_OFFLOAD; + scctx->isc_capenable = IXL_CAPS; - /* This allocates the memory and early settings */ - if (ixlv_setup_queues(sc) != 0) { - device_printf(dev, "%s: setup queues failed!\n", - __func__); - error = EIO; - goto out; - } + INIT_DBG_DEV(dev, "end"); + return (0); - /* Setup the stack interface */ - if (ixlv_setup_interface(dev, sc) != 0) { - device_printf(dev, "%s: setup interface failed!\n", - __func__); - error = EIO; - goto out; - } +err_res_buf: + free(sc->vf_res, M_DEVBUF); +err_aq: + i40e_shutdown_adminq(hw); +err_pci_res: + ixlv_free_pci_resources(sc); +err_early: + ixlv_free_filters(sc); + INIT_DBG_DEV(dev, "end: error %d", error); + return (error); +} - INIT_DBG_DEV(dev, "Queue memory and interface setup"); +static int +ixlv_if_attach_post(if_ctx_t ctx) +{ + device_t dev; + struct ixlv_sc *sc; + struct i40e_hw *hw; + struct ixl_vsi *vsi; + int error = 0; - /* Do queue interrupt setup */ - if (ixlv_assign_msix(sc) != 0) { - device_printf(dev, "%s: allocating queue interrupts failed!\n", - __func__); - error = ENXIO; - goto out; - } + INIT_DBG_DEV(dev, "begin"); + + dev = iflib_get_dev(ctx); + vsi = iflib_get_softc(ctx); + vsi->ifp = iflib_get_ifp(ctx); + sc = (struct ixlv_sc *)vsi->back; + hw = &sc->hw; + + /* Setup the stack interface */ + ixlv_setup_interface(dev, vsi); - /* Start AdminQ taskqueue */ - ixlv_init_taskqueue(sc); + INIT_DBG_DEV(dev, "OS network interface set up."); - /* Initialize stats */ + /* Initialize statistics & add sysctls */ bzero(&sc->vsi.eth_stats, sizeof(struct i40e_eth_stats)); ixlv_add_sysctls(sc); - - /* Register for VLAN events */ - vsi->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, - ixlv_register_vlan, vsi, EVENTHANDLER_PRI_FIRST); - vsi->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, - ixlv_unregister_vlan, vsi, EVENTHANDLER_PRI_FIRST); + /* TODO: Add device sysctls? Like what? */ /* We want AQ enabled early */ ixlv_enable_adminq_irq(hw); @@ -454,68 +474,33 @@ INIT_DBG_DEV(dev, "end"); return (error); +// TODO: Check if any failures can happen above +#if 0 out: - ixlv_free_queues(vsi); -err_res_buf: free(sc->vf_res, M_DEVBUF); -err_aq: i40e_shutdown_adminq(hw); -err_pci_res: ixlv_free_pci_resources(sc); -err_early: - mtx_destroy(&sc->mtx); ixlv_free_filters(sc); INIT_DBG_DEV(dev, "end: error %d", error); return (error); +#endif } -/********************************************************************* - * Device removal routine - * - * The detach entry point is called when the driver is being removed. - * This routine stops the adapter and deallocates all the resources - * that were allocated for driver operation. - * - * return 0 on success, positive on failure - *********************************************************************/ - static int -ixlv_detach(device_t dev) +ixlv_if_detach(if_ctx_t ctx) { - struct ixlv_sc *sc = device_get_softc(dev); - struct ixl_vsi *vsi = &sc->vsi; - struct i40e_hw *hw = &sc->hw; - enum i40e_status_code status; + struct ixl_vsi *vsi = iflib_get_softc(ctx); + struct ixlv_sc *sc = vsi->back; + struct i40e_hw *hw = &sc->hw; + device_t dev = sc->dev; + enum i40e_status_code status; INIT_DBG_DEV(dev, "begin"); - /* Make sure VLANS are not using driver */ - if (vsi->ifp->if_vlantrunk != NULL) { - if_printf(vsi->ifp, "Vlan in use, detach first\n"); - return (EBUSY); - } - - /* Stop driver */ - ether_ifdetach(vsi->ifp); - if (vsi->ifp->if_drv_flags & IFF_DRV_RUNNING) { - mtx_lock(&sc->mtx); - ixlv_stop(sc); - mtx_unlock(&sc->mtx); - } - - /* Unregister VLAN events */ - if (vsi->vlan_attach != NULL) - EVENTHANDLER_DEREGISTER(vlan_config, vsi->vlan_attach); - if (vsi->vlan_detach != NULL) - EVENTHANDLER_DEREGISTER(vlan_unconfig, vsi->vlan_detach); - /* Drain VC mgr */ callout_drain(&sc->vc_mgr.callout); ixlv_disable_adminq_irq(hw); - ixlv_teardown_adminq_msix(sc); - /* Drain admin queue taskqueue */ - taskqueue_free(sc->tq); status = i40e_shutdown_adminq(&sc->hw); if (status != I40E_SUCCESS) { device_printf(dev, @@ -523,114 +508,56 @@ i40e_stat_str(hw, status)); } - if_free(vsi->ifp); free(sc->vf_res, M_DEVBUF); ixlv_free_pci_resources(sc); - ixlv_free_queues(vsi); ixlv_free_filters(sc); - - bus_generic_detach(dev); - mtx_destroy(&sc->mtx); INIT_DBG_DEV(dev, "end"); return (0); } -/********************************************************************* - * - * Shutdown entry point - * - **********************************************************************/ +/* TODO: Do shutdown-specific stuff here */ +static int +ixlv_if_shutdown(if_ctx_t ctx) +{ + int error = 0; + + INIT_DBG_DEV(dev, "begin"); + + /* TODO: Call ixl_if_stop()? */ + + return (error); +} +/* TODO: What is a VF supposed to do in suspend/resume? */ static int -ixlv_shutdown(device_t dev) +ixlv_if_suspend(if_ctx_t ctx) { - struct ixlv_sc *sc = device_get_softc(dev); + int error = 0; INIT_DBG_DEV(dev, "begin"); - mtx_lock(&sc->mtx); - ixlv_stop(sc); - mtx_unlock(&sc->mtx); + /* TODO: Call ixl_if_stop()? */ - INIT_DBG_DEV(dev, "end"); - return (0); + return (error); } -/* - * Configure TXCSUM(IPV6) and TSO(4/6) - * - the hardware handles these together so we - * need to tweak them - */ -static void -ixlv_cap_txcsum_tso(struct ixl_vsi *vsi, struct ifnet *ifp, int mask) -{ - /* Enable/disable TXCSUM/TSO4 */ - if (!(ifp->if_capenable & IFCAP_TXCSUM) - && !(ifp->if_capenable & IFCAP_TSO4)) { - if (mask & IFCAP_TXCSUM) { - ifp->if_capenable |= IFCAP_TXCSUM; - /* enable TXCSUM, restore TSO if previously enabled */ - if (vsi->flags & IXL_FLAGS_KEEP_TSO4) { - vsi->flags &= ~IXL_FLAGS_KEEP_TSO4; - ifp->if_capenable |= IFCAP_TSO4; - } - } - else if (mask & IFCAP_TSO4) { - ifp->if_capenable |= (IFCAP_TXCSUM | IFCAP_TSO4); - vsi->flags &= ~IXL_FLAGS_KEEP_TSO4; - if_printf(ifp, - "TSO4 requires txcsum, enabling both...\n"); - } - } else if((ifp->if_capenable & IFCAP_TXCSUM) - && !(ifp->if_capenable & IFCAP_TSO4)) { - if (mask & IFCAP_TXCSUM) - ifp->if_capenable &= ~IFCAP_TXCSUM; - else if (mask & IFCAP_TSO4) - ifp->if_capenable |= IFCAP_TSO4; - } else if((ifp->if_capenable & IFCAP_TXCSUM) - && (ifp->if_capenable & IFCAP_TSO4)) { - if (mask & IFCAP_TXCSUM) { - vsi->flags |= IXL_FLAGS_KEEP_TSO4; - ifp->if_capenable &= ~(IFCAP_TXCSUM | IFCAP_TSO4); - if_printf(ifp, - "TSO4 requires txcsum, disabling both...\n"); - } else if (mask & IFCAP_TSO4) - ifp->if_capenable &= ~IFCAP_TSO4; - } - - /* Enable/disable TXCSUM_IPV6/TSO6 */ - if (!(ifp->if_capenable & IFCAP_TXCSUM_IPV6) - && !(ifp->if_capenable & IFCAP_TSO6)) { - if (mask & IFCAP_TXCSUM_IPV6) { - ifp->if_capenable |= IFCAP_TXCSUM_IPV6; - if (vsi->flags & IXL_FLAGS_KEEP_TSO6) { - vsi->flags &= ~IXL_FLAGS_KEEP_TSO6; - ifp->if_capenable |= IFCAP_TSO6; - } - } else if (mask & IFCAP_TSO6) { - ifp->if_capenable |= (IFCAP_TXCSUM_IPV6 | IFCAP_TSO6); - vsi->flags &= ~IXL_FLAGS_KEEP_TSO6; - if_printf(ifp, - "TSO6 requires txcsum6, enabling both...\n"); - } - } else if((ifp->if_capenable & IFCAP_TXCSUM_IPV6) - && !(ifp->if_capenable & IFCAP_TSO6)) { - if (mask & IFCAP_TXCSUM_IPV6) - ifp->if_capenable &= ~IFCAP_TXCSUM_IPV6; - else if (mask & IFCAP_TSO6) - ifp->if_capenable |= IFCAP_TSO6; - } else if ((ifp->if_capenable & IFCAP_TXCSUM_IPV6) - && (ifp->if_capenable & IFCAP_TSO6)) { - if (mask & IFCAP_TXCSUM_IPV6) { - vsi->flags |= IXL_FLAGS_KEEP_TSO6; - ifp->if_capenable &= ~(IFCAP_TXCSUM_IPV6 | IFCAP_TSO6); - if_printf(ifp, - "TSO6 requires txcsum6, disabling both...\n"); - } else if (mask & IFCAP_TSO6) - ifp->if_capenable &= ~IFCAP_TSO6; - } +static int +ixlv_if_resume(if_ctx_t ctx) +{ + struct ifnet *ifp = iflib_get_ifp(ctx); + + INIT_DBG_DEV(dev, "begin"); + + /* Read & clear wake-up registers */ + + /* Required after D3->D0 transition */ + if (ifp->if_flags & IFF_UP) + ixlv_if_init(ctx); + + return (0); } +#if 0 /********************************************************************* * Ioctl entry point * @@ -772,6 +699,7 @@ return (error); } +#endif /* ** To do a reinit on the VF is unfortunately more complicated @@ -853,18 +781,21 @@ } } -static void -ixlv_init_locked(struct ixlv_sc *sc) +void +ixlv_if_init(if_ctx_t ctx) { - struct i40e_hw *hw = &sc->hw; - struct ixl_vsi *vsi = &sc->vsi; - struct ixl_queue *que = vsi->queues; - struct ifnet *ifp = vsi->ifp; - int error = 0; + struct ixl_vsi *vsi = iflib_get_softc(ctx); + if_softc_ctx_t scctx = vsi->shared; + struct ixlv_sc *sc = vsi->back; + struct i40e_hw *hw = &sc->hw; + struct ifnet *ifp = iflib_get_ifp(ctx); - INIT_DBG_IF(ifp, "begin"); + struct ixl_tx_queue *tx_que = vsi->tx_queues; + struct ixl_rx_queue *rx_que = vsi->rx_queues; + + int error = 0; - IXLV_CORE_LOCK_ASSERT(sc); + INIT_DBG_IF(ifp, "begin"); /* Do a reinit first if an init has already been done */ if ((sc->init_state == IXLV_RUNNING) || @@ -887,14 +818,6 @@ /* Check for an LAA mac address... */ bcopy(IF_LLADDR(ifp), hw->mac.addr, ETHER_ADDR_LEN); - ifp->if_hwassist = 0; - if (ifp->if_capenable & IFCAP_TSO) - ifp->if_hwassist |= CSUM_TSO; - if (ifp->if_capenable & IFCAP_TXCSUM) - ifp->if_hwassist |= (CSUM_OFFLOAD_IPV4 & ~CSUM_IP); - if (ifp->if_capenable & IFCAP_TXCSUM_IPV6) - ifp->if_hwassist |= CSUM_OFFLOAD_IPV6; - /* Add mac filter for this VF to PF */ if (i40e_validate_mac_addr(hw->mac.addr) == I40E_SUCCESS) { error = ixlv_add_mac_filter(sc, hw->mac.addr, 0); @@ -907,17 +830,19 @@ /* Setup vlan's if needed */ ixlv_setup_vlan_filters(sc); + // TODO: Functionize /* Prepare the queues for operation */ - for (int i = 0; i < vsi->num_queues; i++, que++) { - struct rx_ring *rxr = &que->rxr; - - ixl_init_tx_ring(que); + for (int i = 0; i < vsi->num_tx_queues; i++, tx_que++) { + // TODO: Necessary? Correct? + ixl_init_tx_ring(vsi, tx_que); + } + for (int i = 0; i < vsi->num_rx_queues; i++, rx_que++) { + struct rx_ring *rxr = &rx_que->rxr; - if (vsi->max_frame_size <= MCLBYTES) + if (scctx->isc_max_frame_size <= MCLBYTES) rxr->mbuf_sz = MCLBYTES; else rxr->mbuf_sz = MJUMPAGESIZE; - ixl_init_rx_ring(que); } /* Set initial ITR values */ @@ -938,9 +863,6 @@ ixl_vc_enqueue(&sc->vc_mgr, &sc->enable_queues_cmd, IXLV_FLAG_AQ_ENABLE_QUEUES, ixl_init_cmd_complete, sc); - /* Start the local timer */ - callout_reset(&sc->timer, hz, ixlv_local_timer, sc); - sc->init_state = IXLV_RUNNING; init_done: @@ -948,9 +870,7 @@ return; } -/* -** Init entry point for the stack -*/ +#if 0 void ixlv_init(void *arg) { @@ -984,29 +904,7 @@ sc->init_in_progress = false; mtx_unlock(&sc->mtx); } - -/* - * ixlv_attach() helper function; gathers information about - * the (virtual) hardware for use elsewhere in the driver. - */ -static void -ixlv_init_hw(struct ixlv_sc *sc) -{ - struct i40e_hw *hw = &sc->hw; - device_t dev = sc->dev; - - /* Save off the information about this board */ - hw->vendor_id = pci_get_vendor(dev); - hw->device_id = pci_get_device(dev); - hw->revision_id = pci_read_config(dev, PCIR_REVID, 1); - hw->subsystem_vendor_id = - pci_read_config(dev, PCIR_SUBVEND_0, 2); - hw->subsystem_device_id = - pci_read_config(dev, PCIR_SUBDEV_0, 2); - - hw->bus.device = pci_get_slot(dev); - hw->bus.func = pci_get_function(dev); -} +#endif /* * ixlv_attach() helper function; initalizes the admin queue @@ -1176,297 +1074,677 @@ return (ret_error); } -/* - * Allocate MSI/X vectors, setup the AQ vector early - */ static int -ixlv_init_msix(struct ixlv_sc *sc) +ixlv_if_msix_intr_assign(if_ctx_t ctx, int msix) { - device_t dev = sc->dev; - int rid, want, vectors, queues, available; - int auto_max_queues; + struct ixl_vsi *vsi = iflib_get_softc(ctx); + struct ixlv_sc *sc = vsi->back; + struct ixl_rx_queue *que = vsi->rx_queues; + struct ixl_tx_queue *tx_que = vsi->tx_queues; + int err, i, rid, vector = 0; + char buf[16]; + + /* Admin Que is vector 0*/ + rid = vector + 1; + + err = iflib_irq_alloc_generic(ctx, &vsi->irq, rid, IFLIB_INTR_ADMIN, + ixlv_msix_adminq, sc, 0, "aq"); + if (err) { + iflib_irq_free(ctx, &vsi->irq); + device_printf(iflib_get_dev(ctx), "Failed to register Admin que handler"); + return (err); + } + sc->admvec = vector; + ++vector; + + /* Now set up the stations */ + for (i = 0; i < vsi->num_rx_queues; i++, vector++, que++) { + rid = vector + 1; - rid = PCIR_BAR(IXL_MSIX_BAR); - sc->msix_mem = bus_alloc_resource_any(dev, - SYS_RES_MEMORY, &rid, RF_ACTIVE); - if (!sc->msix_mem) { - /* May not be enabled */ - device_printf(sc->dev, - "Unable to map MSIX table\n"); - goto fail; + snprintf(buf, sizeof(buf), "rxq%d", i); + err = iflib_irq_alloc_generic(ctx, &que->que_irq, rid, IFLIB_INTR_RX, + ixlv_msix_que, que, que->rxr.me, buf); + if (err) { + device_printf(iflib_get_dev(ctx), "Failed to allocate q int %d err: %d", i, err); + vsi->num_rx_queues = i + 1; + goto fail; + } + que->msix = vector; } - available = pci_msix_count(dev); - if (available == 0) { /* system has msix disabled */ - bus_release_resource(dev, SYS_RES_MEMORY, - rid, sc->msix_mem); - sc->msix_mem = NULL; - goto fail; - } - - /* Clamp queues to number of CPUs and # of MSI-X vectors available */ - auto_max_queues = min(mp_ncpus, available - 1); - /* Clamp queues to # assigned to VF by PF */ - auto_max_queues = min(auto_max_queues, sc->vf_res->num_queue_pairs); - - /* Override with tunable value if tunable is less than autoconfig count */ - if ((ixlv_max_queues != 0) && (ixlv_max_queues <= auto_max_queues)) - queues = ixlv_max_queues; - /* Use autoconfig amount if that's lower */ - else if ((ixlv_max_queues != 0) && (ixlv_max_queues > auto_max_queues)) { - device_printf(dev, "ixlv_max_queues (%d) is too large, using " - "autoconfig amount (%d)...\n", - ixlv_max_queues, auto_max_queues); - queues = auto_max_queues; - } - /* Limit maximum auto-configured queues to 8 if no user value is set */ - else - queues = min(auto_max_queues, 8); - -#ifdef RSS - /* If we're doing RSS, clamp at the number of RSS buckets */ - if (queues > rss_getnumbuckets()) - queues = rss_getnumbuckets(); -#endif - - /* - ** Want one vector (RX/TX pair) per queue - ** plus an additional for the admin queue. - */ - want = queues + 1; - if (want <= available) /* Have enough */ - vectors = want; - else { - device_printf(sc->dev, - "MSIX Configuration Problem, " - "%d vectors available but %d wanted!\n", - available, want); - goto fail; + for (i = 0, tx_que = vsi->tx_queues; i < vsi->num_tx_queues; i++, tx_que++) { + snprintf(buf, sizeof(buf), "txq%d", i); + rid = que->msix + 1; + iflib_softirq_alloc_generic(ctx, rid, IFLIB_INTR_TX, tx_que, tx_que->txr.me, buf); } -#ifdef RSS - /* - * If we're doing RSS, the number of queues needs to - * match the number of RSS buckets that are configured. - * - * + If there's more queues than RSS buckets, we'll end - * up with queues that get no traffic. - * - * + If there's more RSS buckets than queues, we'll end - * up having multiple RSS buckets map to the same queue, - * so there'll be some contention. - */ - if (queues != rss_getnumbuckets()) { - device_printf(dev, - "%s: queues (%d) != RSS buckets (%d)" - "; performance will be impacted.\n", - __func__, queues, rss_getnumbuckets()); - } -#endif + return (0); +fail: + iflib_irq_free(ctx, &vsi->irq); + que = vsi->rx_queues; + for (int i = 0; i < vsi->num_rx_queues; i++, que++) + iflib_irq_free(ctx, &que->que_irq); + return (err); +} - if (pci_alloc_msix(dev, &vectors) == 0) { - device_printf(sc->dev, - "Using MSIX interrupts with %d vectors\n", vectors); - sc->msix = vectors; - sc->vsi.num_queues = queues; - } +/* Enable all interrupts */ +static void +ixlv_if_enable_intr(if_ctx_t ctx) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); - /* Next we need to setup the vector for the Admin Queue */ - rid = 1; /* zero vector + 1 */ - sc->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, - &rid, RF_SHAREABLE | RF_ACTIVE); - if (sc->res == NULL) { - device_printf(dev, "Unable to allocate" - " bus resource: AQ interrupt \n"); - goto fail; - } - if (bus_setup_intr(dev, sc->res, - INTR_TYPE_NET | INTR_MPSAFE, NULL, - ixlv_msix_adminq, sc, &sc->tag)) { - sc->res = NULL; - device_printf(dev, "Failed to register AQ handler"); - goto fail; - } - bus_describe_intr(dev, sc->res, sc->tag, "adminq"); + ixlv_enable_intr(vsi); +} - return (vectors); +/* Disable all interrupts */ +static void +ixlv_if_disable_intr(if_ctx_t ctx) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); -fail: - /* The VF driver MUST use MSIX */ - return (0); + ixlv_disable_intr(vsi); } +/* Enable queue interrupt */ static int -ixlv_allocate_pci_resources(struct ixlv_sc *sc) +ixlv_if_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid) { - int rid; - device_t dev = sc->dev; + struct ixl_vsi *vsi = iflib_get_softc(ctx); + struct i40e_hw *hw = vsi->hw; + struct ixl_rx_queue *que = &vsi->rx_queues[rxqid]; - rid = PCIR_BAR(0); - sc->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, - &rid, RF_ACTIVE); + ixlv_enable_queue_irq(hw, que->rxr.me); - if (!(sc->pci_mem)) { - device_printf(dev, "Unable to allocate bus resource: memory\n"); - return (ENXIO); + return (0); +} + +static int +ixlv_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int ntxqs, int ntxqsets) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + struct ixl_tx_queue *que; + int i; + + MPASS(vsi->num_tx_queues > 0); + MPASS(ntxqs == 1); + MPASS(vsi->num_tx_queues == ntxqsets); + + /* Allocate queue structure memory */ + if (!(vsi->tx_queues = + (struct ixl_tx_queue *) malloc(sizeof(struct ixl_tx_queue) *ntxqsets, M_IXLV, M_NOWAIT | M_ZERO))) { + device_printf(iflib_get_dev(ctx), "Unable to allocate TX ring memory\n"); + return (ENOMEM); } + + for (i = 0, que = vsi->tx_queues; i < ntxqsets; i++, que++) { + struct tx_ring *txr = &que->txr; - sc->osdep.mem_bus_space_tag = - rman_get_bustag(sc->pci_mem); - sc->osdep.mem_bus_space_handle = - rman_get_bushandle(sc->pci_mem); - sc->osdep.mem_bus_space_size = rman_get_size(sc->pci_mem); - sc->osdep.flush_reg = I40E_VFGEN_RSTAT; - sc->hw.hw_addr = (u8 *) &sc->osdep.mem_bus_space_handle; + txr->me = i; + que->vsi = vsi; - sc->hw.back = &sc->osdep; + /* get the virtual and physical address of the hardware queues */ + txr->tail = I40E_QTX_TAIL1(txr->me); + txr->tx_base = (struct i40e_tx_desc *)vaddrs[i]; + txr->tx_paddr = paddrs[i]; + txr->que = que; + } + + // TODO: Do a config_gtask_init for admin queue here? + // iflib_config_gtask_init(ctx, &adapter->mod_task, ixgbe_handle_mod, "mod_task"); - /* - ** Explicitly set the guest PCI BUSMASTER capability - ** and we must rewrite the ENABLE in the MSIX control - ** register again at this point to cause the host to - ** successfully initialize us. - ** - ** This must be set before accessing any registers. - */ - { - u16 pci_cmd_word; - int msix_ctrl; - pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); - pci_cmd_word |= PCIM_CMD_BUSMASTEREN; - pci_write_config(dev, PCIR_COMMAND, pci_cmd_word, 2); - pci_find_cap(dev, PCIY_MSIX, &rid); - rid += PCIR_MSIX_CTRL; - msix_ctrl = pci_read_config(dev, rid, 2); - msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE; - pci_write_config(dev, rid, msix_ctrl, 2); + device_printf(iflib_get_dev(ctx), "%s: allocated for %d txqs\n", __func__, vsi->num_tx_queues); + return (0); +} + +static int +ixlv_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nrxqs, int nrxqsets) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + struct ixl_rx_queue *que; + int i; + + MPASS(vsi->num_rx_queues > 0); + MPASS(nrxqs == 1); + MPASS(vsi->num_rx_queues == nrxqsets); + + /* Allocate queue structure memory */ + if (!(vsi->rx_queues = + (struct ixl_rx_queue *) malloc(sizeof(struct ixl_rx_queue) * + nrxqsets, M_IXLV, M_NOWAIT | M_ZERO))) { + device_printf(iflib_get_dev(ctx), "Unable to allocate RX ring memory\n"); + return (ENOMEM); } - /* Disable adminq interrupts (just in case) */ - ixlv_disable_adminq_irq(&sc->hw); + for (i = 0, que = vsi->rx_queues; i < nrxqsets; i++, que++) { + struct rx_ring *rxr = &que->rxr; + + rxr->me = i; + que->vsi = vsi; + + /* get the virtual and physical address of the hardware queues */ + rxr->tail = I40E_QRX_TAIL1(rxr->me); + rxr->rx_base = (union i40e_rx_desc *)vaddrs[i]; + rxr->rx_paddr = paddrs[i]; + rxr->que = que; + } + device_printf(iflib_get_dev(ctx), "%s: allocated for %d rxqs\n", __func__, vsi->num_rx_queues); return (0); } static void -ixlv_free_pci_resources(struct ixlv_sc *sc) +ixlv_if_queues_free(if_ctx_t ctx) { - struct ixl_vsi *vsi = &sc->vsi; - struct ixl_queue *que = vsi->queues; - device_t dev = sc->dev; + struct ixl_vsi *vsi = iflib_get_softc(ctx); - /* We may get here before stations are setup */ - if (que == NULL) - goto early; + if (vsi->tx_queues != NULL) { + free(vsi->tx_queues, M_IXLV); + vsi->tx_queues = NULL; + } + if (vsi->rx_queues != NULL) { + free(vsi->rx_queues, M_IXLV); + vsi->rx_queues = NULL; + } +} - /* - ** Release all msix queue resources: - */ - for (int i = 0; i < vsi->num_queues; i++, que++) { - int rid = que->msix + 1; - if (que->tag != NULL) { - bus_teardown_intr(dev, que->res, que->tag); - que->tag = NULL; +// TODO: Implement +static void +ixlv_if_update_admin_status(if_ctx_t ctx) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + //struct ixlv_sc *sc = vsi->back; + //struct i40e_hw *hw = &sc->hw; + //struct i40e_arq_event_info event; + //i40e_status ret; + //u32 loop = 0; + //u16 opcode + u16 result = 0; + //u64 baudrate; + + /* TODO: Split up + * - Update admin queue stuff + * - Update link status + * - Enqueue aq task + * - Re-enable admin intr + */ + +/* TODO: Does VF reset need to be handled here? */ +#if 0 + if (pf->state & IXL_PF_STATE_EMPR_RESETTING) { + /* Flag cleared at end of this function */ + ixl_handle_empr_reset(pf); + return; + } +#endif + +#if 0 + event.buf_len = IXL_AQ_BUF_SZ; + event.msg_buf = malloc(event.buf_len, + M_IXLV, M_NOWAIT | M_ZERO); + if (!event.msg_buf) { + device_printf(pf->dev, "%s: Unable to allocate memory for Admin" + " Queue event!\n", __func__); + return; + } + + /* clean and process any events */ + do { + ret = i40e_clean_arq_element(hw, &event, &result); + if (ret) + break; + opcode = LE16_TO_CPU(event.desc.opcode); + ixl_dbg(pf, IXL_DBG_AQ, + "Admin Queue event: %#06x\n", opcode); + switch (opcode) { + case i40e_aqc_opc_get_link_status: + ixl_link_event(pf, &event); + break; + case i40e_aqc_opc_send_msg_to_pf: +#ifdef PCI_IOV + ixl_handle_vf_msg(pf, &event); +#endif + break; + case i40e_aqc_opc_event_lan_overflow: + break; + default: +#ifdef IXL_DEBUG + printf("AdminQ unknown event %x\n", opcode); +#endif + break; + } + + } while (result && (loop++ < IXL_ADM_LIMIT)); + + free(event.msg_buf, M_IXLV); +#endif + +#if 0 + /* XXX: This updates the link status */ + if (pf->link_up) { + if (vsi->link_active == FALSE) { + vsi->link_active = TRUE; + baudrate = ixl_max_aq_speed_to_value(pf->link_speed); + iflib_link_state_change(ctx, LINK_STATE_UP, baudrate); + ixl_link_up_msg(pf); + // ixl_ping_all_vfs(adapter); + } - if (que->res != NULL) { - bus_release_resource(dev, SYS_RES_IRQ, rid, que->res); - que->res = NULL; + } else { /* Link down */ + if (vsi->link_active == TRUE) { + vsi->link_active = FALSE; + iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); + // ixl_ping_all_vfs(adapter); } } - -early: - pci_release_msi(dev); +#endif + + /* + * If there are still messages to process, reschedule ourselves. + * Otherwise, re-enable our interrupt and go to sleep. + */ + if (result > 0) + iflib_admin_intr_deferred(ctx); + else + /* TODO: Link/adminq interrupt should be re-enabled in IFDI_LINK_INTR_ENABLE */ + ixlv_enable_intr(vsi); +} - if (sc->msix_mem != NULL) - bus_release_resource(dev, SYS_RES_MEMORY, - PCIR_BAR(IXL_MSIX_BAR), sc->msix_mem); +static void +ixlv_if_multi_set(if_ctx_t ctx) +{ + // struct ixl_vsi *vsi = iflib_get_softc(ctx); + // struct i40e_hw *hw = vsi->hw; + // struct ixlv_sc *sc = vsi->back; + // int mcnt = 0, flags; - if (sc->pci_mem != NULL) - bus_release_resource(dev, SYS_RES_MEMORY, - PCIR_BAR(0), sc->pci_mem); + IOCTL_DEBUGOUT("ixl_if_multi_set: begin"); + + // TODO: Implement +#if 0 + mcnt = if_multiaddr_count(iflib_get_ifp(ctx), MAX_MULTICAST_ADDR); + /* delete existing MC filters */ + ixlv_del_multi(vsi); + + if (__predict_false(mcnt == MAX_MULTICAST_ADDR)) { + // Set promiscuous mode (multicast) + // TODO: This needs to get handled somehow +#if 0 + ixl_vc_enqueue(&sc->vc_mgr, &sc->add_vlan_cmd, + IXLV_FLAG_AQ_CONFIGURE_PROMISC, ixl_init_cmd_complete, sc); +#endif + return; + } + /* (re-)install filters for all mcast addresses */ + mcnt = if_multi_apply(iflib_get_ifp(ctx), ixl_mc_filter_apply, vsi); + + if (mcnt > 0) { + flags = (IXL_FILTER_ADD | IXL_FILTER_USED | IXL_FILTER_MC); + ixlv_add_hw_filters(vsi, flags, mcnt); + } +#endif + + IOCTL_DEBUGOUT("ixl_if_multi_set: end"); } -/* - * Create taskqueue and tasklet for Admin Queue interrupts. - */ static int -ixlv_init_taskqueue(struct ixlv_sc *sc) +ixlv_if_mtu_set(if_ctx_t ctx, uint32_t mtu) { - int error = 0; + struct ixl_vsi *vsi = iflib_get_softc(ctx); - TASK_INIT(&sc->aq_irq, 0, ixlv_do_adminq, sc); + IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); + if (mtu > IXL_MAX_FRAME - ETHER_HDR_LEN - ETHER_CRC_LEN - + ETHER_VLAN_ENCAP_LEN) + return (EINVAL); - sc->tq = taskqueue_create_fast("ixl_adm", M_NOWAIT, - taskqueue_thread_enqueue, &sc->tq); - taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s sc->tq", - device_get_nameunit(sc->dev)); + vsi->shared->isc_max_frame_size = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN + + ETHER_VLAN_ENCAP_LEN; - return (error); + return (0); } -/********************************************************************* - * - * Setup MSIX Interrupt resources and handlers for the VSI queues - * - **********************************************************************/ -static int -ixlv_assign_msix(struct ixlv_sc *sc) +static void +ixlv_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr) { - device_t dev = sc->dev; - struct ixl_vsi *vsi = &sc->vsi; - struct ixl_queue *que = vsi->queues; - struct tx_ring *txr; - int error, rid, vector = 1; -#ifdef RSS - cpuset_t cpu_mask; -#endif + struct ixl_vsi *vsi = iflib_get_softc(ctx); + struct ixlv_sc *sc = (struct ixlv_sc *)vsi->back; + struct i40e_hw *hw = &sc->hw; - for (int i = 0; i < vsi->num_queues; i++, vector++, que++) { - int cpu_id = i; - rid = vector + 1; - txr = &que->txr; - que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, - RF_SHAREABLE | RF_ACTIVE); - if (que->res == NULL) { - device_printf(dev,"Unable to allocate" - " bus resource: que interrupt [%d]\n", vector); - return (ENXIO); - } - /* Set the handler function */ - error = bus_setup_intr(dev, que->res, - INTR_TYPE_NET | INTR_MPSAFE, NULL, - ixlv_msix_que, que, &que->tag); - if (error) { - que->res = NULL; - device_printf(dev, "Failed to register que handler"); - return (error); - } - bus_describe_intr(dev, que->res, que->tag, "que %d", i); - /* Bind the vector to a CPU */ -#ifdef RSS - cpu_id = rss_getcpu(i % rss_getnumbuckets()); + INIT_DEBUGOUT("ixl_media_status: begin"); + + hw->phy.get_link_info = TRUE; + i40e_get_link_status(hw, &sc->link_up); + + ifmr->ifm_status = IFM_AVALID; + ifmr->ifm_active = IFM_ETHER; + + if (!sc->link_up) { + return; + } + + ifmr->ifm_status |= IFM_ACTIVE; + /* Hardware is always full-duplex */ + ifmr->ifm_active |= IFM_FDX; + + // TODO: Check another variable to get link speed +#if 0 + switch (hw->phy.link_info.phy_type) { + /* 100 M */ + case I40E_PHY_TYPE_100BASE_TX: + ifmr->ifm_active |= IFM_100_TX; + break; + /* 1 G */ + case I40E_PHY_TYPE_1000BASE_T: + ifmr->ifm_active |= IFM_1000_T; + break; + case I40E_PHY_TYPE_1000BASE_SX: + ifmr->ifm_active |= IFM_1000_SX; + break; + case I40E_PHY_TYPE_1000BASE_LX: + ifmr->ifm_active |= IFM_1000_LX; + break; + case I40E_PHY_TYPE_1000BASE_T_OPTICAL: + ifmr->ifm_active |= IFM_OTHER; + break; + /* 10 G */ + case I40E_PHY_TYPE_10GBASE_SFPP_CU: + ifmr->ifm_active |= IFM_10G_TWINAX; + break; + case I40E_PHY_TYPE_10GBASE_SR: + ifmr->ifm_active |= IFM_10G_SR; + break; + case I40E_PHY_TYPE_10GBASE_LR: + ifmr->ifm_active |= IFM_10G_LR; + break; + case I40E_PHY_TYPE_10GBASE_T: + ifmr->ifm_active |= IFM_10G_T; + break; + case I40E_PHY_TYPE_XAUI: + case I40E_PHY_TYPE_XFI: + case I40E_PHY_TYPE_10GBASE_AOC: + ifmr->ifm_active |= IFM_OTHER; + break; + /* 25 G */ + case I40E_PHY_TYPE_25GBASE_KR: + ifmr->ifm_active |= IFM_25G_KR; + break; + case I40E_PHY_TYPE_25GBASE_CR: + ifmr->ifm_active |= IFM_25G_CR; + break; + case I40E_PHY_TYPE_25GBASE_SR: + ifmr->ifm_active |= IFM_25G_SR; + break; + case I40E_PHY_TYPE_25GBASE_LR: + ifmr->ifm_active |= IFM_UNKNOWN; + break; + /* 40 G */ + case I40E_PHY_TYPE_40GBASE_CR4: + case I40E_PHY_TYPE_40GBASE_CR4_CU: + ifmr->ifm_active |= IFM_40G_CR4; + break; + case I40E_PHY_TYPE_40GBASE_SR4: + ifmr->ifm_active |= IFM_40G_SR4; + break; + case I40E_PHY_TYPE_40GBASE_LR4: + ifmr->ifm_active |= IFM_40G_LR4; + break; + case I40E_PHY_TYPE_XLAUI: + ifmr->ifm_active |= IFM_OTHER; + break; + case I40E_PHY_TYPE_1000BASE_KX: + ifmr->ifm_active |= IFM_1000_KX; + break; + case I40E_PHY_TYPE_SGMII: + ifmr->ifm_active |= IFM_1000_SGMII; + break; + /* ERJ: What's the difference between these? */ + case I40E_PHY_TYPE_10GBASE_CR1_CU: + case I40E_PHY_TYPE_10GBASE_CR1: + ifmr->ifm_active |= IFM_10G_CR1; + break; + case I40E_PHY_TYPE_10GBASE_KX4: + ifmr->ifm_active |= IFM_10G_KX4; + break; + case I40E_PHY_TYPE_10GBASE_KR: + ifmr->ifm_active |= IFM_10G_KR; + break; + case I40E_PHY_TYPE_SFI: + ifmr->ifm_active |= IFM_10G_SFI; + break; + /* Our single 20G media type */ + case I40E_PHY_TYPE_20GBASE_KR2: + ifmr->ifm_active |= IFM_20G_KR2; + break; + case I40E_PHY_TYPE_40GBASE_KR4: + ifmr->ifm_active |= IFM_40G_KR4; + break; + case I40E_PHY_TYPE_XLPPI: + case I40E_PHY_TYPE_40GBASE_AOC: + ifmr->ifm_active |= IFM_40G_XLPPI; + break; + /* Unknown to driver */ + default: + ifmr->ifm_active |= IFM_UNKNOWN; + break; + } + /* Report flow control status as well */ + if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_TX) + ifmr->ifm_active |= IFM_ETH_TXPAUSE; + if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_RX) + ifmr->ifm_active |= IFM_ETH_RXPAUSE; #endif - bus_bind_intr(dev, que->res, cpu_id); - que->msix = vector; - TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que); - TASK_INIT(&que->task, 0, ixlv_handle_que, que); - que->tq = taskqueue_create_fast("ixlv_que", M_NOWAIT, - taskqueue_thread_enqueue, &que->tq); -#ifdef RSS - CPU_SETOF(cpu_id, &cpu_mask); - taskqueue_start_threads_cpuset(&que->tq, 1, PI_NET, - &cpu_mask, "%s (bucket %d)", - device_get_nameunit(dev), cpu_id); -#else - taskqueue_start_threads(&que->tq, 1, PI_NET, - "%s que", device_get_nameunit(dev)); +} + +static int +ixlv_if_media_change(if_ctx_t ctx) +{ + struct ifmedia *ifm = iflib_get_media(ctx); + + INIT_DEBUGOUT("ixl_media_change: begin"); + + if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) + return (EINVAL); + + if_printf(iflib_get_ifp(ctx), "Media change is not supported.\n"); + return (ENODEV); +} + +// TODO: Rework +static int +ixlv_if_promisc_set(if_ctx_t ctx, int flags) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + struct ifnet *ifp = iflib_get_ifp(ctx); + struct i40e_hw *hw = vsi->hw; + int err; + bool uni = FALSE, multi = FALSE; + + if (flags & IFF_ALLMULTI || + if_multiaddr_count(ifp, MAX_MULTICAST_ADDR) == MAX_MULTICAST_ADDR) + multi = TRUE; + if (flags & IFF_PROMISC) + uni = TRUE; + + err = i40e_aq_set_vsi_unicast_promiscuous(hw, + vsi->seid, uni, NULL, false); + if (err) + return (err); + err = i40e_aq_set_vsi_multicast_promiscuous(hw, + vsi->seid, multi, NULL); + return (err); +} + +static void +ixlv_if_timer(if_ctx_t ctx, uint16_t qid) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + struct ixlv_sc *sc = vsi->back; + //struct i40e_hw *hw = &sc->hw; + //struct ixl_tx_queue *que = &vsi->tx_queues[qid]; + //u32 mask; + +#if 0 + /* + ** Check status of the queues + */ + mask = (I40E_PFINT_DYN_CTLN_INTENA_MASK | + I40E_PFINT_DYN_CTLN_SWINT_TRIG_MASK); + + /* If queue param has outstanding work, trigger sw irq */ + // TODO: TX queues in iflib don't use HW interrupts; does this do anything? + if (que->busy) + wr32(hw, I40E_PFINT_DYN_CTLN(que->txr.me), mask); #endif + // XXX: Is this timer per-queue? + if (qid != 0) + return; + + /* Fire off the adminq task */ + iflib_admin_intr_deferred(ctx); + + /* Update stats */ + ixlv_request_stats(sc); +} + +static void +ixlv_if_vlan_register(if_ctx_t ctx, u16 vtag) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + //struct i40e_hw *hw = vsi->hw; + + if ((vtag == 0) || (vtag > 4095)) /* Invalid */ + return; + + ++vsi->num_vlans; + // TODO: Redo + // ixlv_add_filter(vsi, hw->mac.addr, vtag); +} + +static void +ixlv_if_vlan_unregister(if_ctx_t ctx, u16 vtag) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + //struct i40e_hw *hw = vsi->hw; + + if ((vtag == 0) || (vtag > 4095)) /* Invalid */ + return; + + --vsi->num_vlans; + // TODO: Redo + // ixlv_del_filter(vsi, hw->mac.addr, vtag); +} + +static uint64_t +ixlv_if_get_counter(if_ctx_t ctx, ift_counter cnt) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + if_t ifp = iflib_get_ifp(ctx); + + switch (cnt) { + case IFCOUNTER_IPACKETS: + return (vsi->ipackets); + case IFCOUNTER_IERRORS: + return (vsi->ierrors); + case IFCOUNTER_OPACKETS: + return (vsi->opackets); + case IFCOUNTER_OERRORS: + return (vsi->oerrors); + case IFCOUNTER_COLLISIONS: + /* Collisions are by standard impossible in 40G/10G Ethernet */ + return (0); + case IFCOUNTER_IBYTES: + return (vsi->ibytes); + case IFCOUNTER_OBYTES: + return (vsi->obytes); + case IFCOUNTER_IMCASTS: + return (vsi->imcasts); + case IFCOUNTER_OMCASTS: + return (vsi->omcasts); + case IFCOUNTER_IQDROPS: + return (vsi->iqdrops); + case IFCOUNTER_OQDROPS: + return (vsi->oqdrops); + case IFCOUNTER_NOPROTO: + return (vsi->noproto); + default: + return (if_get_counter_default(ifp, cnt)); + } +} + +static int +ixlv_allocate_pci_resources(struct ixlv_sc *sc) +{ + struct i40e_hw *hw = &sc->hw; + device_t dev = iflib_get_dev(sc->vsi.ctx); + int rid; + + /* Map BAR0 */ + rid = PCIR_BAR(0); + sc->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, + &rid, RF_ACTIVE); + + if (!(sc->pci_mem)) { + device_printf(dev, "Unable to allocate bus resource: PCI memory\n"); + return (ENXIO); } + /* Save off the PCI information */ + hw->vendor_id = pci_get_vendor(dev); + hw->device_id = pci_get_device(dev); + hw->revision_id = pci_read_config(dev, PCIR_REVID, 1); + hw->subsystem_vendor_id = + pci_read_config(dev, PCIR_SUBVEND_0, 2); + hw->subsystem_device_id = + pci_read_config(dev, PCIR_SUBDEV_0, 2); + + hw->bus.device = pci_get_slot(dev); + hw->bus.func = pci_get_function(dev); + + /* Save off register access information */ + sc->osdep.mem_bus_space_tag = + rman_get_bustag(sc->pci_mem); + sc->osdep.mem_bus_space_handle = + rman_get_bushandle(sc->pci_mem); + sc->osdep.mem_bus_space_size = rman_get_size(sc->pci_mem); + sc->osdep.flush_reg = I40E_VFGEN_RSTAT; + sc->osdep.dev = dev; + + sc->hw.hw_addr = (u8 *) &sc->osdep.mem_bus_space_handle; + sc->hw.back = &sc->osdep; + + /* Disable adminq interrupts (just in case) */ + /* TODO: Probably not necessary */ + // ixlv_disable_adminq_irq(&sc->hw); + return (0); } +static void +ixlv_free_pci_resources(struct ixlv_sc *sc) +{ + struct ixl_vsi *vsi = &sc->vsi; + struct ixl_rx_queue *rx_que = vsi->rx_queues; + device_t dev = sc->dev; + + /* We may get here before stations are setup */ + // TODO: Check if we can still check against sc->msix + if ((sc->msix > 0) || (rx_que == NULL)) + goto early; + + /* + ** Release all msix VSI resources: + */ + iflib_irq_free(vsi->ctx, &vsi->irq); + + for (int i = 0; i < vsi->num_rx_queues; i++, rx_que++) + iflib_irq_free(vsi->ctx, &rx_que->que_irq); + +early: + if (sc->pci_mem != NULL) + bus_release_resource(dev, SYS_RES_MEMORY, + PCIR_BAR(0), sc->pci_mem); +} + /* ** Requests a VF reset from the PF. ** @@ -1533,214 +1811,46 @@ * Setup networking device structure and register an interface. * **********************************************************************/ -static int -ixlv_setup_interface(device_t dev, struct ixlv_sc *sc) -{ - struct ifnet *ifp; - struct ixl_vsi *vsi = &sc->vsi; - struct ixl_queue *que = vsi->queues; - - INIT_DBG_DEV(dev, "begin"); - - ifp = vsi->ifp = if_alloc(IFT_ETHER); - if (ifp == NULL) { - device_printf(dev, "%s: could not allocate ifnet" - " structure!\n", __func__); - return (-1); - } - - if_initname(ifp, device_get_name(dev), device_get_unit(dev)); - - ifp->if_mtu = ETHERMTU; - ifp->if_baudrate = IF_Gbps(40); - ifp->if_init = ixlv_init; - ifp->if_softc = vsi; - ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; - ifp->if_ioctl = ixlv_ioctl; - -#if __FreeBSD_version >= 1100000 - if_setgetcounterfn(ifp, ixl_get_counter); -#endif - - ifp->if_transmit = ixl_mq_start; - - ifp->if_qflush = ixl_qflush; - ifp->if_snd.ifq_maxlen = que->num_desc - 2; - - ether_ifattach(ifp, sc->hw.mac.addr); - - vsi->max_frame_size = - ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN - + ETHER_VLAN_ENCAP_LEN; - - /* - * Tell the upper layer(s) we support long frames. - */ - ifp->if_hdrlen = sizeof(struct ether_vlan_header); - - ifp->if_capabilities |= IFCAP_HWCSUM; - ifp->if_capabilities |= IFCAP_HWCSUM_IPV6; - ifp->if_capabilities |= IFCAP_TSO; - ifp->if_capabilities |= IFCAP_JUMBO_MTU; - - ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING - | IFCAP_VLAN_HWTSO - | IFCAP_VLAN_MTU - | IFCAP_VLAN_HWCSUM - | IFCAP_LRO; - ifp->if_capenable = ifp->if_capabilities; - - /* - ** Don't turn this on by default, if vlans are - ** created on another pseudo device (eg. lagg) - ** then vlan events are not passed thru, breaking - ** operation, but with HW FILTER off it works. If - ** using vlans directly on the ixl driver you can - ** enable this and get full hardware tag filtering. - */ - ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; - - /* - * Specify the media types supported by this adapter and register - * callbacks to update media and link information - */ - ifmedia_init(&sc->media, IFM_IMASK, ixlv_media_change, - ixlv_media_status); - - // JFV Add media types later? - - ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); - ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); - - INIT_DBG_DEV(dev, "end"); - return (0); -} - -/* -** Allocate and setup the interface queues -*/ -static int -ixlv_setup_queues(struct ixlv_sc *sc) -{ - device_t dev = sc->dev; - struct ixl_vsi *vsi; - struct ixl_queue *que; - struct tx_ring *txr; - struct rx_ring *rxr; - int rsize, tsize; - int error = I40E_SUCCESS; - - vsi = &sc->vsi; - vsi->back = (void *)sc; - vsi->hw = &sc->hw; - vsi->num_vlans = 0; - - /* Get memory for the station queues */ - if (!(vsi->queues = - (struct ixl_queue *) malloc(sizeof(struct ixl_queue) * - vsi->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { - device_printf(dev, "Unable to allocate queue memory\n"); - error = ENOMEM; - goto early; - } - - for (int i = 0; i < vsi->num_queues; i++) { - que = &vsi->queues[i]; - que->num_desc = ixlv_ringsz; - que->me = i; - que->vsi = vsi; - - txr = &que->txr; - txr->que = que; - txr->tail = I40E_QTX_TAIL1(que->me); - /* Initialize the TX lock */ - snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", - device_get_nameunit(dev), que->me); - mtx_init(&txr->mtx, txr->mtx_name, NULL, MTX_DEF); - /* - ** Create the TX descriptor ring, the extra int is - ** added as the location for HEAD WB. - */ - tsize = roundup2((que->num_desc * - sizeof(struct i40e_tx_desc)) + - sizeof(u32), DBA_ALIGN); - if (i40e_allocate_dma_mem(&sc->hw, - &txr->dma, i40e_mem_reserved, tsize, DBA_ALIGN)) { - device_printf(dev, - "Unable to allocate TX Descriptor memory\n"); - error = ENOMEM; - goto fail; - } - txr->base = (struct i40e_tx_desc *)txr->dma.va; - bzero((void *)txr->base, tsize); - /* Now allocate transmit soft structs for the ring */ - if (ixl_allocate_tx_data(que)) { - device_printf(dev, - "Critical Failure setting up TX structures\n"); - error = ENOMEM; - goto fail; - } - /* Allocate a buf ring */ - txr->br = buf_ring_alloc(ixlv_txbrsz, M_DEVBUF, - M_WAITOK, &txr->mtx); - if (txr->br == NULL) { - device_printf(dev, - "Critical Failure setting up TX buf ring\n"); - error = ENOMEM; - goto fail; - } - - /* - * Next the RX queues... - */ - rsize = roundup2(que->num_desc * - sizeof(union i40e_rx_desc), DBA_ALIGN); - rxr = &que->rxr; - rxr->que = que; - rxr->tail = I40E_QRX_TAIL1(que->me); - - /* Initialize the RX side lock */ - snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", - device_get_nameunit(dev), que->me); - mtx_init(&rxr->mtx, rxr->mtx_name, NULL, MTX_DEF); +static void +ixlv_setup_interface(device_t dev, struct ixl_vsi *vsi) +{ + if_ctx_t ctx = vsi->ctx; + struct ixlv_sc *sc = vsi->back; + struct ifnet *ifp = iflib_get_ifp(ctx); + uint64_t cap; + // struct ixl_queue *que = vsi->queues; - if (i40e_allocate_dma_mem(&sc->hw, - &rxr->dma, i40e_mem_reserved, rsize, 4096)) { //JFV - should this be DBA? - device_printf(dev, - "Unable to allocate RX Descriptor memory\n"); - error = ENOMEM; - goto fail; - } - rxr->base = (union i40e_rx_desc *)rxr->dma.va; - bzero((void *)rxr->base, rsize); + INIT_DBG_DEV(dev, "begin"); - /* Allocate receive soft structs for the ring */ - if (ixl_allocate_rx_data(que)) { - device_printf(dev, - "Critical Failure setting up receive structs\n"); - error = ENOMEM; - goto fail; - } - } + /* initialize fast path functions */ + cap = IXL_CAPS; + if_setifheaderlen(ifp, sizeof(struct ether_vlan_header)); + if_setcapabilitiesbit(ifp, cap, 0); + if_setcapenable(ifp, if_getcapabilities(ifp)); + if_setbaudrate(ifp, IF_Gbps(40)); + /* TODO: Remove VLAN_ENCAP_LEN? */ + vsi->shared->isc_max_frame_size = + ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN + + ETHER_VLAN_ENCAP_LEN; - return (0); + /* + ** Don't turn this on by default, if vlans are + ** created on another pseudo device (eg. lagg) + ** then vlan events are not passed thru, breaking + ** operation, but with HW FILTER off it works. If + ** using vlans directly on the ixl driver you can + ** enable this and get full hardware tag filtering. + */ + if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWFILTER, 0); -fail: - for (int i = 0; i < vsi->num_queues; i++) { - que = &vsi->queues[i]; - rxr = &que->rxr; - txr = &que->txr; - if (rxr->base) - i40e_free_dma_mem(&sc->hw, &rxr->dma); - if (txr->base) - i40e_free_dma_mem(&sc->hw, &txr->dma); - } - free(vsi->queues, M_DEVBUF); + /* Use autoselect media by default */ + ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); + ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); -early: - return (error); + INIT_DBG_DEV(dev, "end"); } +#if 0 /* ** This routine is run via an vlan config EVENT, ** it enables us to use the HW Filter table since @@ -1779,7 +1889,9 @@ mtx_unlock(&sc->mtx); return; } +#endif +#if 0 /* ** This routine is run via an vlan ** unconfig EVENT, remove our entry @@ -1813,6 +1925,7 @@ mtx_unlock(&sc->mtx); return; } +#endif /* ** Get a new filter and add it to the mac filter list. @@ -1851,77 +1964,54 @@ return (f); } -static int -ixlv_teardown_adminq_msix(struct ixlv_sc *sc) -{ - device_t dev = sc->dev; - int error = 0; - - if (sc->tag != NULL) { - bus_teardown_intr(dev, sc->res, sc->tag); - if (error) { - device_printf(dev, "bus_teardown_intr() for" - " interrupt 0 failed\n"); - // return (ENXIO); - } - sc->tag = NULL; - } - if (sc->res != NULL) { - bus_release_resource(dev, SYS_RES_IRQ, 1, sc->res); - if (error) { - device_printf(dev, "bus_release_resource() for" - " interrupt 0 failed\n"); - // return (ENXIO); - } - sc->res = NULL; - } - - return (0); - -} - /* ** Admin Queue interrupt handler */ -static void +static int ixlv_msix_adminq(void *arg) { struct ixlv_sc *sc = arg; struct i40e_hw *hw = &sc->hw; - u32 reg, mask; + // device_t dev = sc->dev; + u32 reg; + bool do_task = FALSE; + + ++sc->admin_irq; reg = rd32(hw, I40E_VFINT_ICR01); - mask = rd32(hw, I40E_VFINT_ICR0_ENA1); - reg = rd32(hw, I40E_VFINT_DYN_CTL01); - reg |= I40E_VFINT_DYN_CTL01_CLEARPBA_MASK; - wr32(hw, I40E_VFINT_DYN_CTL01, reg); + /* Check on the cause */ + if (reg & I40E_VFINT_ICR0_ADMINQ_MASK) + do_task = TRUE; - /* schedule task */ - taskqueue_enqueue(sc->tq, &sc->aq_irq); - return; + if (do_task) + iflib_admin_intr_deferred(sc->vsi.ctx); + else + ixlv_enable_adminq_irq(hw); + + return (FILTER_HANDLED); } void ixlv_enable_intr(struct ixl_vsi *vsi) { struct i40e_hw *hw = vsi->hw; - struct ixl_queue *que = vsi->queues; + struct ixl_rx_queue *que = vsi->rx_queues; ixlv_enable_adminq_irq(hw); - for (int i = 0; i < vsi->num_queues; i++, que++) - ixlv_enable_queue_irq(hw, que->me); + for (int i = 0; i < vsi->num_rx_queues; i++, que++) + ixlv_enable_queue_irq(hw, que->rxr.me); } void ixlv_disable_intr(struct ixl_vsi *vsi) { struct i40e_hw *hw = vsi->hw; - struct ixl_queue *que = vsi->queues; + struct ixl_rx_queue *que = vsi->rx_queues; ixlv_disable_adminq_irq(hw); - for (int i = 0; i < vsi->num_queues; i++, que++) - ixlv_disable_queue_irq(hw, que->me); + for (int i = 0; i < vsi->num_rx_queues; i++, que++) + ixlv_disable_queue_irq(hw, que->rxr.me); } @@ -1932,7 +2022,6 @@ wr32(hw, I40E_VFINT_ICR0_ENA1, 0); /* flush */ rd32(hw, I40E_VFGEN_RSTAT); - return; } static void @@ -1944,7 +2033,6 @@ wr32(hw, I40E_VFINT_ICR0_ENA1, I40E_VFINT_ICR0_ENA1_ADMINQ_MASK); /* flush */ rd32(hw, I40E_VFGEN_RSTAT); - return; } static void @@ -1975,24 +2063,18 @@ { struct i40e_hw *hw = &sc->hw; struct ixl_vsi *vsi = &sc->vsi; - struct ixl_queue *que = vsi->queues; + struct ixl_rx_queue *rx_que = vsi->rx_queues; vsi->rx_itr_setting = ixlv_rx_itr; - vsi->tx_itr_setting = ixlv_tx_itr; - for (int i = 0; i < vsi->num_queues; i++, que++) { - struct tx_ring *txr = &que->txr; - struct rx_ring *rxr = &que->rxr; + for (int i = 0; i < vsi->num_rx_queues; i++, rx_que++) { + // struct tx_ring *txr = &que->txr; + struct rx_ring *rxr = &rx_que->rxr; wr32(hw, I40E_VFINT_ITRN1(IXL_RX_ITR, i), vsi->rx_itr_setting); rxr->itr = vsi->rx_itr_setting; rxr->latency = IXL_AVE_LATENCY; - - wr32(hw, I40E_VFINT_ITRN1(IXL_TX_ITR, i), - vsi->tx_itr_setting); - txr->itr = vsi->tx_itr_setting; - txr->latency = IXL_AVE_LATENCY; } } @@ -2001,7 +2083,7 @@ ** interrupt moderation value. */ static void -ixlv_set_queue_rx_itr(struct ixl_queue *que) +ixlv_set_queue_rx_itr(struct ixl_rx_queue *que) { struct ixl_vsi *vsi = que->vsi; struct i40e_hw *hw = vsi->hw; @@ -2052,7 +2134,7 @@ ((9 * rx_itr) + rxr->itr); rxr->itr = min(rx_itr, IXL_MAX_ITR); wr32(hw, I40E_VFINT_ITRN1(IXL_RX_ITR, - que->me), rxr->itr); + que->rxr.me), rxr->itr); } } else { /* We may have have toggled to non-dynamic */ if (vsi->rx_itr_setting & IXL_ITR_DYNAMIC) @@ -2061,7 +2143,7 @@ if (rxr->itr != vsi->rx_itr_setting) { rxr->itr = vsi->rx_itr_setting; wr32(hw, I40E_VFINT_ITRN1(IXL_RX_ITR, - que->me), rxr->itr); + que->rxr.me), rxr->itr); } } rxr->bytes = 0; @@ -2069,221 +2151,58 @@ return; } - +#if 0 /* -** Provide a update to the queue TX -** interrupt moderation value. -*/ -static void -ixlv_set_queue_tx_itr(struct ixl_queue *que) + * TODO: Make sure this properly handles admin queue / single rx queue intr + * + * Use this intr handler when there is only one msix interrupt (shared between + * admin queue and RX queue) + */ +int +ixlv_intr(void *arg) { - struct ixl_vsi *vsi = que->vsi; - struct i40e_hw *hw = vsi->hw; - struct tx_ring *txr = &que->txr; - u16 tx_itr; - u16 tx_latency = 0; - int tx_bytes; - - - /* Idle, do nothing */ - if (txr->bytes == 0) - return; - - if (ixlv_dynamic_tx_itr) { - tx_bytes = txr->bytes/txr->itr; - tx_itr = txr->itr; - - switch (txr->latency) { - case IXL_LOW_LATENCY: - if (tx_bytes > 10) { - tx_latency = IXL_AVE_LATENCY; - tx_itr = IXL_ITR_20K; - } - break; - case IXL_AVE_LATENCY: - if (tx_bytes > 20) { - tx_latency = IXL_BULK_LATENCY; - tx_itr = IXL_ITR_8K; - } else if (tx_bytes <= 10) { - tx_latency = IXL_LOW_LATENCY; - tx_itr = IXL_ITR_100K; - } - break; - case IXL_BULK_LATENCY: - if (tx_bytes <= 20) { - tx_latency = IXL_AVE_LATENCY; - tx_itr = IXL_ITR_20K; - } - break; - } - - txr->latency = tx_latency; + struct ixlv_sc *sc = arg; + struct i40e_hw *hw = &sc->hw; + struct ixl_vsi *vsi = &sc->vsi; + struct ixl_rx_queue *que = vsi->rx_queues; + u32 icr0; - if (tx_itr != txr->itr) { - /* do an exponential smoothing */ - tx_itr = (10 * tx_itr * txr->itr) / - ((9 * tx_itr) + txr->itr); - txr->itr = min(tx_itr, IXL_MAX_ITR); - wr32(hw, I40E_VFINT_ITRN1(IXL_TX_ITR, - que->me), txr->itr); - } + sc->admin_irq++; - } else { /* We may have have toggled to non-dynamic */ - if (vsi->tx_itr_setting & IXL_ITR_DYNAMIC) - vsi->tx_itr_setting = ixlv_tx_itr; - /* Update the hardware if needed */ - if (txr->itr != vsi->tx_itr_setting) { - txr->itr = vsi->tx_itr_setting; - wr32(hw, I40E_VFINT_ITRN1(IXL_TX_ITR, - que->me), txr->itr); - } - } - txr->bytes = 0; - txr->packets = 0; - return; -} + icr0 = rd32(hw, I40E_PFINT_ICR0); + if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) + iflib_admin_intr_deferred(vsi->ctx); -/* -** -** MSIX Interrupt Handlers and Tasklets -** -*/ -static void -ixlv_handle_que(void *context, int pending) -{ - struct ixl_queue *que = context; - struct ixl_vsi *vsi = que->vsi; - struct i40e_hw *hw = vsi->hw; - struct tx_ring *txr = &que->txr; - struct ifnet *ifp = vsi->ifp; - bool more; - - if (ifp->if_drv_flags & IFF_DRV_RUNNING) { - more = ixl_rxeof(que, IXL_RX_LIMIT); - mtx_lock(&txr->mtx); - ixl_txeof(que); - if (!drbr_empty(ifp, txr->br)) - ixl_mq_start_locked(ifp, txr); - mtx_unlock(&txr->mtx); - if (more) { - taskqueue_enqueue(que->tq, &que->task); - return; - } - } + ixlv_enable_adminq_irq(hw); - /* Reenable this interrupt - hmmm */ - ixlv_enable_queue_irq(hw, que->me); - return; + if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) + return (FILTER_SCHEDULE_THREAD); + else + return (FILTER_HANDLED); } - +#endif /********************************************************************* * * MSIX Queue Interrupt Service routine * **********************************************************************/ -static void +static int ixlv_msix_que(void *arg) { - struct ixl_queue *que = arg; - struct ixl_vsi *vsi = que->vsi; - struct i40e_hw *hw = vsi->hw; - struct tx_ring *txr = &que->txr; - bool more_tx, more_rx; - - /* Spurious interrupts are ignored */ - if (!(vsi->ifp->if_drv_flags & IFF_DRV_RUNNING)) - return; + struct ixl_rx_queue *que = arg; ++que->irqs; - more_rx = ixl_rxeof(que, IXL_RX_LIMIT); - - mtx_lock(&txr->mtx); - more_tx = ixl_txeof(que); - /* - ** Make certain that if the stack - ** has anything queued the task gets - ** scheduled to handle it. - */ - if (!drbr_empty(vsi->ifp, txr->br)) - more_tx = 1; - mtx_unlock(&txr->mtx); - ixlv_set_queue_rx_itr(que); - ixlv_set_queue_tx_itr(que); - - if (more_tx || more_rx) - taskqueue_enqueue(que->tq, &que->task); - else - ixlv_enable_queue_irq(hw, que->me); - - return; -} - - -/********************************************************************* - * - * Media Ioctl callback - * - * This routine is called whenever the user queries the status of - * the interface using ifconfig. - * - **********************************************************************/ -static void -ixlv_media_status(struct ifnet * ifp, struct ifmediareq * ifmr) -{ - struct ixl_vsi *vsi = ifp->if_softc; - struct ixlv_sc *sc = vsi->back; - - INIT_DBG_IF(ifp, "begin"); - - mtx_lock(&sc->mtx); - - ixlv_update_link_status(sc); - - ifmr->ifm_status = IFM_AVALID; - ifmr->ifm_active = IFM_ETHER; - - if (!sc->link_up) { - mtx_unlock(&sc->mtx); - INIT_DBG_IF(ifp, "end: link not up"); - return; - } - - ifmr->ifm_status |= IFM_ACTIVE; - /* Hardware is always full-duplex */ - ifmr->ifm_active |= IFM_FDX; - mtx_unlock(&sc->mtx); - INIT_DBG_IF(ifp, "end"); - return; -} - -/********************************************************************* - * - * Media Ioctl callback - * - * This routine is called when the user changes speed/duplex using - * media/mediopt option with ifconfig. - * - **********************************************************************/ -static int -ixlv_media_change(struct ifnet * ifp) -{ - struct ixl_vsi *vsi = ifp->if_softc; - struct ifmedia *ifm = &vsi->media; - INIT_DBG_IF(ifp, "begin"); - - if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) - return (EINVAL); + // TODO: Re-enable queue? Or is that done in handler? - INIT_DBG_IF(ifp, "end"); - return (0); + return (FILTER_SCHEDULE_THREAD); } - +#if 0 /********************************************************************* * Multicast Initialization * @@ -2423,123 +2342,7 @@ IOCTL_DBG_IF(ifp, "end"); } - -/********************************************************************* - * Timer routine - * - * This routine checks for link status,updates statistics, - * and runs the watchdog check. - * - **********************************************************************/ - -static void -ixlv_local_timer(void *arg) -{ - struct ixlv_sc *sc = arg; - struct i40e_hw *hw = &sc->hw; - struct ixl_vsi *vsi = &sc->vsi; - struct ixl_queue *que = vsi->queues; - device_t dev = sc->dev; - struct tx_ring *txr; - int hung = 0; - u32 mask, val; - s32 timer, new_timer; - - IXLV_CORE_LOCK_ASSERT(sc); - - /* If Reset is in progress just bail */ - if (sc->init_state == IXLV_RESET_PENDING) - return; - - /* Check for when PF triggers a VF reset */ - val = rd32(hw, I40E_VFGEN_RSTAT) & - I40E_VFGEN_RSTAT_VFR_STATE_MASK; - - if (val != I40E_VFR_VFACTIVE - && val != I40E_VFR_COMPLETED) { - DDPRINTF(dev, "reset in progress! (%d)", val); - return; - } - - ixlv_request_stats(sc); - - /* clean and process any events */ - taskqueue_enqueue(sc->tq, &sc->aq_irq); - - /* - ** Check status on the queues for a hang - */ - mask = (I40E_VFINT_DYN_CTLN1_INTENA_MASK | - I40E_VFINT_DYN_CTLN1_SWINT_TRIG_MASK | - I40E_VFINT_DYN_CTLN1_ITR_INDX_MASK); - - for (int i = 0; i < vsi->num_queues; i++, que++) { - txr = &que->txr; - timer = atomic_load_acq_32(&txr->watchdog_timer); - if (timer > 0) { - new_timer = timer - hz; - if (new_timer <= 0) { - atomic_store_rel_32(&txr->watchdog_timer, -1); - device_printf(dev, "WARNING: queue %d " - "appears to be hung!\n", que->me); - ++hung; - } else { - /* - * If this fails, that means something in the TX path has updated - * the watchdog, so it means the TX path is still working and - * the watchdog doesn't need to countdown. - */ - atomic_cmpset_rel_32(&txr->watchdog_timer, timer, new_timer); - /* Any queues with outstanding work get a sw irq */ - wr32(hw, I40E_VFINT_DYN_CTLN1(que->me), mask); - } - } - } - /* Reset when a queue shows hung */ - if (hung) - goto hung; - - callout_reset(&sc->timer, hz, ixlv_local_timer, sc); - return; - -hung: - device_printf(dev, "WARNING: Resetting!\n"); - sc->init_state = IXLV_RESET_REQUIRED; - sc->watchdog_events++; - ixlv_stop(sc); - ixlv_init_locked(sc); -} - -/* -** Note: this routine updates the OS on the link state -** the real check of the hardware only happens with -** a link interrupt. -*/ -void -ixlv_update_link_status(struct ixlv_sc *sc) -{ - struct ixl_vsi *vsi = &sc->vsi; - struct ifnet *ifp = vsi->ifp; - - if (sc->link_up){ - if (vsi->link_active == FALSE) { - if (bootverbose) - if_printf(ifp,"Link is Up, %d Gbps\n", - (sc->link_speed == I40E_LINK_SPEED_40GB) ? 40:10); - vsi->link_active = TRUE; - if_link_state_change(ifp, LINK_STATE_UP); - } - } else { /* Link down */ - if (vsi->link_active == TRUE) { - if (bootverbose) - if_printf(ifp,"Link is Down\n"); - if_link_state_change(ifp, LINK_STATE_DOWN); - vsi->link_active = FALSE; - } - } - - return; -} +#endif /********************************************************************* * @@ -2557,8 +2360,6 @@ ifp = sc->vsi.ifp; INIT_DBG_IF(ifp, "begin"); - IXLV_CORE_LOCK_ASSERT(sc); - ixl_vc_flush(&sc->vc_mgr); ixlv_disable_queues(sc); @@ -2573,42 +2374,13 @@ INIT_DBG_IF(ifp, "end"); } - -/********************************************************************* - * - * Free all station queue structs. - * - **********************************************************************/ static void -ixlv_free_queues(struct ixl_vsi *vsi) +ixlv_if_stop(if_ctx_t ctx) { + struct ixl_vsi *vsi = iflib_get_softc(ctx); struct ixlv_sc *sc = (struct ixlv_sc *)vsi->back; - struct ixl_queue *que = vsi->queues; - for (int i = 0; i < vsi->num_queues; i++, que++) { - struct tx_ring *txr = &que->txr; - struct rx_ring *rxr = &que->rxr; - - if (!mtx_initialized(&txr->mtx)) /* uninitialized */ - continue; - IXL_TX_LOCK(txr); - ixl_free_que_tx(que); - if (txr->base) - i40e_free_dma_mem(&sc->hw, &txr->dma); - IXL_TX_UNLOCK(txr); - IXL_TX_LOCK_DESTROY(txr); - - if (!mtx_initialized(&rxr->mtx)) /* uninitialized */ - continue; - IXL_RX_LOCK(rxr); - ixl_free_que_rx(que); - if (rxr->base) - i40e_free_dma_mem(&sc->hw, &rxr->dma); - IXL_RX_UNLOCK(rxr); - IXL_RX_LOCK_DESTROY(rxr); - - } - free(vsi->queues, M_DEVBUF); + ixlv_stop(sc); } static void @@ -2625,7 +2397,7 @@ #endif /* Don't set up RSS if using a single queue */ - if (vsi->num_queues == 1) { + if (vsi->num_rx_queues == 1) { wr32(hw, I40E_VFQF_HENA(0), 0); wr32(hw, I40E_VFQF_HENA(1), 0); ixl_flush(hw); @@ -2671,7 +2443,7 @@ /* Populate the LUT with max no. of queues in round robin fashion */ for (i = 0, j = 0; i < IXL_RSS_VSI_LUT_SIZE; i++, j++) { - if (j == vsi->num_queues) + if (j == vsi->num_rx_queues) j = 0; #ifdef RSS /* @@ -2803,21 +2575,6 @@ return (0); } -/* -** Tasklet handler for MSIX Adminq interrupts -** - done outside interrupt context since it might sleep -*/ -static void -ixlv_do_adminq(void *context, int pending) -{ - struct ixlv_sc *sc = context; - - mtx_lock(&sc->mtx); - ixlv_do_adminq_locked(sc); - mtx_unlock(&sc->mtx); - return; -} - static void ixlv_do_adminq_locked(struct ixlv_sc *sc) { @@ -2830,8 +2587,6 @@ i40e_status ret; bool aq_error = false; - IXLV_CORE_LOCK_ASSERT(sc); - event.buf_len = IXL_AQ_BUF_SZ; event.msg_buf = sc->aq_buffer; v_msg = (struct i40e_virtchnl_msg *)&event.desc; @@ -2890,7 +2645,8 @@ device_printf(dev, "WARNING: Resetting!\n"); sc->init_state = IXLV_RESET_REQUIRED; ixlv_stop(sc); - ixlv_init_locked(sc); + // TODO: Make stop/init calls match + ixlv_if_init(sc->vsi.ctx); } ixlv_enable_adminq_irq(hw); } @@ -2906,15 +2662,19 @@ struct sysctl_oid *tree = device_get_sysctl_tree(dev); struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree); - struct sysctl_oid *vsi_node, *queue_node; - struct sysctl_oid_list *vsi_list, *queue_list; + struct sysctl_oid *vsi_node; //, *queue_node; + struct sysctl_oid_list *vsi_list; //, *queue_list; #define QUEUE_NAME_LEN 32 - char queue_namebuf[QUEUE_NAME_LEN]; + //char queue_namebuf[QUEUE_NAME_LEN]; + +#if 0 + struct ixl_tx_queue *tx_queues = vsi->tx_queues; + struct ixl_rx_queue *rx_queues = vsi->rx_queues; - struct ixl_queue *queues = vsi->queues; struct tx_ring *txr; struct rx_ring *rxr; +#endif /* Driver statistics sysctls */ SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events", @@ -2959,6 +2719,7 @@ entry++; } +#if 0 /* Queue sysctls */ for (int q = 0; q < vsi->num_queues; q++) { snprintf(queue_namebuf, QUEUE_NAME_LEN, "que%d", q); @@ -3023,6 +2784,7 @@ "Ticks before watchdog event is triggered"); #endif } +#endif } static void @@ -3034,7 +2796,6 @@ sc->vlan_filters = malloc(sizeof(struct ixlv_vlan_filter), M_DEVBUF, M_NOWAIT | M_ZERO); SLIST_INIT(sc->vlan_filters); - return; } static void @@ -3053,7 +2814,6 @@ SLIST_REMOVE_HEAD(sc->vlan_filters, next); free(v, M_DEVBUF); } - return; } #ifdef IXL_DEBUG Index: sys/dev/ixl/ixl.h =================================================================== --- sys/dev/ixl/ixl.h +++ sys/dev/ixl/ixl.h @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2013-2015, Intel Corporation + Copyright (c) 2013-2017, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -61,6 +61,7 @@ #include #include #include +#include #include #include @@ -102,80 +103,10 @@ #include #endif +#include "ifdi_if.h" #include "i40e_type.h" #include "i40e_prototype.h" - -#define MAC_FORMAT "%02x:%02x:%02x:%02x:%02x:%02x" -#define MAC_FORMAT_ARGS(mac_addr) \ - (mac_addr)[0], (mac_addr)[1], (mac_addr)[2], (mac_addr)[3], \ - (mac_addr)[4], (mac_addr)[5] -#define ON_OFF_STR(is_set) ((is_set) ? "On" : "Off") - -#ifdef IXL_DEBUG - -#define _DBG_PRINTF(S, ...) printf("%s: " S "\n", __func__, ##__VA_ARGS__) -#define _DEV_DBG_PRINTF(dev, S, ...) device_printf(dev, "%s: " S "\n", __func__, ##__VA_ARGS__) -#define _IF_DBG_PRINTF(ifp, S, ...) if_printf(ifp, "%s: " S "\n", __func__, ##__VA_ARGS__) - -/* Defines for printing generic debug information */ -#define DPRINTF(...) _DBG_PRINTF(__VA_ARGS__) -#define DDPRINTF(...) _DEV_DBG_PRINTF(__VA_ARGS__) -#define IDPRINTF(...) _IF_DBG_PRINTF(__VA_ARGS__) - -/* Defines for printing specific debug information */ -#define DEBUG_INIT 1 -#define DEBUG_IOCTL 1 -#define DEBUG_HW 1 - -#define INIT_DEBUGOUT(...) if (DEBUG_INIT) _DBG_PRINTF(__VA_ARGS__) -#define INIT_DBG_DEV(...) if (DEBUG_INIT) _DEV_DBG_PRINTF(__VA_ARGS__) -#define INIT_DBG_IF(...) if (DEBUG_INIT) _IF_DBG_PRINTF(__VA_ARGS__) - -#define IOCTL_DEBUGOUT(...) if (DEBUG_IOCTL) _DBG_PRINTF(__VA_ARGS__) -#define IOCTL_DBG_IF2(ifp, S, ...) if (DEBUG_IOCTL) \ - if_printf(ifp, S "\n", ##__VA_ARGS__) -#define IOCTL_DBG_IF(...) if (DEBUG_IOCTL) _IF_DBG_PRINTF(__VA_ARGS__) - -#define HW_DEBUGOUT(...) if (DEBUG_HW) _DBG_PRINTF(__VA_ARGS__) - -#else /* no IXL_DEBUG */ -#define DEBUG_INIT 0 -#define DEBUG_IOCTL 0 -#define DEBUG_HW 0 - -#define DPRINTF(...) -#define DDPRINTF(...) -#define IDPRINTF(...) - -#define INIT_DEBUGOUT(...) -#define INIT_DBG_DEV(...) -#define INIT_DBG_IF(...) -#define IOCTL_DEBUGOUT(...) -#define IOCTL_DBG_IF2(...) -#define IOCTL_DBG_IF(...) -#define HW_DEBUGOUT(...) -#endif /* IXL_DEBUG */ - -enum ixl_dbg_mask { - IXL_DBG_INFO = 0x00000001, - IXL_DBG_EN_DIS = 0x00000002, - IXL_DBG_AQ = 0x00000004, - IXL_DBG_NVMUPD = 0x00000008, - - IXL_DBG_IOCTL_KNOWN = 0x00000010, - IXL_DBG_IOCTL_UNKNOWN = 0x00000020, - IXL_DBG_IOCTL_ALL = 0x00000030, - - I40E_DEBUG_RSS = 0x00000100, - - IXL_DBG_IOV = 0x00001000, - IXL_DBG_IOV_VC = 0x00002000, - - IXL_DBG_SWITCH_INFO = 0x00010000, - IXL_DBG_I2C = 0x00020000, - - IXL_DBG_ALL = 0xFFFFFFFF -}; +#include "ixl_debug.h" /* Tunables */ @@ -221,7 +152,10 @@ #define IXL_MSIX_BAR 3 #define IXL_ADM_LIMIT 2 -#define IXL_TSO_SIZE 65535 +// TODO: Find out which TSO_SIZE to use +//#define IXL_TSO_SIZE 65535 +#define IXL_TSO_SIZE ((255*1024)-1) +#define IXL_TX_BUF_SZ ((u32) 1514) #define IXL_AQ_BUF_SZ ((u32) 4096) #define IXL_RX_HDR 128 #define IXL_RX_LIMIT 512 @@ -256,8 +190,7 @@ #define IXL_NVM_VERSION_HI_MASK (0xf << IXL_NVM_VERSION_HI_SHIFT) /* - * Interrupt Moderation parameters - * Multiply ITR values by 2 for real ITR value + * Interrupt Moderation parameters */ #define IXL_MAX_ITR 0x0FF0 #define IXL_ITR_100K 0x0005 @@ -284,10 +217,6 @@ #define CSUM_OFFLOAD_IPV6 (CSUM_TCP_IPV6|CSUM_UDP_IPV6|CSUM_SCTP_IPV6) #define CSUM_OFFLOAD (CSUM_OFFLOAD_IPV4|CSUM_OFFLOAD_IPV6|CSUM_TSO) -/* Misc flags for ixl_vsi.flags */ -#define IXL_FLAGS_KEEP_TSO4 (1 << 0) -#define IXL_FLAGS_KEEP_TSO6 (1 << 1) - #define IXL_VF_RESET_TIMEOUT 100 #define IXL_VSI_DATA_PORT 0x01 @@ -298,14 +227,6 @@ #define IXL_RX_CTX_BASE_UNITS 128 #define IXL_TX_CTX_BASE_UNITS 128 -#define IXL_VPINT_LNKLSTN_REG(hw, vector, vf_num) \ - I40E_VPINT_LNKLSTN(((vector) - 1) + \ - (((hw)->func_caps.num_msix_vectors_vf - 1) * (vf_num))) - -#define IXL_VFINT_DYN_CTLN_REG(hw, vector, vf_num) \ - I40E_VFINT_DYN_CTLN(((vector) - 1) + \ - (((hw)->func_caps.num_msix_vectors_vf - 1) * (vf_num))) - #define IXL_PF_PCI_CIAA_VF_DEVICE_STATUS 0xAA #define IXL_PF_PCI_CIAD_VF_TRANS_PENDING_MASK 0x20 @@ -341,15 +262,12 @@ BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK) | \ BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK)) -#define IXL_TX_LOCK(_sc) mtx_lock(&(_sc)->mtx) -#define IXL_TX_UNLOCK(_sc) mtx_unlock(&(_sc)->mtx) -#define IXL_TX_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->mtx) -#define IXL_TX_TRYLOCK(_sc) mtx_trylock(&(_sc)->mtx) -#define IXL_TX_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->mtx, MA_OWNED) +#define IXL_CAPS \ + (IFCAP_TSO4 | IFCAP_TSO6 | IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6 | IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | \ + IFCAP_VLAN_HWFILTER | IFCAP_VLAN_HWTSO | IFCAP_HWCSUM | \ + IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM | IFCAP_VLAN_HWTSO | \ + IFCAP_VLAN_MTU | IFCAP_HWCSUM_IPV6 | IFCAP_JUMBO_MTU | IFCAP_LRO) -#define IXL_RX_LOCK(_sc) mtx_lock(&(_sc)->mtx) -#define IXL_RX_UNLOCK(_sc) mtx_unlock(&(_sc)->mtx) -#define IXL_RX_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->mtx) /* Pre-11 counter(9) compatibility */ #if __FreeBSD_version >= 1100036 @@ -397,22 +315,6 @@ unsigned int index; } ixl_vendor_info_t; - -struct ixl_tx_buf { - u32 eop_index; - struct mbuf *m_head; - bus_dmamap_t map; - bus_dma_tag_t tag; -}; - -struct ixl_rx_buf { - struct mbuf *m_head; - struct mbuf *m_pack; - struct mbuf *fmp; - bus_dmamap_t hmap; - bus_dmamap_t pmap; -}; - /* ** This struct has multiple uses, multicast ** addresses, vlans, and mac filters all use it. @@ -428,123 +330,113 @@ * The Transmit ring control struct */ struct tx_ring { - struct ixl_queue *que; - struct mtx mtx; + struct ixl_tx_queue *que; u32 tail; - struct i40e_tx_desc *base; - struct i40e_dma_mem dma; - u16 next_avail; - u16 next_to_clean; - u16 atr_rate; - u16 atr_count; - u32 itr; - u32 latency; - struct ixl_tx_buf *buffers; - volatile u16 avail; - u32 cmd; - bus_dma_tag_t tx_tag; - bus_dma_tag_t tso_tag; - char mtx_name[16]; - struct buf_ring *br; - s32 watchdog_timer; + struct i40e_tx_desc *tx_base; + u64 tx_paddr; + u32 me; + /* For reporting completed packet status */ + qidx_t *tx_rsq; + qidx_t tx_rs_cidx; + qidx_t tx_rs_pidx; + qidx_t tx_cidx_processed; - /* Used for Dynamic ITR calculation */ - u32 packets; - u32 bytes; + // TODO: Re-add ITR calculation fields /* Soft Stats */ u64 tx_bytes; - u64 no_desc; u64 total_packets; }; - /* * The Receive ring control struct */ struct rx_ring { - struct ixl_queue *que; - struct mtx mtx; - union i40e_rx_desc *base; - struct i40e_dma_mem dma; - struct lro_ctrl lro; - bool lro_enabled; - bool hdr_split; + struct ixl_rx_queue *que; + union i40e_rx_desc *rx_base; + uint64_t rx_paddr; bool discard; - u32 next_refresh; - u32 next_check; u32 itr; u32 latency; - char mtx_name[16]; - struct ixl_rx_buf *buffers; u32 mbuf_sz; u32 tail; - bus_dma_tag_t htag; - bus_dma_tag_t ptag; + u32 me; /* Used for Dynamic ITR calculation */ u32 packets; u32 bytes; /* Soft stats */ + // TODO: Remove since no header split u64 split; u64 rx_packets; u64 rx_bytes; + // TODO: Change to discarded? u64 desc_errs; - u64 not_done; }; /* -** Driver queue struct: this is the interrupt container -** for the associated tx and rx ring pair. +** Driver queue structs +// TODO: Add to this comment? */ -struct ixl_queue { +struct ixl_tx_queue { struct ixl_vsi *vsi; - u32 me; - u32 msix; /* This queue's MSIX vector */ - u32 eims; /* This queue's EIMS bit */ - struct resource *res; - void *tag; - int num_desc; /* both tx and rx */ struct tx_ring txr; - struct rx_ring rxr; - struct task task; - struct task tx_task; - struct taskqueue *tq; - - /* Queue stats */ + /* Interrupt */ + struct if_irq que_irq; // TODO: Add comment + u32 msix; + /* Stats */ u64 irqs; u64 tso; - u64 mbuf_defrag_failed; - u64 mbuf_hdr_failed; - u64 mbuf_pkt_failed; - u64 tx_dmamap_failed; - u64 dropped_pkts; u64 mss_too_small; }; +struct ixl_rx_queue { + struct ixl_vsi *vsi; + struct rx_ring rxr; + struct if_irq que_irq; // TODO: Add comment + u32 msix; /* This queue's MSIX vector */ + /* Stats */ + u64 irqs; +}; + +#define DOWNCAST(sctx) ((struct ixl_vsi *)(sctx)) // TODO: Check if ixgbe has something similar + /* ** Virtual Station Interface */ SLIST_HEAD(ixl_ftl_head, ixl_mac_filter); + struct ixl_vsi { - void *back; + if_ctx_t ctx; + if_softc_ctx_t shared; + struct ifnet *ifp; - device_t dev; + struct ifmedia *media; + +// TODO: I don't like these defines +#define num_rx_queues shared->isc_nrxqsets +#define num_tx_queues shared->isc_ntxqsets +// This conflicts with a shared code struct definition +// #define max_frame_size shared->isc_max_frame_size + + void *back; struct i40e_hw *hw; - struct ifmedia media; - enum i40e_vsi_type type; + // TODO: Remove? + u64 que_mask; int id; - u16 num_queues; + u16 vsi_num; + enum i40e_vsi_type type; u32 rx_itr_setting; u32 tx_itr_setting; - u16 max_frame_size; - - struct ixl_queue *queues; /* head of queues */ - - u16 vsi_num; + struct ixl_tx_queue *tx_queues; /* TX queue array */ + struct ixl_rx_queue *rx_queues; /* RX queue array */ bool link_active; u16 seid; + u32 link_speed; + + struct if_irq irq; + u16 uplink_seid; u16 downlink_seid; @@ -555,10 +447,9 @@ /* Contains readylist & stat counter id */ struct i40e_aqc_vsi_properties_data info; - eventhandler_tag vlan_attach; - eventhandler_tag vlan_detach; u16 num_vlans; + // TODO: Maybe these things should get their own struct /* Per-VSI stats from hardware */ struct i40e_eth_stats eth_stats; struct i40e_eth_stats eth_stats_offsets; @@ -582,25 +473,11 @@ /* Misc. */ u64 flags; + /* Stats sysctls for this VSI */ struct sysctl_oid *vsi_node; }; /* -** Find the number of unrefreshed RX descriptors -*/ -static inline u16 -ixl_rx_unrefreshed(struct ixl_queue *que) -{ - struct rx_ring *rxr = &que->rxr; - - if (rxr->next_check > rxr->next_refresh) - return (rxr->next_check - rxr->next_refresh - 1); - else - return ((que->num_desc + rxr->next_check) - - rxr->next_refresh - 1); -} - -/* ** Find the next available unused filter */ static inline struct ixl_mac_filter * @@ -623,14 +500,7 @@ static inline bool cmp_etheraddr(const u8 *ea1, const u8 *ea2) { - bool cmp = FALSE; - - if ((ea1[0] == ea2[0]) && (ea1[1] == ea2[1]) && - (ea1[2] == ea2[2]) && (ea1[3] == ea2[3]) && - (ea1[4] == ea2[4]) && (ea1[5] == ea2[5])) - cmp = TRUE; - - return (cmp); + return (bcmp(ea1, ea2, 6) == 0); } /* @@ -664,30 +534,21 @@ char *description; }; -static uint8_t ixl_bcast_addr[ETHER_ADDR_LEN] = - {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; - /********************************************************************* * TXRX Function prototypes *********************************************************************/ -int ixl_allocate_tx_data(struct ixl_queue *); -int ixl_allocate_rx_data(struct ixl_queue *); -void ixl_init_tx_ring(struct ixl_queue *); -int ixl_init_rx_ring(struct ixl_queue *); -bool ixl_rxeof(struct ixl_queue *, int); -bool ixl_txeof(struct ixl_queue *); -void ixl_free_que_tx(struct ixl_queue *); -void ixl_free_que_rx(struct ixl_queue *); - int ixl_mq_start(struct ifnet *, struct mbuf *); int ixl_mq_start_locked(struct ifnet *, struct tx_ring *); void ixl_deferred_mq_start(void *, int); void ixl_free_vsi(struct ixl_vsi *); void ixl_qflush(struct ifnet *); +/********************************************************************* + * Common Function prototypes + *********************************************************************/ + /* Common function prototypes between PF/VF driver */ -#if __FreeBSD_version >= 1100000 -uint64_t ixl_get_counter(if_t ifp, ift_counter cnt); -#endif -void ixl_get_default_rss_key(u32 *); +void ixl_init_tx_ring(struct ixl_vsi *vsi, struct ixl_tx_queue *que); +void ixl_set_queue_rx_itr(struct ixl_rx_queue *que); +void ixl_get_default_rss_key(u32 *); #endif /* _IXL_H_ */ Index: sys/dev/ixl/ixl_debug.h =================================================================== --- /dev/null +++ sys/dev/ixl/ixl_debug.h @@ -0,0 +1,110 @@ +/****************************************************************************** + + Copyright (c) 2013-2016, Intel Corporation + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of the Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +******************************************************************************/ +/*$FreeBSD$*/ + +#ifndef _IXL_DEBUG_H_ +#define _IXL_DEBUG_H_ + +#define MAC_FORMAT "%02x:%02x:%02x:%02x:%02x:%02x" +#define MAC_FORMAT_ARGS(mac_addr) \ + (mac_addr)[0], (mac_addr)[1], (mac_addr)[2], (mac_addr)[3], \ + (mac_addr)[4], (mac_addr)[5] +#define ON_OFF_STR(is_set) ((is_set) ? "On" : "Off") + +#ifdef IXL_DEBUG + +#define _DBG_PRINTF(S, ...) printf("%s: " S "\n", __func__, ##__VA_ARGS__) +#define _DEV_DBG_PRINTF(dev, S, ...) device_printf(dev, "%s: " S "\n", __func__, ##__VA_ARGS__) +#define _IF_DBG_PRINTF(ifp, S, ...) if_printf(ifp, "%s: " S "\n", __func__, ##__VA_ARGS__) + +/* Defines for printing generic debug information */ +#define DPRINTF(...) _DBG_PRINTF(__VA_ARGS__) +#define DDPRINTF(...) _DEV_DBG_PRINTF(__VA_ARGS__) +#define IDPRINTF(...) _IF_DBG_PRINTF(__VA_ARGS__) + +/* Defines for printing specific debug information */ +#define DEBUG_INIT 1 +#define DEBUG_IOCTL 1 +#define DEBUG_HW 1 + +#define INIT_DEBUGOUT(...) if (DEBUG_INIT) _DBG_PRINTF(__VA_ARGS__) +#define INIT_DBG_DEV(...) if (DEBUG_INIT) _DEV_DBG_PRINTF(__VA_ARGS__) +#define INIT_DBG_IF(...) if (DEBUG_INIT) _IF_DBG_PRINTF(__VA_ARGS__) + +#define IOCTL_DEBUGOUT(...) if (DEBUG_IOCTL) _DBG_PRINTF(__VA_ARGS__) +#define IOCTL_DBG_IF2(ifp, S, ...) if (DEBUG_IOCTL) \ + if_printf(ifp, S "\n", ##__VA_ARGS__) +#define IOCTL_DBG_IF(...) if (DEBUG_IOCTL) _IF_DBG_PRINTF(__VA_ARGS__) + +#define HW_DEBUGOUT(...) if (DEBUG_HW) _DBG_PRINTF(__VA_ARGS__) + +#else /* no IXL_DEBUG */ +#define DEBUG_INIT 0 +#define DEBUG_IOCTL 0 +#define DEBUG_HW 0 + +#define DPRINTF(...) +#define DDPRINTF(...) +#define IDPRINTF(...) + +#define INIT_DEBUGOUT(...) +#define INIT_DBG_DEV(...) +#define INIT_DBG_IF(...) +#define IOCTL_DEBUGOUT(...) +#define IOCTL_DBG_IF2(...) +#define IOCTL_DBG_IF(...) +#define HW_DEBUGOUT(...) +#endif /* IXL_DEBUG */ + +enum ixl_dbg_mask { + IXL_DBG_INFO = 0x00000001, + IXL_DBG_EN_DIS = 0x00000002, + IXL_DBG_AQ = 0x00000004, + IXL_DBG_NVMUPD = 0x00000008, + + IXL_DBG_IOCTL_KNOWN = 0x00000010, + IXL_DBG_IOCTL_UNKNOWN = 0x00000020, + IXL_DBG_IOCTL_ALL = 0x00000030, + + I40E_DEBUG_RSS = 0x00000100, + + IXL_DBG_IOV = 0x00001000, + IXL_DBG_IOV_VC = 0x00002000, + + IXL_DBG_SWITCH_INFO = 0x00010000, + IXL_DBG_I2C = 0x00020000, + + IXL_DBG_ALL = 0xFFFFFFFF +}; + +#endif /* _IXL_DEBUG_H_ */ Index: sys/dev/ixl/ixl_pf.h =================================================================== --- sys/dev/ixl/ixl_pf.h +++ sys/dev/ixl/ixl_pf.h @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2013-2015, Intel Corporation + Copyright (c) 2013-2017, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -45,15 +45,40 @@ #define VF_FLAG_PROMISC_CAP 0x08 #define VF_FLAG_MAC_ANTI_SPOOF 0x10 -#define IXL_PF_STATE_EMPR_RESETTING (1 << 0) +#define IXL_ICR0_CRIT_ERR_MASK \ + (I40E_PFINT_ICR0_PCI_EXCEPTION_MASK | \ + I40E_PFINT_ICR0_ECC_ERR_MASK | \ + I40E_PFINT_ICR0_PE_CRITERR_MASK) + +/* VF Interrupts */ +#define IXL_VPINT_LNKLSTN_REG(hw, vector, vf_num) \ + I40E_VPINT_LNKLSTN(((vector) - 1) + \ + (((hw)->func_caps.num_msix_vectors_vf - 1) * (vf_num))) + +#define IXL_VFINT_DYN_CTLN_REG(hw, vector, vf_num) \ + I40E_VFINT_DYN_CTLN(((vector) - 1) + \ + (((hw)->func_caps.num_msix_vectors_vf - 1) * (vf_num))) + +/* Used in struct ixl_pf's state field */ +enum ixl_pf_state { + IXL_PF_STATE_ADAPTER_RESETTING = (1 << 0), + IXL_PF_STATE_MDD_PENDING = (1 << 1), + IXL_PF_STATE_PF_RESET_REQ = (1 << 2), + IXL_PF_STATE_VF_RESET_REQ = (1 << 3), + IXL_PF_STATE_PF_CRIT_ERR = (1 << 4), + IXL_PF_STATE_CORE_RESET_REQ = (1 << 5), + IXL_PF_STATE_GLOB_RESET_REQ = (1 << 6), + IXL_PF_STATE_EMP_RESET_REQ = (1 << 7), +}; struct ixl_vf { struct ixl_vsi vsi; - uint32_t vf_flags; + u32 vf_flags; + u32 num_mdd_events; - uint8_t mac[ETHER_ADDR_LEN]; - uint16_t vf_num; - uint32_t version; + u8 mac[ETHER_ADDR_LEN]; + u16 vf_num; + u32 version; struct ixl_pf_qtag qtag; struct sysctl_ctx_list ctx; @@ -61,58 +86,36 @@ /* Physical controller structure */ struct ixl_pf { + struct ixl_vsi vsi; + struct i40e_hw hw; struct i40e_osdep osdep; device_t dev; - struct ixl_vsi vsi; struct resource *pci_mem; - struct resource *msix_mem; - - /* - * Interrupt resources: this set is - * either used for legacy, or for Link - * when doing MSIX - */ - void *tag; - struct resource *res; - - struct callout timer; - int msix; + #ifdef IXL_IW int iw_msix; bool iw_enabled; #endif int if_flags; - int state; - bool init_in_progress; + + u32 qbase; + u32 admvec; + u32 state; u8 supported_speeds; struct ixl_pf_qmgr qmgr; struct ixl_pf_qtag qtag; - /* Tunable values */ - bool enable_msix; - int max_queues; - int ringsz; - bool enable_tx_fc_filter; - int dynamic_rx_itr; - int dynamic_tx_itr; - int tx_itr; - int rx_itr; - - struct mtx pf_mtx; - - u32 qbase; - u32 admvec; - struct task adminq; - struct taskqueue *tq; - bool link_up; u32 link_speed; int advertised_speed; int fc; /* link flow ctrl setting */ + + /* Debug levels */ enum ixl_dbg_mask dbg_mask; + int vc_debug_lvl; bool has_i2c; /* Misc stats maintained by the driver */ @@ -129,7 +132,16 @@ int num_vfs; uint16_t veb_seid; struct task vflr_task; - int vc_debug_lvl; + + /* Tunable values */ + bool enable_msix; + bool enable_hwb; + int max_queues; + int ringsz; + bool enable_tx_fc_filter; + int dynamic_rx_itr; + int dynamic_tx_itr; + int rx_itr; }; /* @@ -190,13 +202,6 @@ #define i40e_send_vf_nack(pf, vf, op, st) \ ixl_send_vf_nack_msg((pf), (vf), (op), (st), __FILE__, __LINE__) -#define IXL_PF_LOCK_INIT(_sc, _name) \ - mtx_init(&(_sc)->pf_mtx, _name, "IXL PF Lock", MTX_DEF) -#define IXL_PF_LOCK(_sc) mtx_lock(&(_sc)->pf_mtx) -#define IXL_PF_UNLOCK(_sc) mtx_unlock(&(_sc)->pf_mtx) -#define IXL_PF_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->pf_mtx) -#define IXL_PF_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->pf_mtx, MA_OWNED) - /* Debug printing */ #define ixl_dbg(p, m, s, ...) ixl_debug_core(p, m, s, ##__VA_ARGS__) void ixl_debug_core(struct ixl_pf *, enum ixl_dbg_mask, char *, ...); @@ -207,12 +212,8 @@ /* For netmap(4) compatibility */ #define ixl_disable_intr(vsi) ixl_disable_rings_intr(vsi) -/* - * PF-only function declarations - */ - +/* PF-only function declarations */ void ixl_set_busmaster(device_t); -void ixl_set_msix_enable(device_t); int ixl_setup_interface(device_t, struct ixl_vsi *); void ixl_print_nvm_cmd(device_t, struct i40e_nvm_access *); char * ixl_aq_speed_to_str(enum i40e_aq_link_speed); @@ -223,9 +224,17 @@ void ixl_local_timer(void *); void ixl_register_vlan(void *, struct ifnet *, u16); void ixl_unregister_vlan(void *, struct ifnet *, u16); -void ixl_intr(void *); -void ixl_msix_que(void *); -void ixl_msix_adminq(void *); + +/* IFLIB interface shared */ +// TODO: Maybe make a public interface to these instead? +void ixl_if_init(if_ctx_t ctx); +void ixl_if_stop(if_ctx_t ctx); + +/* Interrupt handlers */ +int ixl_intr(void *); +int ixl_msix_que(void *); +int ixl_msix_adminq(void *); + void ixl_do_adminq(void *, int); int ixl_res_alloc_cmp(const void *, const void *); @@ -256,8 +265,7 @@ void ixl_add_vsi_sysctls(struct ixl_pf *pf, struct ixl_vsi *vsi, struct sysctl_ctx_list *ctx, const char *sysctl_name); int ixl_get_hw_capabilities(struct ixl_pf *); void ixl_link_up_msg(struct ixl_pf *); -void ixl_update_link_status(struct ixl_pf *); -int ixl_allocate_pci_resources(struct ixl_pf *); +void ixl_update_link_status(if_ctx_t ctx); int ixl_setup_stations(struct ixl_pf *); int ixl_switch_config(struct ixl_pf *); void ixl_stop_locked(struct ixl_pf *); @@ -273,7 +281,7 @@ void ixl_configure_intr0_msix(struct ixl_pf *); void ixl_configure_queue_intr_msix(struct ixl_pf *); void ixl_free_adminq_tq(struct ixl_pf *); -int ixl_setup_legacy(struct ixl_pf *); +//int ixl_assign_vsi_legacy(struct ixl_pf *); int ixl_init_msix(struct ixl_pf *); void ixl_configure_itr(struct ixl_pf *); void ixl_configure_legacy(struct ixl_pf *); @@ -296,8 +304,8 @@ void ixl_handle_empr_reset(struct ixl_pf *); int ixl_rebuild_hw_structs_after_reset(struct ixl_pf *); -void ixl_set_queue_rx_itr(struct ixl_queue *); -void ixl_set_queue_tx_itr(struct ixl_queue *); +//void ixl_set_queue_rx_itr(struct ixl_rx_queue *); +//void ixl_set_queue_tx_itr(struct ixl_queue *); void ixl_add_filter(struct ixl_vsi *, u8 *, s16 vlan); void ixl_del_filter(struct ixl_vsi *, u8 *, s16 vlan); @@ -336,6 +344,8 @@ void ixl_free_mac_filters(struct ixl_vsi *vsi); void ixl_update_vsi_stats(struct ixl_vsi *); void ixl_vsi_reset_stats(struct ixl_vsi *); +u64 ixl_max_aq_speed_to_value(u8); +void ixl_handle_vflr(void *, int); /* * I2C Function prototypes Index: sys/dev/ixl/ixl_pf_iov.c =================================================================== --- sys/dev/ixl/ixl_pf_iov.c +++ sys/dev/ixl/ixl_pf_iov.c @@ -89,8 +89,11 @@ int iov_error; /* SR-IOV is only supported when MSI-X is in use. */ +// TODO: Make iflib stop using VSI as its softc +#if 0 if (pf->msix <= 1) return; +#endif pf_schema = pci_iov_schema_alloc_node(); vf_schema = pci_iov_schema_alloc_node(); @@ -157,7 +160,7 @@ htole16(I40E_AQ_VSI_PROP_QUEUE_MAP_VALID); vsi_ctx.info.mapping_flags = htole16(I40E_AQ_VSI_QUE_MAP_NONCONTIG); - /* ERJ: Only scattered allocation is supported for VFs right now */ + /* XXX: Only scattered allocation is supported for VFs right now */ for (i = 0; i < vf->qtag.num_active; i++) vsi_ctx.info.queue_mapping[i] = vf->qtag.qidx[i]; for (; i < nitems(vsi_ctx.info.queue_mapping); i++) @@ -172,8 +175,10 @@ return (ixl_adminq_err_to_errno(hw->aq.asq_last_status)); vf->vsi.seid = vsi_ctx.seid; vf->vsi.vsi_num = vsi_ctx.vsi_number; - // vf->vsi.first_queue = vf->qtag.qidx[0]; - vf->vsi.num_queues = vf->qtag.num_active; + // TODO: How to deal with num tx queues / num rx queues split? + // I don't think just assigning this variable is going to work + vf->vsi.num_rx_queues = vf->qtag.num_active; + vf->vsi.num_tx_queues = vf->qtag.num_active; code = i40e_aq_get_vsi_params(hw, &vsi_ctx, NULL); if (code != I40E_SUCCESS) @@ -204,7 +209,7 @@ vf->vsi.hw_filters_add = 0; vf->vsi.hw_filters_del = 0; - ixl_add_filter(&vf->vsi, ixl_bcast_addr, IXL_VLAN_ANY); + // ixl_add_filter(&vf->vsi, ixl_bcast_addr, IXL_VLAN_ANY); ixl_reconfigure_filters(&vf->vsi); return (0); @@ -253,7 +258,7 @@ /* Program index of each VF queue into PF queue space * (This is only needed if QTABLE is enabled) */ - for (i = 0; i < vf->vsi.num_queues; i++) { + for (i = 0; i < vf->vsi.num_tx_queues; i++) { qtable = ixl_pf_qidx_from_vsi_qidx(&vf->qtag, i) << I40E_VPLAN_QTABLE_QINDEX_SHIFT; @@ -266,7 +271,7 @@ /* Map queues allocated to VF to its VSI; * This mapping matches the VF-wide mapping since the VF * is only given a single VSI */ - for (i = 0; i < vf->vsi.num_queues; i++) + for (i = 0; i < vf->vsi.num_tx_queues; i++) ixl_vf_map_vsi_queue(hw, vf, i, ixl_pf_qidx_from_vsi_qidx(&vf->qtag, i)); @@ -335,7 +340,8 @@ ixl_vf_unregister_intr(hw, vpint_reg); } - vf->vsi.num_queues = 0; + vf->vsi.num_tx_queues = 0; + vf->vsi.num_rx_queues = 0; } static int @@ -533,13 +539,13 @@ I40E_VIRTCHNL_VF_OFFLOAD_VLAN); reply.num_vsis = 1; - reply.num_queue_pairs = vf->vsi.num_queues; + reply.num_queue_pairs = vf->vsi.num_tx_queues; reply.max_vectors = pf->hw.func_caps.num_msix_vectors_vf; reply.rss_key_size = 52; reply.rss_lut_size = 64; reply.vsi_res[0].vsi_id = vf->vsi.vsi_num; reply.vsi_res[0].vsi_type = I40E_VSI_SRIOV; - reply.vsi_res[0].num_queue_pairs = vf->vsi.num_queues; + reply.vsi_res[0].num_queue_pairs = vf->vsi.num_tx_queues; memcpy(reply.vsi_res[0].default_mac_addr, vf->mac, ETHER_ADDR_LEN); ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_GET_VF_RESOURCES, @@ -674,9 +680,9 @@ } info = msg; - if (info->num_queue_pairs == 0 || info->num_queue_pairs > vf->vsi.num_queues) { + if (info->num_queue_pairs == 0 || info->num_queue_pairs > vf->vsi.num_tx_queues) { device_printf(pf->dev, "VF %d: invalid # of qpairs (msg has %d, VSI has %d)\n", - vf->vf_num, info->num_queue_pairs, vf->vsi.num_queues); + vf->vf_num, info->num_queue_pairs, vf->vsi.num_tx_queues); i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM); return; @@ -705,7 +711,7 @@ if (pair->txq.vsi_id != vf->vsi.vsi_num || pair->rxq.vsi_id != vf->vsi.vsi_num || pair->txq.queue_id != pair->rxq.queue_id || - pair->txq.queue_id >= vf->vsi.num_queues) { + pair->txq.queue_id >= vf->vsi.num_tx_queues) { i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM); @@ -854,7 +860,7 @@ if (vector->rxq_map != 0) { largest_rxq = fls(vector->rxq_map) - 1; - if (largest_rxq >= vf->vsi.num_queues) { + if (largest_rxq >= vf->vsi.num_rx_queues) { i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, I40E_ERR_PARAM); @@ -864,7 +870,7 @@ if (vector->txq_map != 0) { largest_txq = fls(vector->txq_map) - 1; - if (largest_txq >= vf->vsi.num_queues) { + if (largest_txq >= vf->vsi.num_tx_queues) { i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, I40E_ERR_PARAM); @@ -911,7 +917,7 @@ for (int i = 0; i < 32; i++) { if ((1 << i) & select->tx_queues) { /* Warn if queue is out of VF allocation range */ - if (i >= vf->vsi.num_queues) { + if (i >= vf->vsi.num_tx_queues) { device_printf(pf->dev, "VF %d: TX ring %d is outside of VF VSI allocation!\n", vf->vf_num, i); break; @@ -936,7 +942,7 @@ for (int i = 0; i < 32; i++) { if ((1 << i) & select->rx_queues) { /* Warn if queue is out of VF allocation range */ - if (i >= vf->vsi.num_queues) { + if (i >= vf->vsi.num_rx_queues) { device_printf(pf->dev, "VF %d: RX ring %d is outside of VF VSI allocation!\n", vf->vf_num, i); break; @@ -990,7 +996,7 @@ for (int i = 0; i < 32; i++) { if ((1 << i) & select->tx_queues) { /* Warn if queue is out of VF allocation range */ - if (i >= vf->vsi.num_queues) { + if (i >= vf->vsi.num_tx_queues) { device_printf(pf->dev, "VF %d: TX ring %d is outside of VF VSI allocation!\n", vf->vf_num, i); break; @@ -1016,7 +1022,7 @@ for (int i = 0; i < 32; i++) { if ((1 << i) & select->rx_queues) { /* Warn if queue is out of VF allocation range */ - if (i >= vf->vsi.num_queues) { + if (i >= vf->vsi.num_rx_queues) { device_printf(pf->dev, "VF %d: RX ring %d is outside of VF VSI allocation!\n", vf->vf_num, i); break; @@ -1058,6 +1064,8 @@ static bool ixl_bcast_mac(const uint8_t *addr) { + static uint8_t ixl_bcast_addr[ETHER_ADDR_LEN] = + {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; return (cmp_etheraddr(addr, ixl_bcast_addr)); } @@ -1600,7 +1608,7 @@ pf = arg; hw = &pf->hw; - IXL_PF_LOCK(pf); + /* TODO: May need to lock this */ for (i = 0; i < pf->num_vfs; i++) { global_vf_num = hw->func_caps.vf_base_id + i; @@ -1619,12 +1627,13 @@ } } + atomic_clear_32(&pf->state, IXL_PF_STATE_VF_RESET_REQ); icr0 = rd32(hw, I40E_PFINT_ICR0_ENA); icr0 |= I40E_PFINT_ICR0_ENA_VFLR_MASK; wr32(hw, I40E_PFINT_ICR0_ENA, icr0); ixl_flush(hw); - IXL_PF_UNLOCK(pf); + // TODO: IXL_PF_UNLOCK() was here } static int @@ -1694,7 +1703,7 @@ hw = &pf->hw; pf_vsi = &pf->vsi; - IXL_PF_LOCK(pf); + // TODO: IXL_PF_LOCK() was here pf->vfs = malloc(sizeof(struct ixl_vf) * num_vfs, M_IXL, M_NOWAIT | M_ZERO); @@ -1716,13 +1725,13 @@ } pf->num_vfs = num_vfs; - IXL_PF_UNLOCK(pf); + // TODO: IXL_PF_UNLOCK() was here return (0); fail: free(pf->vfs, M_IXL); pf->vfs = NULL; - IXL_PF_UNLOCK(pf); + // TODO: IXL_PF_UNLOCK() was here return (error); } @@ -1741,7 +1750,7 @@ vsi = &pf->vsi; ifp = vsi->ifp; - IXL_PF_LOCK(pf); + // TODO: IXL_PF_LOCK() was here for (i = 0; i < pf->num_vfs; i++) { if (pf->vfs[i].vsi.seid != 0) i40e_aq_delete_element(hw, pf->vfs[i].vsi.seid, NULL); @@ -1761,7 +1770,7 @@ pf->vfs = NULL; pf->num_vfs = 0; - IXL_PF_UNLOCK(pf); + // TODO: IXL_PF_UNLOCK() was here /* Do this after the unlock as sysctl_ctx_free might sleep. */ for (i = 0; i < num_vfs; i++) @@ -1814,7 +1823,7 @@ pf = device_get_softc(dev); vf = &pf->vfs[vfnum]; - IXL_PF_LOCK(pf); + // TODO: IXL_PF_LOCK() was here vf->vf_num = vfnum; vf->vsi.back = pf; @@ -1854,7 +1863,7 @@ ixl_reset_vf(pf, vf); out: - IXL_PF_UNLOCK(pf); + // TODO: IXL_PF_UNLOCK() was here if (error == 0) { snprintf(sysctl_name, sizeof(sysctl_name), "vf%d", vfnum); ixl_add_vsi_sysctls(pf, &vf->vsi, &vf->ctx, sysctl_name); Index: sys/dev/ixl/ixl_pf_main.c =================================================================== --- sys/dev/ixl/ixl_pf_main.c +++ sys/dev/ixl/ixl_pf_main.c @@ -44,14 +44,6 @@ #include "ixl_iw_int.h" #endif -#ifdef DEV_NETMAP -#include -#include -#include -#endif /* DEV_NETMAP */ - -static int ixl_setup_queue(struct ixl_queue *, struct ixl_pf *, int); -static u64 ixl_max_aq_speed_to_value(u8); static u8 ixl_convert_sysctl_aq_link_speed(u8, bool); /* Sysctls */ @@ -60,7 +52,6 @@ static int ixl_current_speed(SYSCTL_HANDLER_ARGS); static int ixl_sysctl_show_fw(SYSCTL_HANDLER_ARGS); static int ixl_sysctl_unallocated_queues(SYSCTL_HANDLER_ARGS); -static int ixl_sysctl_pf_tx_itr(SYSCTL_HANDLER_ARGS); static int ixl_sysctl_pf_rx_itr(SYSCTL_HANDLER_ARGS); /* Debug Sysctls */ @@ -80,6 +71,10 @@ static int ixl_sysctl_fec_fc_request(SYSCTL_HANDLER_ARGS); static int ixl_sysctl_fec_rs_request(SYSCTL_HANDLER_ARGS); static int ixl_sysctl_fec_auto_enable(SYSCTL_HANDLER_ARGS); +static int ixl_sysctl_do_pf_reset(SYSCTL_HANDLER_ARGS); +static int ixl_sysctl_do_core_reset(SYSCTL_HANDLER_ARGS); +static int ixl_sysctl_do_global_reset(SYSCTL_HANDLER_ARGS); +static int ixl_sysctl_do_emp_reset(SYSCTL_HANDLER_ARGS); #ifdef IXL_DEBUG static int ixl_sysctl_qtx_tail_handler(SYSCTL_HANDLER_ARGS); static int ixl_sysctl_qrx_tail_handler(SYSCTL_HANDLER_ARGS); @@ -141,35 +136,16 @@ } static void -ixl_configure_tx_itr(struct ixl_pf *pf) -{ - struct i40e_hw *hw = &pf->hw; - struct ixl_vsi *vsi = &pf->vsi; - struct ixl_queue *que = vsi->queues; - - vsi->tx_itr_setting = pf->tx_itr; - - for (int i = 0; i < vsi->num_queues; i++, que++) { - struct tx_ring *txr = &que->txr; - - wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, i), - vsi->tx_itr_setting); - txr->itr = vsi->tx_itr_setting; - txr->latency = IXL_AVE_LATENCY; - } -} - -static void ixl_configure_rx_itr(struct ixl_pf *pf) { struct i40e_hw *hw = &pf->hw; struct ixl_vsi *vsi = &pf->vsi; - struct ixl_queue *que = vsi->queues; + struct ixl_rx_queue *rx_que = vsi->rx_queues; vsi->rx_itr_setting = pf->rx_itr; - for (int i = 0; i < vsi->num_queues; i++, que++) { - struct rx_ring *rxr = &que->rxr; + for (int i = 0; i < vsi->num_rx_queues; i++, rx_que++) { + struct rx_ring *rxr = &rx_que->rxr; wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, i), vsi->rx_itr_setting); @@ -184,140 +160,10 @@ void ixl_configure_itr(struct ixl_pf *pf) { - ixl_configure_tx_itr(pf); + // ixl_configure_tx_itr(pf); ixl_configure_rx_itr(pf); } - -/********************************************************************* - * Init entry point - * - * This routine is used in two ways. It is used by the stack as - * init entry point in network interface structure. It is also used - * by the driver as a hw/sw initialization routine to get to a - * consistent state. - * - * return 0 on success, positive on failure - **********************************************************************/ -void -ixl_init_locked(struct ixl_pf *pf) -{ - struct i40e_hw *hw = &pf->hw; - struct ixl_vsi *vsi = &pf->vsi; - struct ifnet *ifp = vsi->ifp; - device_t dev = pf->dev; - struct i40e_filter_control_settings filter; - u8 tmpaddr[ETHER_ADDR_LEN]; - int ret; - - INIT_DEBUGOUT("ixl_init_locked: begin"); - IXL_PF_LOCK_ASSERT(pf); - - ixl_stop_locked(pf); - - /* - * If the aq is dead here, it probably means something outside of the driver - * did something to the adapter, like a PF reset. - * So rebuild the driver's state here if that occurs. - */ - if (!i40e_check_asq_alive(&pf->hw)) { - device_printf(dev, "Admin Queue is down; resetting...\n"); - ixl_teardown_hw_structs(pf); - ixl_reset(pf); - } - - /* Get the latest mac address... User might use a LAA */ - bcopy(IF_LLADDR(vsi->ifp), tmpaddr, - I40E_ETH_LENGTH_OF_ADDRESS); - if (!cmp_etheraddr(hw->mac.addr, tmpaddr) && - (i40e_validate_mac_addr(tmpaddr) == I40E_SUCCESS)) { - ixl_del_filter(vsi, hw->mac.addr, IXL_VLAN_ANY); - bcopy(tmpaddr, hw->mac.addr, - I40E_ETH_LENGTH_OF_ADDRESS); - ret = i40e_aq_mac_address_write(hw, - I40E_AQC_WRITE_TYPE_LAA_ONLY, - hw->mac.addr, NULL); - if (ret) { - device_printf(dev, "LLA address" - "change failed!!\n"); - return; - } - } - - ixl_add_filter(vsi, hw->mac.addr, IXL_VLAN_ANY); - - /* Set the various hardware offload abilities */ - ifp->if_hwassist = 0; - if (ifp->if_capenable & IFCAP_TSO) - ifp->if_hwassist |= CSUM_TSO; - if (ifp->if_capenable & IFCAP_TXCSUM) - ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); - if (ifp->if_capenable & IFCAP_TXCSUM_IPV6) - ifp->if_hwassist |= (CSUM_TCP_IPV6 | CSUM_UDP_IPV6); - - /* Set up the device filtering */ - bzero(&filter, sizeof(filter)); - filter.enable_ethtype = TRUE; - filter.enable_macvlan = TRUE; - filter.enable_fdir = FALSE; - filter.hash_lut_size = I40E_HASH_LUT_SIZE_512; - if (i40e_set_filter_control(hw, &filter)) - device_printf(dev, "i40e_set_filter_control() failed\n"); - - /* Prepare the VSI: rings, hmc contexts, etc... */ - if (ixl_initialize_vsi(vsi)) { - device_printf(dev, "initialize vsi failed!!\n"); - return; - } - - /* Set up RSS */ - ixl_config_rss(pf); - - /* Add protocol filters to list */ - ixl_init_filters(vsi); - - /* Setup vlan's if needed */ - ixl_setup_vlan_filters(vsi); - - /* Set up MSI/X routing and the ITR settings */ - if (pf->msix > 1) { - ixl_configure_queue_intr_msix(pf); - ixl_configure_itr(pf); - } else - ixl_configure_legacy(pf); - - ixl_enable_rings(vsi); - - i40e_aq_set_default_vsi(hw, vsi->seid, NULL); - - ixl_reconfigure_filters(vsi); - - /* And now turn on interrupts */ - ixl_enable_intr(vsi); - - /* Get link info */ - hw->phy.get_link_info = TRUE; - i40e_get_link_status(hw, &pf->link_up); - ixl_update_link_status(pf); - - /* Start the local timer */ - callout_reset(&pf->timer, hz, ixl_local_timer, pf); - - /* Now inform the stack we're ready */ - ifp->if_drv_flags |= IFF_DRV_RUNNING; - -#ifdef IXL_IW - if (ixl_enable_iwarp && pf->iw_enabled) { - ret = ixl_iw_pf_init(pf); - if (ret) - device_printf(dev, - "initialize iwarp failed, code %d\n", ret); - } -#endif - -} - - /********************************************************************* * * Get the hardware capabilities @@ -330,31 +176,32 @@ struct i40e_aqc_list_capabilities_element_resp *buf; struct i40e_hw *hw = &pf->hw; device_t dev = pf->dev; - int error, len; - u16 needed; - bool again = TRUE; + enum i40e_status_code status; + bool again = TRUE; + int len; + u16 needed; len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp); retry: if (!(buf = (struct i40e_aqc_list_capabilities_element_resp *) - malloc(len, M_DEVBUF, M_NOWAIT | M_ZERO))) { + malloc(len, M_IXL, M_NOWAIT | M_ZERO))) { device_printf(dev, "Unable to allocate cap memory\n"); return (ENOMEM); } /* This populates the hw struct */ - error = i40e_aq_discover_capabilities(hw, buf, len, + status = i40e_aq_discover_capabilities(hw, buf, len, &needed, i40e_aqc_opc_list_func_capabilities, NULL); - free(buf, M_DEVBUF); + free(buf, M_IXL); if ((pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) && (again == TRUE)) { /* retry once with a larger buffer */ again = FALSE; len = needed; goto retry; - } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) { - device_printf(dev, "capability discovery failed: %d\n", - pf->hw.aq.asq_last_status); + } else if (status != I40E_SUCCESS) { + device_printf(dev, "capability discovery failed; status %s, error %s\n", + i40e_stat_str(hw, status), i40e_aq_str(hw, hw->aq.asq_last_status)); return (ENODEV); } @@ -386,79 +233,7 @@ if (osdep->i2c_intfc_num != -1) pf->has_i2c = true; - return (error); -} - -void -ixl_cap_txcsum_tso(struct ixl_vsi *vsi, struct ifnet *ifp, int mask) -{ - device_t dev = vsi->dev; - - /* Enable/disable TXCSUM/TSO4 */ - if (!(ifp->if_capenable & IFCAP_TXCSUM) - && !(ifp->if_capenable & IFCAP_TSO4)) { - if (mask & IFCAP_TXCSUM) { - ifp->if_capenable |= IFCAP_TXCSUM; - /* enable TXCSUM, restore TSO if previously enabled */ - if (vsi->flags & IXL_FLAGS_KEEP_TSO4) { - vsi->flags &= ~IXL_FLAGS_KEEP_TSO4; - ifp->if_capenable |= IFCAP_TSO4; - } - } - else if (mask & IFCAP_TSO4) { - ifp->if_capenable |= (IFCAP_TXCSUM | IFCAP_TSO4); - vsi->flags &= ~IXL_FLAGS_KEEP_TSO4; - device_printf(dev, - "TSO4 requires txcsum, enabling both...\n"); - } - } else if((ifp->if_capenable & IFCAP_TXCSUM) - && !(ifp->if_capenable & IFCAP_TSO4)) { - if (mask & IFCAP_TXCSUM) - ifp->if_capenable &= ~IFCAP_TXCSUM; - else if (mask & IFCAP_TSO4) - ifp->if_capenable |= IFCAP_TSO4; - } else if((ifp->if_capenable & IFCAP_TXCSUM) - && (ifp->if_capenable & IFCAP_TSO4)) { - if (mask & IFCAP_TXCSUM) { - vsi->flags |= IXL_FLAGS_KEEP_TSO4; - ifp->if_capenable &= ~(IFCAP_TXCSUM | IFCAP_TSO4); - device_printf(dev, - "TSO4 requires txcsum, disabling both...\n"); - } else if (mask & IFCAP_TSO4) - ifp->if_capenable &= ~IFCAP_TSO4; - } - - /* Enable/disable TXCSUM_IPV6/TSO6 */ - if (!(ifp->if_capenable & IFCAP_TXCSUM_IPV6) - && !(ifp->if_capenable & IFCAP_TSO6)) { - if (mask & IFCAP_TXCSUM_IPV6) { - ifp->if_capenable |= IFCAP_TXCSUM_IPV6; - if (vsi->flags & IXL_FLAGS_KEEP_TSO6) { - vsi->flags &= ~IXL_FLAGS_KEEP_TSO6; - ifp->if_capenable |= IFCAP_TSO6; - } - } else if (mask & IFCAP_TSO6) { - ifp->if_capenable |= (IFCAP_TXCSUM_IPV6 | IFCAP_TSO6); - vsi->flags &= ~IXL_FLAGS_KEEP_TSO6; - device_printf(dev, - "TSO6 requires txcsum6, enabling both...\n"); - } - } else if((ifp->if_capenable & IFCAP_TXCSUM_IPV6) - && !(ifp->if_capenable & IFCAP_TSO6)) { - if (mask & IFCAP_TXCSUM_IPV6) - ifp->if_capenable &= ~IFCAP_TXCSUM_IPV6; - else if (mask & IFCAP_TSO6) - ifp->if_capenable |= IFCAP_TSO6; - } else if ((ifp->if_capenable & IFCAP_TXCSUM_IPV6) - && (ifp->if_capenable & IFCAP_TSO6)) { - if (mask & IFCAP_TXCSUM_IPV6) { - vsi->flags |= IXL_FLAGS_KEEP_TSO6; - ifp->if_capenable &= ~(IFCAP_TXCSUM_IPV6 | IFCAP_TSO6); - device_printf(dev, - "TSO6 requires txcsum6, disabling both...\n"); - } else if (mask & IFCAP_TSO6) - ifp->if_capenable &= ~IFCAP_TSO6; - } + return (0); } /* For the set_advertise sysctl */ @@ -475,8 +250,8 @@ NULL); if (status) { /* Non-fatal error */ - device_printf(dev, "%s: i40e_aq_get_phy_capabilities() error %d\n", - __func__, status); + device_printf(dev, "%s: i40e_aq_get_phy_capabilities() error %s\n", + __func__, i40e_stat_str(hw, status)); return; } @@ -496,19 +271,20 @@ status = i40e_shutdown_lan_hmc(hw); if (status) { device_printf(dev, - "init: LAN HMC shutdown failure; status %d\n", status); + "init: LAN HMC shutdown failure; status %s\n", + i40e_stat_str(hw, status)); goto err_out; } } - // XXX: This gets called when we know the adminq is inactive; - // so we already know it's setup when we get here. - +#if 0 /* Shutdown admin queue */ status = i40e_shutdown_adminq(hw); if (status) device_printf(dev, - "init: Admin Queue shutdown failure; status %d\n", status); + "init: Admin Queue shutdown failure; status %s\n", + i40e_stat_str(hw, status)); +#endif err_out: return (status); @@ -519,10 +295,9 @@ { struct i40e_hw *hw = &pf->hw; device_t dev = pf->dev; - u8 set_fc_err_mask; + u32 reg; int error = 0; - // XXX: clear_hw() actually writes to hw registers -- maybe this isn't necessary i40e_clear_hw(hw); error = i40e_pf_reset(hw); if (error) { @@ -531,217 +306,82 @@ goto err_out; } - error = i40e_init_adminq(hw); - if (error) { - device_printf(dev, "init: Admin queue init failure;" - " status code %d", error); - error = EIO; - goto err_out; - } - - i40e_clear_pxe_mode(hw); - - error = ixl_get_hw_capabilities(pf); - if (error) { - device_printf(dev, "init: Error retrieving HW capabilities;" - " status code %d\n", error); - goto err_out; - } - - error = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, - hw->func_caps.num_rx_qp, 0, 0); - if (error) { - device_printf(dev, "init: LAN HMC init failed; status code %d\n", - error); - error = EIO; - goto err_out; - } - - error = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); - if (error) { - device_printf(dev, "init: LAN HMC config failed; status code %d\n", - error); - error = EIO; - goto err_out; - } - - // XXX: possible fix for panic, but our failure recovery is still broken - error = ixl_switch_config(pf); - if (error) { - device_printf(dev, "init: ixl_switch_config() failed: %d\n", - error); - goto err_out; - } - - error = i40e_aq_set_phy_int_mask(hw, IXL_DEFAULT_PHY_INT_MASK, - NULL); - if (error) { - device_printf(dev, "init: i40e_aq_set_phy_mask() failed: err %d," - " aq_err %d\n", error, hw->aq.asq_last_status); - error = EIO; - goto err_out; - } - - error = i40e_set_fc(hw, &set_fc_err_mask, true); - if (error) { - device_printf(dev, "init: setting link flow control failed; retcode %d," - " fc_err_mask 0x%02x\n", error, set_fc_err_mask); - goto err_out; - } - - // XXX: (Rebuild VSIs?) + ixl_rebuild_hw_structs_after_reset(pf); - /* Firmware delay workaround */ - if (((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver < 33)) || - (hw->aq.fw_maj_ver < 4)) { - i40e_msec_delay(75); - error = i40e_aq_set_link_restart_an(hw, TRUE, NULL); - if (error) { - device_printf(dev, "init: link restart failed, aq_err %d\n", - hw->aq.asq_last_status); - goto err_out; - } - } + /* The PF reset should have cleared any critical errors */ + atomic_clear_32(&pf->state, IXL_PF_STATE_PF_CRIT_ERR); + atomic_clear_32(&pf->state, IXL_PF_STATE_PF_RESET_REQ); + reg = rd32(hw, I40E_PFINT_ICR0_ENA); + reg |= IXL_ICR0_CRIT_ERR_MASK; + wr32(hw, I40E_PFINT_ICR0_ENA, reg); err_out: return (error); } /* -** MSIX Interrupt Handlers and Tasklets -*/ -void -ixl_handle_que(void *context, int pending) -{ - struct ixl_queue *que = context; - struct ixl_vsi *vsi = que->vsi; - struct i40e_hw *hw = vsi->hw; - struct tx_ring *txr = &que->txr; - struct ifnet *ifp = vsi->ifp; - bool more; - - if (ifp->if_drv_flags & IFF_DRV_RUNNING) { - more = ixl_rxeof(que, IXL_RX_LIMIT); - IXL_TX_LOCK(txr); - ixl_txeof(que); - if (!drbr_empty(ifp, txr->br)) - ixl_mq_start_locked(ifp, txr); - IXL_TX_UNLOCK(txr); - if (more) { - taskqueue_enqueue(que->tq, &que->task); - return; - } - } - - /* Reenable this interrupt - hmmm */ - ixl_enable_queue(hw, que->me); - return; -} - - -/********************************************************************* - * - * Legacy Interrupt Service routine - * - **********************************************************************/ -void + * TODO: Make sure this properly handles admin queue / single rx queue intr + */ +int ixl_intr(void *arg) { struct ixl_pf *pf = arg; struct i40e_hw *hw = &pf->hw; struct ixl_vsi *vsi = &pf->vsi; - struct ixl_queue *que = vsi->queues; - struct ifnet *ifp = vsi->ifp; - struct tx_ring *txr = &que->txr; - u32 icr0; - bool more_tx, more_rx; + struct ixl_rx_queue *que = vsi->rx_queues; + u32 icr0; - pf->admin_irq++; + device_printf(iflib_get_dev(vsi->ctx), "%s: begin\n", __func__); - /* Protect against spurious interrupts */ - if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) - return; + // pf->admin_irq++ + ++que->irqs; icr0 = rd32(hw, I40E_PFINT_ICR0); - #ifdef PCI_IOV if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) - taskqueue_enqueue(pf->tq, &pf->vflr_task); + iflib_iov_intr_deferred(vsi->ctx); #endif - if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) { - taskqueue_enqueue(pf->tq, &pf->adminq); - } - - if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) { - ++que->irqs; + // TODO!: Do the stuff that's done in ixl_msix_adminq here, too! - more_rx = ixl_rxeof(que, IXL_RX_LIMIT); - - IXL_TX_LOCK(txr); - more_tx = ixl_txeof(que); - if (!drbr_empty(vsi->ifp, txr->br)) - more_tx = 1; - IXL_TX_UNLOCK(txr); - } + if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) + iflib_admin_intr_deferred(vsi->ctx); + // TODO: Is intr0 enabled somewhere else? ixl_enable_intr0(hw); -} + // device_printf(iflib_get_dev(vsi->ctx), "%s: end (regular)\n", __func__); + if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) + return (FILTER_SCHEDULE_THREAD); + else + return (FILTER_HANDLED); +} /********************************************************************* * * MSIX VSI Interrupt Service routine * **********************************************************************/ -void +int ixl_msix_que(void *arg) { - struct ixl_queue *que = arg; - struct ixl_vsi *vsi = que->vsi; - struct i40e_hw *hw = vsi->hw; - struct tx_ring *txr = &que->txr; - bool more_tx, more_rx; - - /* Protect against spurious interrupts */ - if (!(vsi->ifp->if_drv_flags & IFF_DRV_RUNNING)) - return; + struct ixl_rx_queue *que = arg; ++que->irqs; - more_rx = ixl_rxeof(que, IXL_RX_LIMIT); - - IXL_TX_LOCK(txr); - more_tx = ixl_txeof(que); - /* - ** Make certain that if the stack - ** has anything queued the task gets - ** scheduled to handle it. - */ - if (!drbr_empty(vsi->ifp, txr->br)) - more_tx = 1; - IXL_TX_UNLOCK(txr); - ixl_set_queue_rx_itr(que); - ixl_set_queue_tx_itr(que); - if (more_tx || more_rx) - taskqueue_enqueue(que->tq, &que->task); - else - ixl_enable_queue(hw, que->me); - - return; + return (FILTER_SCHEDULE_THREAD); } - /********************************************************************* * * MSIX Admin Queue Interrupt Service routine * **********************************************************************/ -void +int ixl_msix_adminq(void *arg) { struct ixl_pf *pf = arg; @@ -750,23 +390,31 @@ u32 reg, mask, rstat_reg; bool do_task = FALSE; + DDPRINTF(dev, "begin"); + ++pf->admin_irq; + ixl_disable_intr0(hw); + reg = rd32(hw, I40E_PFINT_ICR0); + // For masking off interrupt causes that need to be handled before + // they can be re-enabled mask = rd32(hw, I40E_PFINT_ICR0_ENA); /* Check on the cause */ if (reg & I40E_PFINT_ICR0_ADMINQ_MASK) { - mask &= ~I40E_PFINT_ICR0_ADMINQ_MASK; + mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK; do_task = TRUE; } if (reg & I40E_PFINT_ICR0_MAL_DETECT_MASK) { - ixl_handle_mdd_event(pf); - mask &= ~I40E_PFINT_ICR0_MAL_DETECT_MASK; + mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; + atomic_set_32(&pf->state, IXL_PF_STATE_MDD_PENDING); + do_task = TRUE; } if (reg & I40E_PFINT_ICR0_GRST_MASK) { + mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK; device_printf(dev, "Reset Requested!\n"); rstat_reg = rd32(hw, I40E_GLGEN_RSTAT); rstat_reg = (rstat_reg & I40E_GLGEN_RSTAT_RESET_TYPE_MASK) @@ -782,20 +430,38 @@ break; case I40E_RESET_EMPR: printf("EMPR\n"); - atomic_set_int(&pf->state, IXL_PF_STATE_EMPR_RESETTING); break; default: printf("POR\n"); break; } - /* overload admin queue task to check reset progress */ + atomic_set_int(&pf->state, IXL_PF_STATE_ADAPTER_RESETTING); do_task = TRUE; } - if (reg & I40E_PFINT_ICR0_ECC_ERR_MASK) { + /* + * PE / PCI / ECC exceptions are all handled in the same way: + * mask out these three causes, then request a PF reset + * + * TODO: I think at least ECC error requires a GLOBR, not PFR + */ + if (reg & I40E_PFINT_ICR0_ECC_ERR_MASK) device_printf(dev, "ECC Error detected!\n"); + if (reg & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) + device_printf(dev, "PCI Exception detected!\n"); + if (reg & I40E_PFINT_ICR0_PE_CRITERR_MASK) + device_printf(dev, "Critical Protocol Engine Error detected!\n"); + /* Checks against the conditions above */ + if (reg & IXL_ICR0_CRIT_ERR_MASK) { + mask &= ~IXL_ICR0_CRIT_ERR_MASK; + atomic_set_32(&pf->state, + IXL_PF_STATE_PF_RESET_REQ | IXL_PF_STATE_PF_CRIT_ERR); + do_task = TRUE; } + // TODO: Linux driver never re-enables this interrupt once it has been detected + // Then what is supposed to happen? A PF reset? Should it never happen? + // TODO: Parse out this error into something human readable if (reg & I40E_PFINT_ICR0_HMC_ERR_MASK) { reg = rd32(hw, I40E_PFHMC_ERRORINFO); if (reg & I40E_PFHMC_ERRORINFO_ERROR_DETECTED_MASK) { @@ -807,56 +473,19 @@ } } - if (reg & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) { - device_printf(dev, "PCI Exception detected!\n"); - } - #ifdef PCI_IOV if (reg & I40E_PFINT_ICR0_VFLR_MASK) { mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK; - taskqueue_enqueue(pf->tq, &pf->vflr_task); + atomic_set_32(&pf->state, IXL_PF_STATE_VF_RESET_REQ); + do_task = TRUE; } #endif + wr32(hw, I40E_PFINT_ICR0_ENA, mask); if (do_task) - taskqueue_enqueue(pf->tq, &pf->adminq); + return (FILTER_SCHEDULE_THREAD); else - ixl_enable_intr0(hw); -} - -void -ixl_set_promisc(struct ixl_vsi *vsi) -{ - struct ifnet *ifp = vsi->ifp; - struct i40e_hw *hw = vsi->hw; - int err, mcnt = 0; - bool uni = FALSE, multi = FALSE; - - if (ifp->if_flags & IFF_ALLMULTI) - multi = TRUE; - else { /* Need to count the multicast addresses */ - struct ifmultiaddr *ifma; - if_maddr_rlock(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { - if (ifma->ifma_addr->sa_family != AF_LINK) - continue; - if (mcnt == MAX_MULTICAST_ADDR) - break; - mcnt++; - } - if_maddr_runlock(ifp); - } - - if (mcnt >= MAX_MULTICAST_ADDR) - multi = TRUE; - if (ifp->if_flags & IFF_PROMISC) - uni = TRUE; - - err = i40e_aq_set_vsi_unicast_promiscuous(hw, - vsi->seid, uni, NULL, TRUE); - err = i40e_aq_set_vsi_multicast_promiscuous(hw, - vsi->seid, multi, NULL); - return; + return (FILTER_HANDLED); } /********************************************************************* @@ -911,7 +540,6 @@ } IOCTL_DEBUGOUT("ixl_add_multi: end"); - return; } void @@ -951,78 +579,6 @@ ixl_del_hw_filters(vsi, mcnt); } - -/********************************************************************* - * Timer routine - * - * This routine checks for link status,updates statistics, - * and runs the watchdog check. - * - * Only runs when the driver is configured UP and RUNNING. - * - **********************************************************************/ - -void -ixl_local_timer(void *arg) -{ - struct ixl_pf *pf = arg; - struct i40e_hw *hw = &pf->hw; - struct ixl_vsi *vsi = &pf->vsi; - struct ixl_queue *que = vsi->queues; - device_t dev = pf->dev; - struct tx_ring *txr; - int hung = 0; - u32 mask; - s32 timer, new_timer; - - IXL_PF_LOCK_ASSERT(pf); - - /* Fire off the adminq task */ - taskqueue_enqueue(pf->tq, &pf->adminq); - - /* Update stats */ - ixl_update_stats_counters(pf); - - /* Check status of the queues */ - mask = (I40E_PFINT_DYN_CTLN_INTENA_MASK | - I40E_PFINT_DYN_CTLN_SWINT_TRIG_MASK | - I40E_PFINT_DYN_CTLN_ITR_INDX_MASK); - - for (int i = 0; i < vsi->num_queues; i++, que++) { - txr = &que->txr; - timer = atomic_load_acq_32(&txr->watchdog_timer); - if (timer > 0) { - new_timer = timer - hz; - if (new_timer <= 0) { - atomic_store_rel_32(&txr->watchdog_timer, -1); - device_printf(dev, "WARNING: queue %d " - "appears to be hung!\n", que->me); - ++hung; - } else { - /* - * If this fails, that means something in the TX path has updated - * the watchdog, so it means the TX path is still working and - * the watchdog doesn't need to countdown. - */ - atomic_cmpset_rel_32(&txr->watchdog_timer, timer, new_timer); - /* Any queues with outstanding work get a sw irq */ - wr32(hw, I40E_PFINT_DYN_CTLN(que->me), mask); - } - } - } - /* Reset when a queue shows hung */ - if (hung) - goto hung; - - callout_reset(&pf->timer, hz, ixl_local_timer, pf); - return; - -hung: - device_printf(dev, "WARNING: Resetting!\n"); - pf->watchdog_events++; - ixl_init_locked(pf); -} - void ixl_link_up_msg(struct ixl_pf *pf) { @@ -1044,477 +600,6 @@ } /* -** Note: this routine updates the OS on the link state -** the real check of the hardware only happens with -** a link interrupt. -*/ -void -ixl_update_link_status(struct ixl_pf *pf) -{ - struct ixl_vsi *vsi = &pf->vsi; - struct ifnet *ifp = vsi->ifp; - device_t dev = pf->dev; - - if (pf->link_up) { - if (vsi->link_active == FALSE) { - vsi->link_active = TRUE; - ifp->if_baudrate = ixl_max_aq_speed_to_value(pf->link_speed); - if_link_state_change(ifp, LINK_STATE_UP); - ixl_link_up_msg(pf); - } - } else { /* Link down */ - if (vsi->link_active == TRUE) { - if (bootverbose) - device_printf(dev, "Link is Down\n"); - if_link_state_change(ifp, LINK_STATE_DOWN); - vsi->link_active = FALSE; - } - } - - return; -} - -/********************************************************************* - * - * This routine disables all traffic on the adapter by issuing a - * global reset on the MAC and deallocates TX/RX buffers. - * - **********************************************************************/ - -void -ixl_stop_locked(struct ixl_pf *pf) -{ - struct ixl_vsi *vsi = &pf->vsi; - struct ifnet *ifp = vsi->ifp; - - INIT_DEBUGOUT("ixl_stop: begin\n"); - - IXL_PF_LOCK_ASSERT(pf); - -#ifdef IXL_IW - /* Stop iWARP device */ - if (ixl_enable_iwarp && pf->iw_enabled) - ixl_iw_pf_stop(pf); -#endif - - /* Stop the local timer */ - callout_stop(&pf->timer); - - ixl_disable_rings_intr(vsi); - ixl_disable_rings(vsi); - - /* Tell the stack that the interface is no longer active */ - ifp->if_drv_flags &= ~(IFF_DRV_RUNNING); -} - -void -ixl_stop(struct ixl_pf *pf) -{ - IXL_PF_LOCK(pf); - ixl_stop_locked(pf); - IXL_PF_UNLOCK(pf); -} - -/********************************************************************* - * - * Setup MSIX Interrupt resources and handlers for the VSI - * - **********************************************************************/ -int -ixl_setup_legacy(struct ixl_pf *pf) -{ - device_t dev = pf->dev; - int error, rid = 0; - - if (pf->msix == 1) - rid = 1; - pf->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, - &rid, RF_SHAREABLE | RF_ACTIVE); - if (pf->res == NULL) { - device_printf(dev, "bus_alloc_resource_any() for" - " legacy/msi interrupt\n"); - return (ENXIO); - } - - /* Set the handler function */ - error = bus_setup_intr(dev, pf->res, - INTR_TYPE_NET | INTR_MPSAFE, NULL, - ixl_intr, pf, &pf->tag); - if (error) { - pf->res = NULL; - device_printf(dev, "bus_setup_intr() for legacy/msi" - " interrupt handler failed, error %d\n", error); - return (ENXIO); - } - error = bus_describe_intr(dev, pf->res, pf->tag, "irq"); - if (error) { - /* non-fatal */ - device_printf(dev, "bus_describe_intr() for Admin Queue" - " interrupt name failed, error %d\n", error); - } - - return (0); -} - -int -ixl_setup_adminq_tq(struct ixl_pf *pf) -{ - device_t dev = pf->dev; - int error = 0; - - /* Tasklet for Admin Queue interrupts */ - TASK_INIT(&pf->adminq, 0, ixl_do_adminq, pf); -#ifdef PCI_IOV - /* VFLR Tasklet */ - TASK_INIT(&pf->vflr_task, 0, ixl_handle_vflr, pf); -#endif - /* Create and start Admin Queue taskqueue */ - pf->tq = taskqueue_create_fast("ixl_aq", M_NOWAIT, - taskqueue_thread_enqueue, &pf->tq); - if (!pf->tq) { - device_printf(dev, "taskqueue_create_fast (for AQ) returned NULL!\n"); - return (ENOMEM); - } - error = taskqueue_start_threads(&pf->tq, 1, PI_NET, "%s aq", - device_get_nameunit(dev)); - if (error) { - device_printf(dev, "taskqueue_start_threads (for AQ) error: %d\n", - error); - taskqueue_free(pf->tq); - return (error); - } - return (0); -} - -int -ixl_setup_queue_tqs(struct ixl_vsi *vsi) -{ - struct ixl_queue *que = vsi->queues; - device_t dev = vsi->dev; -#ifdef RSS - int cpu_id = 0; - cpuset_t cpu_mask; -#endif - - /* Create queue tasks and start queue taskqueues */ - for (int i = 0; i < vsi->num_queues; i++, que++) { - TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que); - TASK_INIT(&que->task, 0, ixl_handle_que, que); - que->tq = taskqueue_create_fast("ixl_que", M_NOWAIT, - taskqueue_thread_enqueue, &que->tq); -#ifdef RSS - CPU_SETOF(cpu_id, &cpu_mask); - taskqueue_start_threads_cpuset(&que->tq, 1, PI_NET, - &cpu_mask, "%s (bucket %d)", - device_get_nameunit(dev), cpu_id); -#else - taskqueue_start_threads(&que->tq, 1, PI_NET, - "%s (que %d)", device_get_nameunit(dev), que->me); -#endif - } - - return (0); -} - -void -ixl_free_adminq_tq(struct ixl_pf *pf) -{ - if (pf->tq) { - taskqueue_free(pf->tq); - pf->tq = NULL; - } -} - -void -ixl_free_queue_tqs(struct ixl_vsi *vsi) -{ - struct ixl_queue *que = vsi->queues; - - for (int i = 0; i < vsi->num_queues; i++, que++) { - if (que->tq) { - taskqueue_free(que->tq); - que->tq = NULL; - } - } -} - -int -ixl_setup_adminq_msix(struct ixl_pf *pf) -{ - device_t dev = pf->dev; - int rid, error = 0; - - /* Admin IRQ rid is 1, vector is 0 */ - rid = 1; - /* Get interrupt resource from bus */ - pf->res = bus_alloc_resource_any(dev, - SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); - if (!pf->res) { - device_printf(dev, "bus_alloc_resource_any() for Admin Queue" - " interrupt failed [rid=%d]\n", rid); - return (ENXIO); - } - /* Then associate interrupt with handler */ - error = bus_setup_intr(dev, pf->res, - INTR_TYPE_NET | INTR_MPSAFE, NULL, - ixl_msix_adminq, pf, &pf->tag); - if (error) { - pf->res = NULL; - device_printf(dev, "bus_setup_intr() for Admin Queue" - " interrupt handler failed, error %d\n", error); - return (ENXIO); - } - error = bus_describe_intr(dev, pf->res, pf->tag, "aq"); - if (error) { - /* non-fatal */ - device_printf(dev, "bus_describe_intr() for Admin Queue" - " interrupt name failed, error %d\n", error); - } - pf->admvec = 0; - - return (0); -} - -/* - * Allocate interrupt resources from bus and associate an interrupt handler - * to those for the VSI's queues. - */ -int -ixl_setup_queue_msix(struct ixl_vsi *vsi) -{ - device_t dev = vsi->dev; - struct ixl_queue *que = vsi->queues; - struct tx_ring *txr; - int error, rid, vector = 1; - - /* Queue interrupt vector numbers start at 1 (adminq intr is 0) */ - for (int i = 0; i < vsi->num_queues; i++, vector++, que++) { - int cpu_id = i; - rid = vector + 1; - txr = &que->txr; - que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, - RF_SHAREABLE | RF_ACTIVE); - if (!que->res) { - device_printf(dev, "bus_alloc_resource_any() for" - " Queue %d interrupt failed [rid=%d]\n", - que->me, rid); - return (ENXIO); - } - /* Set the handler function */ - error = bus_setup_intr(dev, que->res, - INTR_TYPE_NET | INTR_MPSAFE, NULL, - ixl_msix_que, que, &que->tag); - if (error) { - device_printf(dev, "bus_setup_intr() for Queue %d" - " interrupt handler failed, error %d\n", - que->me, error); - bus_release_resource(dev, SYS_RES_IRQ, rid, que->res); - return (error); - } - error = bus_describe_intr(dev, que->res, que->tag, "q%d", i); - if (error) { - device_printf(dev, "bus_describe_intr() for Queue %d" - " interrupt name failed, error %d\n", - que->me, error); - } - /* Bind the vector to a CPU */ -#ifdef RSS - cpu_id = rss_getcpu(i % rss_getnumbuckets()); -#endif - error = bus_bind_intr(dev, que->res, cpu_id); - if (error) { - device_printf(dev, "bus_bind_intr() for Queue %d" - " to CPU %d failed, error %d\n", - que->me, cpu_id, error); - } - que->msix = vector; - } - - return (0); -} - -/* - * When used in a virtualized environment PCI BUSMASTER capability may not be set - * so explicity set it here and rewrite the ENABLE in the MSIX control register - * at this point to cause the host to successfully initialize us. - */ -void -ixl_set_busmaster(device_t dev) -{ - u16 pci_cmd_word; - - pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); - pci_cmd_word |= PCIM_CMD_BUSMASTEREN; - pci_write_config(dev, PCIR_COMMAND, pci_cmd_word, 2); -} - -/* - * rewrite the ENABLE in the MSIX control register - * to cause the host to successfully initialize us. - */ -void -ixl_set_msix_enable(device_t dev) -{ - int msix_ctrl, rid; - - pci_find_cap(dev, PCIY_MSIX, &rid); - rid += PCIR_MSIX_CTRL; - msix_ctrl = pci_read_config(dev, rid, 2); - msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE; - pci_write_config(dev, rid, msix_ctrl, 2); -} - -/* - * Allocate MSI/X vectors from the OS. - * Returns 0 for legacy, 1 for MSI, >1 for MSIX. - */ -int -ixl_init_msix(struct ixl_pf *pf) -{ - device_t dev = pf->dev; - struct i40e_hw *hw = &pf->hw; - int auto_max_queues; - int rid, want, vectors, queues, available; -#ifdef IXL_IW - int iw_want, iw_vectors; - - pf->iw_msix = 0; -#endif - - /* Override by tuneable */ - if (!pf->enable_msix) - goto no_msix; - - /* Ensure proper operation in virtualized environment */ - ixl_set_busmaster(dev); - - /* First try MSI/X */ - rid = PCIR_BAR(IXL_MSIX_BAR); - pf->msix_mem = bus_alloc_resource_any(dev, - SYS_RES_MEMORY, &rid, RF_ACTIVE); - if (!pf->msix_mem) { - /* May not be enabled */ - device_printf(pf->dev, - "Unable to map MSIX table\n"); - goto no_msix; - } - - available = pci_msix_count(dev); - if (available < 2) { - /* system has msix disabled (0), or only one vector (1) */ - bus_release_resource(dev, SYS_RES_MEMORY, - rid, pf->msix_mem); - pf->msix_mem = NULL; - goto no_msix; - } - - /* Clamp max number of queues based on: - * - # of MSI-X vectors available - * - # of cpus available - * - # of queues that can be assigned to the LAN VSI - */ - auto_max_queues = min(mp_ncpus, available - 1); - if (hw->mac.type == I40E_MAC_X722) - auto_max_queues = min(auto_max_queues, 128); - else - auto_max_queues = min(auto_max_queues, 64); - - /* Override with tunable value if tunable is less than autoconfig count */ - if ((pf->max_queues != 0) && (pf->max_queues <= auto_max_queues)) - queues = pf->max_queues; - /* Use autoconfig amount if that's lower */ - else if ((pf->max_queues != 0) && (pf->max_queues > auto_max_queues)) { - device_printf(dev, "ixl_max_queues (%d) is too large, using " - "autoconfig amount (%d)...\n", - pf->max_queues, auto_max_queues); - queues = auto_max_queues; - } - /* Limit maximum auto-configured queues to 8 if no user value is set */ - else - queues = min(auto_max_queues, 8); - -#ifdef RSS - /* If we're doing RSS, clamp at the number of RSS buckets */ - if (queues > rss_getnumbuckets()) - queues = rss_getnumbuckets(); -#endif - - /* - ** Want one vector (RX/TX pair) per queue - ** plus an additional for the admin queue. - */ - want = queues + 1; - if (want <= available) /* Have enough */ - vectors = want; - else { - device_printf(pf->dev, - "MSIX Configuration Problem, " - "%d vectors available but %d wanted!\n", - available, want); - pf->msix_mem = NULL; - goto no_msix; /* Will go to Legacy setup */ - } - -#ifdef IXL_IW - if (ixl_enable_iwarp) { - /* iWARP wants additional vector for CQP */ - iw_want = mp_ncpus + 1; - available -= vectors; - if (available > 0) { - iw_vectors = (available >= iw_want) ? - iw_want : available; - vectors += iw_vectors; - } else - iw_vectors = 0; - } -#endif - - ixl_set_msix_enable(dev); - if (pci_alloc_msix(dev, &vectors) == 0) { - device_printf(pf->dev, - "Using MSIX interrupts with %d vectors\n", vectors); - pf->msix = vectors; -#ifdef IXL_IW - if (ixl_enable_iwarp) - pf->iw_msix = iw_vectors; -#endif - - pf->vsi.num_queues = queues; -#ifdef RSS - /* - * If we're doing RSS, the number of queues needs to - * match the number of RSS buckets that are configured. - * - * + If there's more queues than RSS buckets, we'll end - * up with queues that get no traffic. - * - * + If there's more RSS buckets than queues, we'll end - * up having multiple RSS buckets map to the same queue, - * so there'll be some contention. - */ - if (queues != rss_getnumbuckets()) { - device_printf(dev, - "%s: queues (%d) != RSS buckets (%d)" - "; performance will be impacted.\n", - __func__, queues, rss_getnumbuckets()); - } -#endif - return (vectors); - } -no_msix: - vectors = pci_msi_count(dev); - pf->vsi.num_queues = 1; - pf->max_queues = 1; - if (vectors == 1 && pci_alloc_msi(dev, &vectors) == 0) - device_printf(pf->dev, "Using an MSI interrupt\n"); - else { - vectors = 0; - device_printf(pf->dev, "Using a Legacy interrupt\n"); - } - return (vectors); -} - -/* * Configure admin queue/misc interrupt cause registers in hardware. */ void @@ -1535,7 +620,6 @@ I40E_PFINT_ICR0_ENA_VFLR_MASK | I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK; wr32(hw, I40E_PFINT_ICR0_ENA, reg); - /* * 0x7FF is the end of the queue list. * This means we won't use MSI-X vector 0 for a queue interrupt @@ -1554,29 +638,41 @@ /* * Configure queue interrupt cause registers in hardware. + * + * Linked list for each vector LNKLSTN(i) -> RQCTL(i) -> TQCTL(i) -> EOL */ void ixl_configure_queue_intr_msix(struct ixl_pf *pf) { - struct i40e_hw *hw = &pf->hw; + struct i40e_hw *hw = &pf->hw; struct ixl_vsi *vsi = &pf->vsi; u32 reg; u16 vector = 1; - for (int i = 0; i < vsi->num_queues; i++, vector++) { + // TODO: See if max is really necessary + for (int i = 0; i < max(vsi->num_rx_queues, vsi->num_tx_queues); i++, vector++) { + /* Make sure interrupt is disabled */ wr32(hw, I40E_PFINT_DYN_CTLN(i), 0); - /* First queue type is RX / 0 */ - wr32(hw, I40E_PFINT_LNKLSTN(i), i); + /* Set linked list head to point to corresponding RX queue + * e.g. vector 1 (LNKLSTN register 0) points to queue pair 0's RX queue */ + reg = ((i << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT) + & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK) | + ((I40E_QUEUE_TYPE_RX << I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT) + & I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_MASK); + wr32(hw, I40E_PFINT_LNKLSTN(i), reg); reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK | + // reg = /* I40E_QINT_RQCTL_CAUSE_ENA_MASK | */ (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | (i << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | (I40E_QUEUE_TYPE_TX << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT); wr32(hw, I40E_QINT_RQCTL(i), reg); + // TODO: Give this a TX ITR when its re-setup reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK | - (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | + //reg = /* I40E_QINT_TQCTL_CAUSE_ENA_MASK | */ + (IXL_ITR_NONE << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | (IXL_QUEUE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | (I40E_QUEUE_TYPE_RX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); @@ -1585,28 +681,21 @@ } /* - * Configure for MSI single vector operation + * Configure for single interrupt vector operation */ void ixl_configure_legacy(struct ixl_pf *pf) { struct i40e_hw *hw = &pf->hw; struct ixl_vsi *vsi = &pf->vsi; - struct ixl_queue *que = vsi->queues; - struct rx_ring *rxr = &que->rxr; - struct tx_ring *txr = &que->txr; - u32 reg; + u32 reg; /* Configure ITR */ - vsi->tx_itr_setting = pf->tx_itr; - wr32(hw, I40E_PFINT_ITR0(IXL_TX_ITR), - vsi->tx_itr_setting); - txr->itr = vsi->tx_itr_setting; - vsi->rx_itr_setting = pf->rx_itr; wr32(hw, I40E_PFINT_ITR0(IXL_RX_ITR), vsi->rx_itr_setting); - rxr->itr = vsi->rx_itr_setting; + /* XXX: Assuming only 1 queue in single interrupt mode */ + vsi->rx_queues[0].rxr.itr = vsi->rx_itr_setting; /* Setup "other" causes */ reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK @@ -1632,158 +721,33 @@ /* Associate the queue pair to the vector and enable the q int */ reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK | (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) - | (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); - wr32(hw, I40E_QINT_RQCTL(0), reg); - - reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK - | (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | (IXL_QUEUE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); - wr32(hw, I40E_QINT_TQCTL(0), reg); -} - -int -ixl_allocate_pci_resources(struct ixl_pf *pf) -{ - int rid; - struct i40e_hw *hw = &pf->hw; - device_t dev = pf->dev; - - /* Map BAR0 */ - rid = PCIR_BAR(0); - pf->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, - &rid, RF_ACTIVE); - - if (!(pf->pci_mem)) { - device_printf(dev, "Unable to allocate bus resource: PCI memory\n"); - return (ENXIO); - } - - /* Save off the PCI information */ - hw->vendor_id = pci_get_vendor(dev); - hw->device_id = pci_get_device(dev); - hw->revision_id = pci_read_config(dev, PCIR_REVID, 1); - hw->subsystem_vendor_id = - pci_read_config(dev, PCIR_SUBVEND_0, 2); - hw->subsystem_device_id = - pci_read_config(dev, PCIR_SUBDEV_0, 2); - - hw->bus.device = pci_get_slot(dev); - hw->bus.func = pci_get_function(dev); - - /* Save off register access information */ - pf->osdep.mem_bus_space_tag = - rman_get_bustag(pf->pci_mem); - pf->osdep.mem_bus_space_handle = - rman_get_bushandle(pf->pci_mem); - pf->osdep.mem_bus_space_size = rman_get_size(pf->pci_mem); - pf->osdep.flush_reg = I40E_GLGEN_STAT; - pf->hw.hw_addr = (u8 *) &pf->osdep.mem_bus_space_handle; - - pf->hw.back = &pf->osdep; - - return (0); -} - -/* - * Teardown and release the admin queue/misc vector - * interrupt. - */ -int -ixl_teardown_adminq_msix(struct ixl_pf *pf) -{ - device_t dev = pf->dev; - int rid, error = 0; - - if (pf->admvec) /* we are doing MSIX */ - rid = pf->admvec + 1; - else - (pf->msix != 0) ? (rid = 1):(rid = 0); - - if (pf->tag != NULL) { - bus_teardown_intr(dev, pf->res, pf->tag); - if (error) { - device_printf(dev, "bus_teardown_intr() for" - " interrupt 0 failed\n"); - // return (ENXIO); - } - pf->tag = NULL; - } - if (pf->res != NULL) { - bus_release_resource(dev, SYS_RES_IRQ, rid, pf->res); - if (error) { - device_printf(dev, "bus_release_resource() for" - " interrupt 0 failed [rid=%d]\n", rid); - // return (ENXIO); - } - pf->res = NULL; - } - - return (0); -} - -int -ixl_teardown_queue_msix(struct ixl_vsi *vsi) -{ - struct ixl_pf *pf = (struct ixl_pf *)vsi->back; - struct ixl_queue *que = vsi->queues; - device_t dev = vsi->dev; - int rid, error = 0; - - /* We may get here before stations are setup */ - if ((pf->msix < 2) || (que == NULL)) - return (0); - - /* Release all MSIX queue resources */ - for (int i = 0; i < vsi->num_queues; i++, que++) { - rid = que->msix + 1; - if (que->tag != NULL) { - error = bus_teardown_intr(dev, que->res, que->tag); - if (error) { - device_printf(dev, "bus_teardown_intr() for" - " Queue %d interrupt failed\n", - que->me); - // return (ENXIO); - } - que->tag = NULL; - } - if (que->res != NULL) { - error = bus_release_resource(dev, SYS_RES_IRQ, rid, que->res); - if (error) { - device_printf(dev, "bus_release_resource() for" - " Queue %d interrupt failed [rid=%d]\n", - que->me, rid); - // return (ENXIO); - } - que->res = NULL; - } - } - - return (0); + wr32(hw, I40E_QINT_RQCTL(0), reg); } void ixl_free_pci_resources(struct ixl_pf *pf) { - device_t dev = pf->dev; - int memrid; + struct ixl_vsi *vsi = &pf->vsi; + device_t dev = iflib_get_dev(vsi->ctx); + struct ixl_rx_queue *rx_que = vsi->rx_queues; - ixl_teardown_queue_msix(&pf->vsi); - ixl_teardown_adminq_msix(pf); + /* We may get here before stations are setup */ + if (rx_que == NULL) + goto early; - if (pf->msix > 0) - pci_release_msi(dev); - - memrid = PCIR_BAR(IXL_MSIX_BAR); + /* + ** Release all msix VSI resources: + */ + iflib_irq_free(vsi->ctx, &vsi->irq); - if (pf->msix_mem != NULL) - bus_release_resource(dev, SYS_RES_MEMORY, - memrid, pf->msix_mem); + for (int i = 0; i < vsi->num_rx_queues; i++, rx_que++) + iflib_irq_free(vsi->ctx, &rx_que->que_irq); +early: if (pf->pci_mem != NULL) bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(0), pf->pci_mem); - - return; } void @@ -1791,69 +755,69 @@ { /* Display supported media types */ if (phy_types & (I40E_CAP_PHY_TYPE_100BASE_TX)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_100_TX, 0, NULL); + ifmedia_add(vsi->media, IFM_ETHER | IFM_100_TX, 0, NULL); if (phy_types & (I40E_CAP_PHY_TYPE_1000BASE_T)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_T, 0, NULL); + ifmedia_add(vsi->media, IFM_ETHER | IFM_1000_T, 0, NULL); if (phy_types & (I40E_CAP_PHY_TYPE_1000BASE_SX)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_SX, 0, NULL); + ifmedia_add(vsi->media, IFM_ETHER | IFM_1000_SX, 0, NULL); if (phy_types & (I40E_CAP_PHY_TYPE_1000BASE_LX)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_LX, 0, NULL); + ifmedia_add(vsi->media, IFM_ETHER | IFM_1000_LX, 0, NULL); if (phy_types & (I40E_CAP_PHY_TYPE_XAUI) || phy_types & (I40E_CAP_PHY_TYPE_XFI) || phy_types & (I40E_CAP_PHY_TYPE_10GBASE_SFPP_CU)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX, 0, NULL); + ifmedia_add(vsi->media, IFM_ETHER | IFM_10G_TWINAX, 0, NULL); if (phy_types & (I40E_CAP_PHY_TYPE_10GBASE_SR)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SR, 0, NULL); + ifmedia_add(vsi->media, IFM_ETHER | IFM_10G_SR, 0, NULL); if (phy_types & (I40E_CAP_PHY_TYPE_10GBASE_LR)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_LR, 0, NULL); + ifmedia_add(vsi->media, IFM_ETHER | IFM_10G_LR, 0, NULL); if (phy_types & (I40E_CAP_PHY_TYPE_10GBASE_T)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_T, 0, NULL); + ifmedia_add(vsi->media, IFM_ETHER | IFM_10G_T, 0, NULL); if (phy_types & (I40E_CAP_PHY_TYPE_40GBASE_CR4) || phy_types & (I40E_CAP_PHY_TYPE_40GBASE_CR4_CU) || phy_types & (I40E_CAP_PHY_TYPE_40GBASE_AOC) || phy_types & (I40E_CAP_PHY_TYPE_XLAUI) || phy_types & (I40E_CAP_PHY_TYPE_40GBASE_KR4)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_CR4, 0, NULL); + ifmedia_add(vsi->media, IFM_ETHER | IFM_40G_CR4, 0, NULL); if (phy_types & (I40E_CAP_PHY_TYPE_40GBASE_SR4)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_SR4, 0, NULL); + ifmedia_add(vsi->media, IFM_ETHER | IFM_40G_SR4, 0, NULL); if (phy_types & (I40E_CAP_PHY_TYPE_40GBASE_LR4)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_LR4, 0, NULL); + ifmedia_add(vsi->media, IFM_ETHER | IFM_40G_LR4, 0, NULL); if (phy_types & (I40E_CAP_PHY_TYPE_1000BASE_KX)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_KX, 0, NULL); + ifmedia_add(vsi->media, IFM_ETHER | IFM_1000_KX, 0, NULL); if (phy_types & (I40E_CAP_PHY_TYPE_10GBASE_CR1_CU) || phy_types & (I40E_CAP_PHY_TYPE_10GBASE_CR1)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_CR1, 0, NULL); + ifmedia_add(vsi->media, IFM_ETHER | IFM_10G_CR1, 0, NULL); if (phy_types & (I40E_CAP_PHY_TYPE_10GBASE_AOC)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX_LONG, 0, NULL); + ifmedia_add(vsi->media, IFM_ETHER | IFM_10G_TWINAX_LONG, 0, NULL); if (phy_types & (I40E_CAP_PHY_TYPE_SFI)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SFI, 0, NULL); + ifmedia_add(vsi->media, IFM_ETHER | IFM_10G_SFI, 0, NULL); if (phy_types & (I40E_CAP_PHY_TYPE_10GBASE_KX4)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_KX4, 0, NULL); + ifmedia_add(vsi->media, IFM_ETHER | IFM_10G_KX4, 0, NULL); if (phy_types & (I40E_CAP_PHY_TYPE_10GBASE_KR)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_KR, 0, NULL); + ifmedia_add(vsi->media, IFM_ETHER | IFM_10G_KR, 0, NULL); if (phy_types & (I40E_CAP_PHY_TYPE_20GBASE_KR2)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_20G_KR2, 0, NULL); + ifmedia_add(vsi->media, IFM_ETHER | IFM_20G_KR2, 0, NULL); if (phy_types & (I40E_CAP_PHY_TYPE_40GBASE_KR4)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_KR4, 0, NULL); + ifmedia_add(vsi->media, IFM_ETHER | IFM_40G_KR4, 0, NULL); if (phy_types & (I40E_CAP_PHY_TYPE_XLPPI)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_XLPPI, 0, NULL); + ifmedia_add(vsi->media, IFM_ETHER | IFM_40G_XLPPI, 0, NULL); if (phy_types & (I40E_CAP_PHY_TYPE_25GBASE_KR)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_25G_KR, 0, NULL); + ifmedia_add(vsi->media, IFM_ETHER | IFM_25G_KR, 0, NULL); if (phy_types & (I40E_CAP_PHY_TYPE_25GBASE_CR)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_25G_CR, 0, NULL); + ifmedia_add(vsi->media, IFM_ETHER | IFM_25G_CR, 0, NULL); if (phy_types & (I40E_CAP_PHY_TYPE_25GBASE_SR)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_25G_SR, 0, NULL); + ifmedia_add(vsi->media, IFM_ETHER | IFM_25G_SR, 0, NULL); if (phy_types & (I40E_CAP_PHY_TYPE_25GBASE_LR)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_UNKNOWN, 0, NULL); + ifmedia_add(vsi->media, IFM_ETHER | IFM_UNKNOWN, 0, NULL); } /********************************************************************* @@ -1864,64 +828,27 @@ int ixl_setup_interface(device_t dev, struct ixl_vsi *vsi) { - struct ixl_pf *pf = (struct ixl_pf *)vsi->back; - struct ifnet *ifp; - struct i40e_hw *hw = vsi->hw; - struct ixl_queue *que = vsi->queues; + if_ctx_t ctx = vsi->ctx; + struct ixl_pf *pf = vsi->back; + struct i40e_hw *hw = &pf->hw; + struct ifnet *ifp = iflib_get_ifp(ctx); struct i40e_aq_get_phy_abilities_resp abilities; enum i40e_status_code aq_error = 0; - - INIT_DEBUGOUT("ixl_setup_interface: begin"); - - ifp = vsi->ifp = if_alloc(IFT_ETHER); - if (ifp == NULL) { - device_printf(dev, "can not allocate ifnet structure\n"); - return (-1); - } - if_initname(ifp, device_get_name(dev), device_get_unit(dev)); - ifp->if_mtu = ETHERMTU; - ifp->if_init = ixl_init; - ifp->if_softc = vsi; - ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; - ifp->if_ioctl = ixl_ioctl; - -#if __FreeBSD_version >= 1100036 - if_setgetcounterfn(ifp, ixl_get_counter); -#endif - - ifp->if_transmit = ixl_mq_start; - - ifp->if_qflush = ixl_qflush; - - ifp->if_snd.ifq_maxlen = que->num_desc - 2; - - vsi->max_frame_size = + uint64_t cap; + + INIT_DBG_DEV(dev, "begin"); + + /* initialize fast path functions */ + cap = IXL_CAPS; + if_setifheaderlen(ifp, sizeof(struct ether_vlan_header)); + if_setcapabilitiesbit(ifp, cap, 0); + if_setcapenable(ifp, if_getcapabilities(ifp)); + if_setbaudrate(ifp, IF_Gbps(40)); + /* TODO: Remove VLAN_ENCAP_LEN? */ + vsi->shared->isc_max_frame_size = ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN + ETHER_VLAN_ENCAP_LEN; - /* Set TSO limits */ - ifp->if_hw_tsomax = IP_MAXPACKET - (ETHER_HDR_LEN + ETHER_CRC_LEN); - ifp->if_hw_tsomaxsegcount = IXL_MAX_TSO_SEGS; - ifp->if_hw_tsomaxsegsize = PAGE_SIZE; - - /* - * Tell the upper layer(s) we support long frames. - */ - ifp->if_hdrlen = sizeof(struct ether_vlan_header); - - ifp->if_capabilities |= IFCAP_HWCSUM; - ifp->if_capabilities |= IFCAP_HWCSUM_IPV6; - ifp->if_capabilities |= IFCAP_TSO; - ifp->if_capabilities |= IFCAP_JUMBO_MTU; - ifp->if_capabilities |= IFCAP_LRO; - - /* VLAN capabilties */ - ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING - | IFCAP_VLAN_HWTSO - | IFCAP_VLAN_MTU - | IFCAP_VLAN_HWCSUM; - ifp->if_capenable = ifp->if_capabilities; - /* ** Don't turn this on by default, if vlans are ** created on another pseudo device (eg. lagg) @@ -1930,19 +857,13 @@ ** using vlans directly on the ixl driver you can ** enable this and get full hardware tag filtering. */ - ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; - - /* - * Specify the media types supported by this adapter and register - * callbacks to update media and link information - */ - ifmedia_init(&vsi->media, IFM_IMASK, ixl_media_change, - ixl_media_status); + if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWFILTER, 0); aq_error = i40e_aq_get_phy_capabilities(hw, FALSE, TRUE, &abilities, NULL); /* May need delay to detect fiber correctly */ if (aq_error == I40E_ERR_UNKNOWN_PHY) { + /* TODO: Maybe just retry this in a task... */ i40e_msec_delay(200); aq_error = i40e_aq_get_phy_capabilities(hw, FALSE, TRUE, &abilities, NULL); @@ -1959,13 +880,11 @@ pf->supported_speeds = abilities.link_speed; ifp->if_baudrate = ixl_max_aq_speed_to_value(pf->supported_speeds); - ixl_add_ifmedia(vsi, hw->phy.phy_types); + ixl_add_ifmedia(vsi, abilities.phy_type); /* Use autoselect media by default */ - ifmedia_add(&vsi->media, IFM_ETHER | IFM_AUTO, 0, NULL); - ifmedia_set(&vsi->media, IFM_ETHER | IFM_AUTO); - - ether_ifattach(ifp, hw->mac.addr); + ifmedia_add(vsi->media, IFM_ETHER | IFM_AUTO, 0, NULL); + ifmedia_set(vsi->media, IFM_ETHER | IFM_AUTO); return (0); } @@ -1976,8 +895,8 @@ void ixl_link_event(struct ixl_pf *pf, struct i40e_arq_event_info *e) { - struct i40e_hw *hw = &pf->hw; - device_t dev = pf->dev; + struct i40e_hw *hw = &pf->hw; + device_t dev = iflib_get_dev(pf->vsi.ctx); struct i40e_aqc_get_link_status *status = (struct i40e_aqc_get_link_status *)&e->desc.params.raw; @@ -1992,8 +911,7 @@ device_printf(dev, "Link failed because " "an unqualified module was detected!\n"); - /* Update OS link info */ - ixl_update_link_status(pf); + /* OS link info is updated elsewhere */ } /********************************************************************* @@ -2008,7 +926,7 @@ { struct i40e_hw *hw = &pf->hw; struct ixl_vsi *vsi = &pf->vsi; - device_t dev = vsi->dev; + device_t dev = iflib_get_dev(vsi->ctx); struct i40e_aqc_get_switch_config_resp *sw_config; u8 aq_buf[I40E_AQ_LARGE_BUF]; int ret; @@ -2029,7 +947,7 @@ sw_config->header.num_reported, sw_config->header.num_total); for (int i = 0; i < sw_config->header.num_reported; i++) { device_printf(dev, - "-> %d: type=%d seid=%d uplink=%d downlink=%d\n", i, + "%d: type=%d seid=%d uplink=%d downlink=%d\n", i, sw_config->element[i].element_type, sw_config->element[i].seid, sw_config->element[i].uplink_seid, @@ -2053,9 +971,11 @@ int ixl_initialize_vsi(struct ixl_vsi *vsi) { + if_softc_ctx_t scctx = iflib_get_softc_ctx(vsi->ctx); struct ixl_pf *pf = vsi->back; - struct ixl_queue *que = vsi->queues; - device_t dev = vsi->dev; + struct ixl_tx_queue *tx_que = vsi->tx_queues; + struct ixl_rx_queue *rx_que = vsi->rx_queues; + device_t dev = iflib_get_dev(vsi->ctx); struct i40e_hw *hw = vsi->hw; struct i40e_vsi_context ctxt; int tc_queues; @@ -2103,6 +1023,7 @@ /* Set VLAN receive stripping mode */ ctxt.info.valid_sections |= I40E_AQ_VSI_PROP_VLAN_VALID; ctxt.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL; + // TODO: Call function to get this cap bit, instead if (vsi->ifp->if_capenable & IFCAP_VLAN_HWTAGGING) ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH; else @@ -2134,26 +1055,27 @@ return (err); } - for (int i = 0; i < vsi->num_queues; i++, que++) { - struct tx_ring *txr = &que->txr; - struct rx_ring *rxr = &que->rxr; + for (int i = 0; i < vsi->num_tx_queues; i++, tx_que++) { + struct tx_ring *txr = &tx_que->txr; struct i40e_hmc_obj_txq tctx; - struct i40e_hmc_obj_rxq rctx; u32 txctl; - u16 size; /* Setup the HMC TX Context */ - size = que->num_desc * sizeof(struct i40e_tx_desc); memset(&tctx, 0, sizeof(struct i40e_hmc_obj_txq)); tctx.new_context = 1; - tctx.base = (txr->dma.pa/IXL_TX_CTX_BASE_UNITS); - tctx.qlen = que->num_desc; + tctx.base = (txr->tx_paddr/IXL_TX_CTX_BASE_UNITS); + tctx.qlen = scctx->isc_ntxd[0]; tctx.fc_ena = 0; tctx.rdylist = vsi->info.qs_handle[0]; /* index is TC */ /* Enable HEAD writeback */ - tctx.head_wb_ena = 1; - tctx.head_wb_addr = txr->dma.pa + - (que->num_desc * sizeof(struct i40e_tx_desc)); + if (pf->enable_hwb) { + tctx.head_wb_ena = 1; + tctx.head_wb_addr = txr->tx_paddr + + (scctx->isc_ntxd[0] * sizeof(struct i40e_tx_desc)); + } else { + tctx.head_wb_ena = 0; + tctx.head_wb_addr = 0; + } tctx.rdylist_act = 0; err = i40e_clear_lan_tx_queue_context(hw, i); if (err) { @@ -2173,10 +1095,14 @@ ixl_flush(hw); /* Do ring (re)init */ - ixl_init_tx_ring(que); + ixl_init_tx_ring(vsi, tx_que); + } + for (int i = 0; i < vsi->num_rx_queues; i++, rx_que++) { + struct rx_ring *rxr = &rx_que->rxr; + struct i40e_hmc_obj_rxq rctx; /* Next setup the HMC RX Context */ - if (vsi->max_frame_size <= MCLBYTES) + if (scctx->isc_max_frame_size <= MCLBYTES) rxr->mbuf_sz = MCLBYTES; else rxr->mbuf_sz = MJUMPAGESIZE; @@ -2188,13 +1114,13 @@ rctx.dbuff = rxr->mbuf_sz >> I40E_RXQ_CTX_DBUFF_SHIFT; /* ignore header split for now */ rctx.hbuff = 0 >> I40E_RXQ_CTX_HBUFF_SHIFT; - rctx.rxmax = (vsi->max_frame_size < max_rxmax) ? - vsi->max_frame_size : max_rxmax; + rctx.rxmax = (scctx->isc_max_frame_size < max_rxmax) ? + scctx->isc_max_frame_size : max_rxmax; rctx.dtype = 0; rctx.dsize = 1; /* do 32byte descriptors */ rctx.hsplit_0 = 0; /* no HDR split initially */ - rctx.base = (rxr->dma.pa/IXL_RX_CTX_BASE_UNITS); - rctx.qlen = que->num_desc; + rctx.base = (rxr->rx_paddr/IXL_RX_CTX_BASE_UNITS); + rctx.qlen = scctx->isc_nrxd[0]; rctx.tphrdesc_ena = 1; rctx.tphwdesc_ena = 1; rctx.tphdata_ena = 0; @@ -2217,70 +1143,10 @@ device_printf(dev, "Unable to set RX context %d\n", i); break; } - err = ixl_init_rx_ring(que); - if (err) { - device_printf(dev, "Fail in init_rx_ring %d\n", i); - break; - } -#ifdef DEV_NETMAP - /* preserve queue */ - if (vsi->ifp->if_capenable & IFCAP_NETMAP) { - struct netmap_adapter *na = NA(vsi->ifp); - struct netmap_kring *kring = &na->rx_rings[i]; - int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring); - wr32(vsi->hw, I40E_QRX_TAIL(que->me), t); - } else -#endif /* DEV_NETMAP */ - wr32(vsi->hw, I40E_QRX_TAIL(que->me), que->num_desc - 1); } return (err); } - -/********************************************************************* - * - * Free all VSI structs. - * - **********************************************************************/ -void -ixl_free_vsi(struct ixl_vsi *vsi) -{ - struct ixl_pf *pf = (struct ixl_pf *)vsi->back; - struct ixl_queue *que = vsi->queues; - - /* Free station queues */ - if (!vsi->queues) - goto free_filters; - - for (int i = 0; i < vsi->num_queues; i++, que++) { - struct tx_ring *txr = &que->txr; - struct rx_ring *rxr = &que->rxr; - - if (!mtx_initialized(&txr->mtx)) /* uninitialized */ - continue; - IXL_TX_LOCK(txr); - ixl_free_que_tx(que); - if (txr->base) - i40e_free_dma_mem(&pf->hw, &txr->dma); - IXL_TX_UNLOCK(txr); - IXL_TX_LOCK_DESTROY(txr); - - if (!mtx_initialized(&rxr->mtx)) /* uninitialized */ - continue; - IXL_RX_LOCK(rxr); - ixl_free_que_rx(que); - if (rxr->base) - i40e_free_dma_mem(&pf->hw, &rxr->dma); - IXL_RX_UNLOCK(rxr); - IXL_RX_LOCK_DESTROY(rxr); - } - free(vsi->queues, M_DEVBUF); - -free_filters: - /* Free VSI filter list */ - ixl_free_mac_filters(vsi); -} - void ixl_free_mac_filters(struct ixl_vsi *vsi) { @@ -2294,153 +1160,11 @@ } /* - * Fill out fields in queue struct and setup tx/rx memory and structs - */ -static int -ixl_setup_queue(struct ixl_queue *que, struct ixl_pf *pf, int index) -{ - device_t dev = pf->dev; - struct i40e_hw *hw = &pf->hw; - struct ixl_vsi *vsi = &pf->vsi; - struct tx_ring *txr = &que->txr; - struct rx_ring *rxr = &que->rxr; - int error = 0; - int rsize, tsize; - - que->num_desc = pf->ringsz; - que->me = index; - que->vsi = vsi; - - txr->que = que; - txr->tail = I40E_QTX_TAIL(que->me); - - /* Initialize the TX lock */ - snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", - device_get_nameunit(dev), que->me); - mtx_init(&txr->mtx, txr->mtx_name, NULL, MTX_DEF); - /* Create the TX descriptor ring */ - tsize = roundup2((que->num_desc * - sizeof(struct i40e_tx_desc)) + - sizeof(u32), DBA_ALIGN); - if (i40e_allocate_dma_mem(hw, - &txr->dma, i40e_mem_reserved, tsize, DBA_ALIGN)) { - device_printf(dev, - "Unable to allocate TX Descriptor memory\n"); - error = ENOMEM; - goto fail; - } - txr->base = (struct i40e_tx_desc *)txr->dma.va; - bzero((void *)txr->base, tsize); - /* Now allocate transmit soft structs for the ring */ - if (ixl_allocate_tx_data(que)) { - device_printf(dev, - "Critical Failure setting up TX structures\n"); - error = ENOMEM; - goto fail; - } - /* Allocate a buf ring */ - txr->br = buf_ring_alloc(DEFAULT_TXBRSZ, M_DEVBUF, - M_NOWAIT, &txr->mtx); - if (txr->br == NULL) { - device_printf(dev, - "Critical Failure setting up TX buf ring\n"); - error = ENOMEM; - goto fail; - } - - rsize = roundup2(que->num_desc * - sizeof(union i40e_rx_desc), DBA_ALIGN); - rxr->que = que; - rxr->tail = I40E_QRX_TAIL(que->me); - - /* Initialize the RX side lock */ - snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", - device_get_nameunit(dev), que->me); - mtx_init(&rxr->mtx, rxr->mtx_name, NULL, MTX_DEF); - - if (i40e_allocate_dma_mem(hw, - &rxr->dma, i40e_mem_reserved, rsize, 4096)) { - device_printf(dev, - "Unable to allocate RX Descriptor memory\n"); - error = ENOMEM; - goto fail; - } - rxr->base = (union i40e_rx_desc *)rxr->dma.va; - bzero((void *)rxr->base, rsize); - /* Allocate receive soft structs for the ring*/ - if (ixl_allocate_rx_data(que)) { - device_printf(dev, - "Critical Failure setting up receive structs\n"); - error = ENOMEM; - goto fail; - } - - return (0); -fail: - if (rxr->base) - i40e_free_dma_mem(&pf->hw, &rxr->dma); - if (mtx_initialized(&rxr->mtx)) - mtx_destroy(&rxr->mtx); - if (txr->br) { - buf_ring_free(txr->br, M_DEVBUF); - txr->br = NULL; - } - if (txr->base) - i40e_free_dma_mem(&pf->hw, &txr->dma); - if (mtx_initialized(&txr->mtx)) - mtx_destroy(&txr->mtx); - - return (error); -} - -/********************************************************************* - * - * Allocate memory for the VSI (virtual station interface) and their - * associated queues, rings and the descriptors associated with each, - * called only once at attach. - * - **********************************************************************/ -int -ixl_setup_stations(struct ixl_pf *pf) -{ - device_t dev = pf->dev; - struct ixl_vsi *vsi; - struct ixl_queue *que; - int error = 0; - - vsi = &pf->vsi; - vsi->back = (void *)pf; - vsi->hw = &pf->hw; - vsi->id = 0; - vsi->num_vlans = 0; - vsi->back = pf; - - /* Get memory for the station queues */ - if (!(vsi->queues = - (struct ixl_queue *) malloc(sizeof(struct ixl_queue) * - vsi->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { - device_printf(dev, "Unable to allocate queue memory\n"); - error = ENOMEM; - return (error); - } - - /* Then setup each queue */ - for (int i = 0; i < vsi->num_queues; i++) { - que = &vsi->queues[i]; - error = ixl_setup_queue(que, pf, i); - if (error) - return (error); - } - - return (0); -} - -/* ** Provide a update to the queue RX ** interrupt moderation value. */ void -ixl_set_queue_rx_itr(struct ixl_queue *que) +ixl_set_queue_rx_itr(struct ixl_rx_queue *que) { struct ixl_vsi *vsi = que->vsi; struct ixl_pf *pf = (struct ixl_pf *)vsi->back; @@ -2491,7 +1215,7 @@ ((9 * rx_itr) + rxr->itr); rxr->itr = min(rx_itr, IXL_MAX_ITR); wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, - que->me), rxr->itr); + rxr->me), rxr->itr); } } else { /* We may have have toggled to non-dynamic */ if (vsi->rx_itr_setting & IXL_ITR_DYNAMIC) @@ -2500,7 +1224,7 @@ if (rxr->itr != vsi->rx_itr_setting) { rxr->itr = vsi->rx_itr_setting; wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, - que->me), rxr->itr); + rxr->me), rxr->itr); } } rxr->bytes = 0; @@ -2508,81 +1232,6 @@ return; } - -/* -** Provide a update to the queue TX -** interrupt moderation value. -*/ -void -ixl_set_queue_tx_itr(struct ixl_queue *que) -{ - struct ixl_vsi *vsi = que->vsi; - struct ixl_pf *pf = (struct ixl_pf *)vsi->back; - struct i40e_hw *hw = vsi->hw; - struct tx_ring *txr = &que->txr; - u16 tx_itr; - u16 tx_latency = 0; - int tx_bytes; - - - /* Idle, do nothing */ - if (txr->bytes == 0) - return; - - if (pf->dynamic_tx_itr) { - tx_bytes = txr->bytes/txr->itr; - tx_itr = txr->itr; - - switch (txr->latency) { - case IXL_LOW_LATENCY: - if (tx_bytes > 10) { - tx_latency = IXL_AVE_LATENCY; - tx_itr = IXL_ITR_20K; - } - break; - case IXL_AVE_LATENCY: - if (tx_bytes > 20) { - tx_latency = IXL_BULK_LATENCY; - tx_itr = IXL_ITR_8K; - } else if (tx_bytes <= 10) { - tx_latency = IXL_LOW_LATENCY; - tx_itr = IXL_ITR_100K; - } - break; - case IXL_BULK_LATENCY: - if (tx_bytes <= 20) { - tx_latency = IXL_AVE_LATENCY; - tx_itr = IXL_ITR_20K; - } - break; - } - - txr->latency = tx_latency; - - if (tx_itr != txr->itr) { - /* do an exponential smoothing */ - tx_itr = (10 * tx_itr * txr->itr) / - ((9 * tx_itr) + txr->itr); - txr->itr = min(tx_itr, IXL_MAX_ITR); - wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, - que->me), txr->itr); - } - - } else { /* We may have have toggled to non-dynamic */ - if (vsi->tx_itr_setting & IXL_ITR_DYNAMIC) - vsi->tx_itr_setting = pf->tx_itr; - /* Update the hardware if needed */ - if (txr->itr != vsi->tx_itr_setting) { - txr->itr = vsi->tx_itr_setting; - wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, - que->me), txr->itr); - } - } - txr->bytes = 0; - txr->packets = 0; - return; -} - void ixl_add_vsi_sysctls(struct ixl_pf *pf, struct ixl_vsi *vsi, struct sysctl_ctx_list *ctx, const char *sysctl_name) @@ -2606,7 +1255,7 @@ * Retrieves I40E_QTX_TAIL value from hardware * for a sysctl. */ -static int +int ixl_sysctl_qtx_tail_handler(SYSCTL_HANDLER_ARGS) { struct ixl_queue *que; @@ -2628,7 +1277,7 @@ * Retrieves I40E_QRX_TAIL value from hardware * for a sysctl. */ -static int +int ixl_sysctl_qrx_tail_handler(SYSCTL_HANDLER_ARGS) { struct ixl_queue *que; @@ -2647,40 +1296,6 @@ #endif /* - * Used to set the Tx ITR value for all of the PF LAN VSI's queues. - * Writes to the ITR registers immediately. - */ -static int -ixl_sysctl_pf_tx_itr(SYSCTL_HANDLER_ARGS) -{ - struct ixl_pf *pf = (struct ixl_pf *)arg1; - device_t dev = pf->dev; - int error = 0; - int requested_tx_itr; - - requested_tx_itr = pf->tx_itr; - error = sysctl_handle_int(oidp, &requested_tx_itr, 0, req); - if ((error) || (req->newptr == NULL)) - return (error); - if (pf->dynamic_tx_itr) { - device_printf(dev, - "Cannot set TX itr value while dynamic TX itr is enabled\n"); - return (EINVAL); - } - if (requested_tx_itr < 0 || requested_tx_itr > IXL_MAX_ITR) { - device_printf(dev, - "Invalid TX itr value; value must be between 0 and %d\n", - IXL_MAX_ITR); - return (EINVAL); - } - - pf->tx_itr = requested_tx_itr; - ixl_configure_tx_itr(pf); - - return (error); -} - -/* * Used to set the Rx ITR value for all of the PF LAN VSI's queues. * Writes to the ITR registers immediately. */ @@ -2717,9 +1332,8 @@ void ixl_add_hw_stats(struct ixl_pf *pf) { - device_t dev = pf->dev; struct ixl_vsi *vsi = &pf->vsi; - struct ixl_queue *queues = vsi->queues; + device_t dev = iflib_get_dev(vsi->ctx); struct i40e_hw_port_stats *pf_stats = &pf->stats; struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); @@ -2727,13 +1341,6 @@ struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree); struct sysctl_oid_list *vsi_list; - struct sysctl_oid *queue_node; - struct sysctl_oid_list *queue_list; - - struct tx_ring *txr; - struct rx_ring *rxr; - char queue_namebuf[QUEUE_NAME_LEN]; - /* Driver statistics */ SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events", CTLFLAG_RD, &pf->watchdog_events, @@ -2745,78 +1352,6 @@ ixl_add_vsi_sysctls(pf, &pf->vsi, ctx, "pf"); vsi_list = SYSCTL_CHILDREN(pf->vsi.vsi_node); - /* Queue statistics */ - for (int q = 0; q < vsi->num_queues; q++) { - snprintf(queue_namebuf, QUEUE_NAME_LEN, "que%d", q); - queue_node = SYSCTL_ADD_NODE(ctx, vsi_list, - OID_AUTO, queue_namebuf, CTLFLAG_RD, NULL, "Queue #"); - queue_list = SYSCTL_CHILDREN(queue_node); - - txr = &(queues[q].txr); - rxr = &(queues[q].rxr); - - SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "mbuf_defrag_failed", - CTLFLAG_RD, &(queues[q].mbuf_defrag_failed), - "m_defrag() failed"); - SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs", - CTLFLAG_RD, &(queues[q].irqs), - "irqs on this queue"); - SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tso_tx", - CTLFLAG_RD, &(queues[q].tso), - "TSO"); - SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_dmamap_failed", - CTLFLAG_RD, &(queues[q].tx_dmamap_failed), - "Driver tx dma failure in xmit"); - SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "mss_too_small", - CTLFLAG_RD, &(queues[q].mss_too_small), - "TSO sends with an MSS less than 64"); - SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail", - CTLFLAG_RD, &(txr->no_desc), - "Queue No Descriptor Available"); - SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets", - CTLFLAG_RD, &(txr->total_packets), - "Queue Packets Transmitted"); - SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_bytes", - CTLFLAG_RD, &(txr->tx_bytes), - "Queue Bytes Transmitted"); - SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets", - CTLFLAG_RD, &(rxr->rx_packets), - "Queue Packets Received"); - SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes", - CTLFLAG_RD, &(rxr->rx_bytes), - "Queue Bytes Received"); - SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_desc_err", - CTLFLAG_RD, &(rxr->desc_errs), - "Queue Rx Descriptor Errors"); - SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "rx_itr", - CTLFLAG_RD, &(rxr->itr), 0, - "Queue Rx ITR Interval"); - SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "tx_itr", - CTLFLAG_RD, &(txr->itr), 0, - "Queue Tx ITR Interval"); -#ifdef IXL_DEBUG - SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_not_done", - CTLFLAG_RD, &(rxr->not_done), - "Queue Rx Descriptors not Done"); - SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "rx_next_refresh", - CTLFLAG_RD, &(rxr->next_refresh), 0, - "Queue Rx Descriptors not Done"); - SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "rx_next_check", - CTLFLAG_RD, &(rxr->next_check), 0, - "Queue Rx Descriptors not Done"); - SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "qtx_tail", - CTLTYPE_UINT | CTLFLAG_RD, &queues[q], - sizeof(struct ixl_queue), - ixl_sysctl_qtx_tail_handler, "IU", - "Queue Transmit Descriptor Tail"); - SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "qrx_tail", - CTLTYPE_UINT | CTLFLAG_RD, &queues[q], - sizeof(struct ixl_queue), - ixl_sysctl_qrx_tail_handler, "IU", - "Queue Receive Descriptor Tail"); -#endif - } - /* MAC stats */ ixl_add_sysctls_mac_stats(ctx, child, pf_stats); } @@ -2995,8 +1530,8 @@ ixl_set_rss_hlut(struct ixl_pf *pf) { struct i40e_hw *hw = &pf->hw; - device_t dev = pf->dev; struct ixl_vsi *vsi = &pf->vsi; + device_t dev = iflib_get_dev(vsi->ctx); int i, que_id; int lut_entry_width; u32 lut = 0; @@ -3017,9 +1552,9 @@ * num_queues.) */ que_id = rss_get_indirection_to_bucket(i); - que_id = que_id % vsi->num_queues; + que_id = que_id % vsi->num_rx_queues; #else - que_id = i % vsi->num_queues; + que_id = i % vsi->num_rx_queues; #endif lut = (que_id & ((0x1 << lut_entry_width) - 1)); hlut_buf[i] = lut; @@ -3049,56 +1584,6 @@ } /* -** This routine is run via an vlan config EVENT, -** it enables us to use the HW Filter table since -** we can get the vlan id. This just creates the -** entry in the soft version of the VFTA, init will -** repopulate the real table. -*/ -void -ixl_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) -{ - struct ixl_vsi *vsi = ifp->if_softc; - struct i40e_hw *hw = vsi->hw; - struct ixl_pf *pf = (struct ixl_pf *)vsi->back; - - if (ifp->if_softc != arg) /* Not our event */ - return; - - if ((vtag == 0) || (vtag > 4095)) /* Invalid */ - return; - - IXL_PF_LOCK(pf); - ++vsi->num_vlans; - ixl_add_filter(vsi, hw->mac.addr, vtag); - IXL_PF_UNLOCK(pf); -} - -/* -** This routine is run via an vlan -** unconfig EVENT, remove our entry -** in the soft vfta. -*/ -void -ixl_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) -{ - struct ixl_vsi *vsi = ifp->if_softc; - struct i40e_hw *hw = vsi->hw; - struct ixl_pf *pf = (struct ixl_pf *)vsi->back; - - if (ifp->if_softc != arg) - return; - - if ((vtag == 0) || (vtag > 4095)) /* Invalid */ - return; - - IXL_PF_LOCK(pf); - --vsi->num_vlans; - ixl_del_filter(vsi, hw->mac.addr, vtag); - IXL_PF_UNLOCK(pf); -} - -/* ** This routine updates vlan filters, called by init ** it scans the filter table and then updates the hw ** after a soft reset. @@ -3131,7 +1616,6 @@ flags = IXL_FILTER_VLAN; flags |= (IXL_FILTER_ADD | IXL_FILTER_USED); ixl_add_hw_filters(vsi, flags, cnt); - return; } /* @@ -3145,8 +1629,7 @@ { struct ixl_pf *pf = (struct ixl_pf *)vsi->back; - /* Add broadcast address */ - ixl_add_filter(vsi, ixl_bcast_addr, IXL_VLAN_ANY); + /* TODO: Set broadcast promiscuous here */ /* * Prevent Tx flow control frames from being sent out by @@ -3299,9 +1782,8 @@ int err, j = 0; pf = vsi->back; - dev = pf->dev; + dev = iflib_get_dev(vsi->ctx); hw = &pf->hw; - IXL_PF_LOCK_ASSERT(pf); a = malloc(sizeof(struct i40e_aqc_add_macvlan_element_data) * cnt, M_DEVBUF, M_NOWAIT | M_ZERO); @@ -3364,7 +1846,7 @@ pf = vsi->back; hw = &pf->hw; - dev = pf->dev; + dev = iflib_get_dev(vsi->ctx); d = malloc(sizeof(struct i40e_aqc_remove_macvlan_element_data) * cnt, M_DEVBUF, M_NOWAIT | M_ZERO); @@ -3496,11 +1978,11 @@ struct ixl_pf *pf = vsi->back; int error = 0; - for (int i = 0; i < vsi->num_queues; i++) { - error = ixl_enable_ring(pf, &pf->qtag, i); - if (error) - return (error); - } + for (int i = 0; i < vsi->num_tx_queues; i++) + error = ixl_enable_tx_ring(pf, &pf->qtag, i); + + for (int i = 0; i < vsi->num_rx_queues; i++) + error = ixl_enable_rx_ring(pf, &pf->qtag, i); return (error); } @@ -3586,11 +2068,12 @@ struct ixl_pf *pf = vsi->back; int error = 0; - for (int i = 0; i < vsi->num_queues; i++) { - error = ixl_disable_ring(pf, &pf->qtag, i); - if (error) - return (error); - } + for (int i = 0; i < vsi->num_tx_queues; i++) + error = ixl_disable_tx_ring(pf, &pf->qtag, i); + + for (int i = 0; i < vsi->num_rx_queues; i++) + error = ixl_disable_rx_ring(pf, &pf->qtag, i); + return (error); } @@ -3606,8 +2089,10 @@ { struct i40e_hw *hw = &pf->hw; device_t dev = pf->dev; + struct ixl_vf *vf; bool mdd_detected = false; bool pf_mdd_detected = false; + bool vf_mdd_detected = false; u32 reg; /* find what triggered the MDD event */ @@ -3647,18 +2132,51 @@ if (reg & I40E_PF_MDET_TX_VALID_MASK) { wr32(hw, I40E_PF_MDET_TX, 0xFFFF); device_printf(dev, - "MDD TX event is for this function!"); + "MDD TX event is for this function!\n"); pf_mdd_detected = true; } reg = rd32(hw, I40E_PF_MDET_RX); if (reg & I40E_PF_MDET_RX_VALID_MASK) { wr32(hw, I40E_PF_MDET_RX, 0xFFFF); device_printf(dev, - "MDD RX event is for this function!"); + "MDD RX event is for this function!\n"); pf_mdd_detected = true; } } + if (pf_mdd_detected) { + atomic_set_32(&pf->state, IXL_PF_STATE_PF_RESET_REQ); + goto end; + } + + // Handle VF detection + for (int i = 0; i < pf->num_vfs && mdd_detected; i++) { + vf = &(pf->vfs[i]); + reg = rd32(hw, I40E_VP_MDET_TX(i)); + if (reg & I40E_VP_MDET_TX_VALID_MASK) { + wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF); + vf->num_mdd_events++; + device_printf(dev, "MDD TX event is for VF %d\n", i); + vf_mdd_detected = true; + } + + reg = rd32(hw, I40E_VP_MDET_RX(i)); + if (reg & I40E_VP_MDET_RX_VALID_MASK) { + wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF); + vf->num_mdd_events++; + device_printf(dev, "MDD RX event is for VF %d\n", i); + vf_mdd_detected = true; + } + + // TODO: Disable VF if there are too many MDD events from it + } + + if (vf_mdd_detected) + atomic_set_32(&pf->state, IXL_PF_STATE_VF_RESET_REQ); + +end: + atomic_clear_32(&pf->state, IXL_PF_STATE_MDD_PENDING); + /* re-enable mdd interrupt cause */ reg = rd32(hw, I40E_PFINT_ICR0_ENA); reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; @@ -3666,16 +2184,18 @@ ixl_flush(hw); } +/* This only enables HW interrupts for the RX queues */ void ixl_enable_intr(struct ixl_vsi *vsi) { - struct ixl_pf *pf = (struct ixl_pf *)vsi->back; + // struct ixl_pf *pf = (struct ixl_pf *)vsi->back; struct i40e_hw *hw = vsi->hw; - struct ixl_queue *que = vsi->queues; + struct ixl_rx_queue *que = vsi->rx_queues; - if (pf->msix > 1) { - for (int i = 0; i < vsi->num_queues; i++, que++) - ixl_enable_queue(hw, que->me); + // TODO: Check iflib interrupt mode instead? + if (vsi->shared->isc_intr == IFLIB_INTR_MSIX) { + for (int i = 0; i < vsi->num_rx_queues; i++, que++) + ixl_enable_queue(hw, que->rxr.me); } else ixl_enable_intr0(hw); } @@ -3684,10 +2204,10 @@ ixl_disable_rings_intr(struct ixl_vsi *vsi) { struct i40e_hw *hw = vsi->hw; - struct ixl_queue *que = vsi->queues; + struct ixl_rx_queue *que = vsi->rx_queues; - for (int i = 0; i < vsi->num_queues; i++, que++) - ixl_disable_queue(hw, que->me); + for (int i = 0; i < vsi->num_rx_queues; i++, que++) + ixl_disable_queue(hw, que->rxr.me); } void @@ -3914,51 +2434,95 @@ struct i40e_hw *hw = &pf->hw; struct ixl_vsi *vsi = &pf->vsi; device_t dev = pf->dev; - bool is_up = false; + u8 set_fc_err_mask; int error = 0; - is_up = !!(vsi->ifp->if_drv_flags & IFF_DRV_RUNNING); + // TODO: Find a way to not do the full init_adminq if we can + // To avoid allocating DMA memory while we have a lock +#if 0 + error = i40e_init_adminq(hw); + if (error) { + device_printf(dev, "init: Admin queue init failure;" + " status code %d", error); + error = EIO; + goto err_out; + } +#else + i40e_resume_aq(hw); +#endif - /* Teardown */ - if (is_up) - ixl_stop(pf); - error = i40e_shutdown_lan_hmc(hw); - if (error) - device_printf(dev, - "Shutdown LAN HMC failed with code %d\n", error); - ixl_disable_intr0(hw); - ixl_teardown_adminq_msix(pf); - error = i40e_shutdown_adminq(hw); - if (error) - device_printf(dev, - "Shutdown Admin queue failed with code %d\n", error); + i40e_clear_pxe_mode(hw); - /* Setup */ - error = i40e_init_adminq(hw); - if (error != 0 && error != I40E_ERR_FIRMWARE_API_VERSION) { - device_printf(dev, "Unable to initialize Admin Queue, error %d\n", + error = ixl_get_hw_capabilities(pf); + if (error) { + device_printf(dev, "init: Error retrieving HW capabilities\n"); + goto err_out; + } + + /* Doesn't call any DMA functions */ + error = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, + hw->func_caps.num_rx_qp, 0, 0); + if (error) { + device_printf(dev, "init: LAN HMC init failed; status code %d\n", + error); + error = EIO; + goto err_out; + } + + error = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); + if (error) { + device_printf(dev, "init: LAN HMC config failed; status code %d\n", error); + error = EIO; + goto err_out; + } + + // XXX: possible fix for panic, but our failure recovery is still broken + error = ixl_switch_config(pf); + if (error) { + device_printf(dev, "init: ixl_switch_config() failed: %d\n", + error); + goto err_out; + } + + error = i40e_aq_set_phy_int_mask(hw, IXL_DEFAULT_PHY_INT_MASK, + NULL); + if (error) { + device_printf(dev, "init: i40e_aq_set_phy_mask() failed: err %d," + " aq_err %d\n", error, hw->aq.asq_last_status); + error = EIO; + goto err_out; } - error = ixl_setup_adminq_msix(pf); + + error = i40e_set_fc(hw, &set_fc_err_mask, true); if (error) { - device_printf(dev, "ixl_setup_adminq_msix error: %d\n", - error); + device_printf(dev, "init: setting link flow control failed; retcode %d," + " fc_err_mask 0x%02x\n", error, set_fc_err_mask); + goto err_out; } - ixl_configure_intr0_msix(pf); - ixl_enable_intr0(hw); - error = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, - hw->func_caps.num_rx_qp, 0, 0); - if (error) { - device_printf(dev, "init_lan_hmc failed: %d\n", error); + + // XXX: (Rebuild VSIs?) + + /* Firmware delay workaround */ + if (((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver < 33)) || + (hw->aq.fw_maj_ver < 4)) { + i40e_msec_delay(75); + error = i40e_aq_set_link_restart_an(hw, TRUE, NULL); + if (error) { + device_printf(dev, "init: link restart failed, aq_err %d\n", + hw->aq.asq_last_status); + goto err_out; + } } - error = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); - if (error) { - device_printf(dev, "configure_lan_hmc failed: %d\n", error); + + /* Keep admin queue interrupts active while driver is loaded */ + if (vsi->shared->isc_intr == IFLIB_INTR_MSIX) { + ixl_configure_intr0_msix(pf); + ixl_enable_intr0(hw); } - if (is_up) - ixl_init(pf); - return (0); +err_out: + return (error); } void @@ -3985,77 +2549,10 @@ ixl_rebuild_hw_structs_after_reset(pf); device_printf(dev, "Rebuilding driver state done.\n"); - atomic_clear_int(&pf->state, IXL_PF_STATE_EMPR_RESETTING); -} - -/* -** Tasklet handler for MSIX Adminq interrupts -** - do outside interrupt since it might sleep -*/ -void -ixl_do_adminq(void *context, int pending) -{ - struct ixl_pf *pf = context; - struct i40e_hw *hw = &pf->hw; - struct i40e_arq_event_info event; - i40e_status ret; - device_t dev = pf->dev; - u32 loop = 0; - u16 opcode, result; - - if (pf->state & IXL_PF_STATE_EMPR_RESETTING) { - /* Flag cleared at end of this function */ - ixl_handle_empr_reset(pf); - return; - } - - /* Admin Queue handling */ - event.buf_len = IXL_AQ_BUF_SZ; - event.msg_buf = malloc(event.buf_len, - M_DEVBUF, M_NOWAIT | M_ZERO); - if (!event.msg_buf) { - device_printf(dev, "%s: Unable to allocate memory for Admin" - " Queue event!\n", __func__); - return; - } - - IXL_PF_LOCK(pf); - /* clean and process any events */ - do { - ret = i40e_clean_arq_element(hw, &event, &result); - if (ret) - break; - opcode = LE16_TO_CPU(event.desc.opcode); - ixl_dbg(pf, IXL_DBG_AQ, - "Admin Queue event: %#06x\n", opcode); - switch (opcode) { - case i40e_aqc_opc_get_link_status: - ixl_link_event(pf, &event); - break; - case i40e_aqc_opc_send_msg_to_pf: -#ifdef PCI_IOV - ixl_handle_vf_msg(pf, &event); -#endif - break; - case i40e_aqc_opc_event_lan_overflow: - default: - break; - } - - } while (result && (loop++ < IXL_ADM_LIMIT)); - - free(event.msg_buf, M_DEVBUF); - - /* - * If there are still messages to process, reschedule ourselves. - * Otherwise, re-enable our interrupt. - */ - if (result > 0) - taskqueue_enqueue(pf->tq, &pf->adminq); - else - ixl_enable_intr0(hw); - - IXL_PF_UNLOCK(pf); + atomic_clear_int(&pf->state, IXL_PF_STATE_ADAPTER_RESETTING); + atomic_clear_int(&pf->state, IXL_PF_STATE_CORE_RESET_REQ); + atomic_clear_int(&pf->state, IXL_PF_STATE_GLOB_RESET_REQ); + atomic_clear_int(&pf->state, IXL_PF_STATE_EMP_RESET_REQ); } /** @@ -4137,8 +2634,6 @@ ixl_update_eth_stats(vsi); tx_discards = es->tx_discards + nsd->tx_dropped_link_down; - for (int i = 0; i < vsi->num_queues; i++) - tx_discards += vsi->queues[i].txr.br->br_drops; /* Update ifnet stats */ IXL_SET_IPACKETS(vsi, es->rx_unicast + @@ -4239,8 +2734,8 @@ void ixl_add_device_sysctls(struct ixl_pf *pf) { - device_t dev = pf->dev; struct i40e_hw *hw = &pf->hw; + device_t dev = pf->dev; struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); struct sysctl_oid_list *ctx_list = @@ -4275,11 +2770,6 @@ "Queues not allocated to a PF or VF"); SYSCTL_ADD_PROC(ctx, ctx_list, - OID_AUTO, "tx_itr", CTLTYPE_INT | CTLFLAG_RW, - pf, 0, ixl_sysctl_pf_tx_itr, "I", - "Immediately set TX ITR value for all queues"); - - SYSCTL_ADD_PROC(ctx, ctx_list, OID_AUTO, "rx_itr", CTLTYPE_INT | CTLFLAG_RW, pf, 0, ixl_sysctl_pf_rx_itr, "I", "Immediately set RX ITR value for all queues"); @@ -4288,10 +2778,6 @@ OID_AUTO, "dynamic_rx_itr", CTLFLAG_RW, &pf->dynamic_rx_itr, 0, "Enable dynamic RX ITR"); - SYSCTL_ADD_INT(ctx, ctx_list, - OID_AUTO, "dynamic_tx_itr", CTLFLAG_RW, - &pf->dynamic_tx_itr, 0, "Enable dynamic TX ITR"); - /* Add FEC sysctls for 25G adapters */ /* * XXX: These settings can be changed, but that isn't supported, @@ -4374,6 +2860,22 @@ OID_AUTO, "disable_fw_link_management", CTLTYPE_INT | CTLFLAG_WR, pf, 0, ixl_sysctl_fw_link_management, "I", "Disable FW Link Management"); + SYSCTL_ADD_PROC(ctx, debug_list, + OID_AUTO, "do_pf_reset", CTLTYPE_INT | CTLFLAG_WR, + pf, 0, ixl_sysctl_do_pf_reset, "I", "Tell HW to initiate a PF reset"); + + SYSCTL_ADD_PROC(ctx, debug_list, + OID_AUTO, "do_core_reset", CTLTYPE_INT | CTLFLAG_WR, + pf, 0, ixl_sysctl_do_core_reset, "I", "Tell HW to initiate a CORE reset"); + + SYSCTL_ADD_PROC(ctx, debug_list, + OID_AUTO, "do_global_reset", CTLTYPE_INT | CTLFLAG_WR, + pf, 0, ixl_sysctl_do_global_reset, "I", "Tell HW to initiate a GLOBAL reset"); + + SYSCTL_ADD_PROC(ctx, debug_list, + OID_AUTO, "do_emp_reset", CTLTYPE_INT | CTLFLAG_WR, + pf, 0, ixl_sysctl_do_emp_reset, "I", "Tell HW to initiate a EMP (entire firmware) reset"); + if (pf->has_i2c) { SYSCTL_ADD_PROC(ctx, debug_list, OID_AUTO, "read_i2c_byte", CTLTYPE_INT | CTLFLAG_RW, @@ -4401,9 +2903,8 @@ struct ixl_pf *pf = (struct ixl_pf *)arg1; int queues; - IXL_PF_LOCK(pf); + // TODO: Does this need its own lock? queues = (int)ixl_pf_qmgr_get_num_free(&pf->qmgr); - IXL_PF_UNLOCK(pf); return sysctl_handle_int(oidp, NULL, queues, req); } @@ -4447,11 +2948,6 @@ } pf->fc = requested_fc; - /* Get new link state */ - i40e_msec_delay(250); - hw->phy.get_link_info = TRUE; - i40e_get_link_status(hw, &pf->link_up); - return (0); } @@ -4505,7 +3001,7 @@ struct i40e_hw *hw = &pf->hw; int error = 0; - ixl_update_link_status(pf); + // ixl_update_link_status(pf); error = sysctl_handle_string(oidp, ixl_aq_speed_to_str(hw->phy.link_info.link_speed), @@ -4632,14 +3128,14 @@ return (error); pf->advertised_speed = requested_ls; - ixl_update_link_status(pf); + // ixl_update_link_status(pf); return (0); } /* * Input: bitmap of enum i40e_aq_link_speed */ -static u64 +u64 ixl_max_aq_speed_to_value(u8 link_speeds) { if (link_speeds & I40E_LINK_SPEED_40GB) @@ -4791,26 +3287,27 @@ if (pf->dbg_mask & IXL_DBG_NVMUPD) ixl_print_nvm_cmd(dev, nvma); - if (pf->state & IXL_PF_STATE_EMPR_RESETTING) { + if (pf->state & IXL_PF_STATE_ADAPTER_RESETTING) { int count = 0; while (count++ < 100) { i40e_msec_delay(100); - if (!(pf->state & IXL_PF_STATE_EMPR_RESETTING)) + if (!(pf->state & IXL_PF_STATE_ADAPTER_RESETTING)) break; } } - if (!(pf->state & IXL_PF_STATE_EMPR_RESETTING)) { - IXL_PF_LOCK(pf); + if (!(pf->state & IXL_PF_STATE_ADAPTER_RESETTING)) { + // TODO: Might need a different lock here + // IXL_PF_LOCK(pf); status = i40e_nvmupd_command(hw, nvma, nvma->data, &perrno); - IXL_PF_UNLOCK(pf); + // IXL_PF_UNLOCK(pf); } else { perrno = -EBUSY; } if (status) - device_printf(dev, "i40e_nvmupd_command status %s, perrno %d\n", - i40e_stat_str(hw, status), perrno); + device_printf(dev, "i40e_nvmupd_command status %d, perrno %d\n", + status, perrno); /* * -EPERM is actually ERESTART, which the kernel interprets as it needing @@ -4822,374 +3319,6 @@ return (perrno); } -/********************************************************************* - * - * Media Ioctl callback - * - * This routine is called whenever the user queries the status of - * the interface using ifconfig. - * - **********************************************************************/ -void -ixl_media_status(struct ifnet * ifp, struct ifmediareq * ifmr) -{ - struct ixl_vsi *vsi = ifp->if_softc; - struct ixl_pf *pf = vsi->back; - struct i40e_hw *hw = &pf->hw; - - INIT_DEBUGOUT("ixl_media_status: begin"); - IXL_PF_LOCK(pf); - - hw->phy.get_link_info = TRUE; - i40e_get_link_status(hw, &pf->link_up); - ixl_update_link_status(pf); - - ifmr->ifm_status = IFM_AVALID; - ifmr->ifm_active = IFM_ETHER; - - if (!pf->link_up) { - IXL_PF_UNLOCK(pf); - return; - } - - ifmr->ifm_status |= IFM_ACTIVE; - - /* Hardware always does full-duplex */ - ifmr->ifm_active |= IFM_FDX; - - switch (hw->phy.link_info.phy_type) { - /* 100 M */ - case I40E_PHY_TYPE_100BASE_TX: - ifmr->ifm_active |= IFM_100_TX; - break; - /* 1 G */ - case I40E_PHY_TYPE_1000BASE_T: - ifmr->ifm_active |= IFM_1000_T; - break; - case I40E_PHY_TYPE_1000BASE_SX: - ifmr->ifm_active |= IFM_1000_SX; - break; - case I40E_PHY_TYPE_1000BASE_LX: - ifmr->ifm_active |= IFM_1000_LX; - break; - case I40E_PHY_TYPE_1000BASE_T_OPTICAL: - ifmr->ifm_active |= IFM_OTHER; - break; - /* 10 G */ - case I40E_PHY_TYPE_10GBASE_SFPP_CU: - ifmr->ifm_active |= IFM_10G_TWINAX; - break; - case I40E_PHY_TYPE_10GBASE_SR: - ifmr->ifm_active |= IFM_10G_SR; - break; - case I40E_PHY_TYPE_10GBASE_LR: - ifmr->ifm_active |= IFM_10G_LR; - break; - case I40E_PHY_TYPE_10GBASE_T: - ifmr->ifm_active |= IFM_10G_T; - break; - case I40E_PHY_TYPE_XAUI: - case I40E_PHY_TYPE_XFI: - case I40E_PHY_TYPE_10GBASE_AOC: - ifmr->ifm_active |= IFM_OTHER; - break; - /* 25 G */ - case I40E_PHY_TYPE_25GBASE_KR: - ifmr->ifm_active |= IFM_25G_KR; - break; - case I40E_PHY_TYPE_25GBASE_CR: - ifmr->ifm_active |= IFM_25G_CR; - break; - case I40E_PHY_TYPE_25GBASE_SR: - ifmr->ifm_active |= IFM_25G_SR; - break; - case I40E_PHY_TYPE_25GBASE_LR: - ifmr->ifm_active |= IFM_UNKNOWN; - break; - /* 40 G */ - case I40E_PHY_TYPE_40GBASE_CR4: - case I40E_PHY_TYPE_40GBASE_CR4_CU: - ifmr->ifm_active |= IFM_40G_CR4; - break; - case I40E_PHY_TYPE_40GBASE_SR4: - ifmr->ifm_active |= IFM_40G_SR4; - break; - case I40E_PHY_TYPE_40GBASE_LR4: - ifmr->ifm_active |= IFM_40G_LR4; - break; - case I40E_PHY_TYPE_XLAUI: - ifmr->ifm_active |= IFM_OTHER; - break; - case I40E_PHY_TYPE_1000BASE_KX: - ifmr->ifm_active |= IFM_1000_KX; - break; - case I40E_PHY_TYPE_SGMII: - ifmr->ifm_active |= IFM_1000_SGMII; - break; - /* ERJ: What's the difference between these? */ - case I40E_PHY_TYPE_10GBASE_CR1_CU: - case I40E_PHY_TYPE_10GBASE_CR1: - ifmr->ifm_active |= IFM_10G_CR1; - break; - case I40E_PHY_TYPE_10GBASE_KX4: - ifmr->ifm_active |= IFM_10G_KX4; - break; - case I40E_PHY_TYPE_10GBASE_KR: - ifmr->ifm_active |= IFM_10G_KR; - break; - case I40E_PHY_TYPE_SFI: - ifmr->ifm_active |= IFM_10G_SFI; - break; - /* Our single 20G media type */ - case I40E_PHY_TYPE_20GBASE_KR2: - ifmr->ifm_active |= IFM_20G_KR2; - break; - case I40E_PHY_TYPE_40GBASE_KR4: - ifmr->ifm_active |= IFM_40G_KR4; - break; - case I40E_PHY_TYPE_XLPPI: - case I40E_PHY_TYPE_40GBASE_AOC: - ifmr->ifm_active |= IFM_40G_XLPPI; - break; - /* Unknown to driver */ - default: - ifmr->ifm_active |= IFM_UNKNOWN; - break; - } - /* Report flow control status as well */ - if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_TX) - ifmr->ifm_active |= IFM_ETH_TXPAUSE; - if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_RX) - ifmr->ifm_active |= IFM_ETH_RXPAUSE; - - IXL_PF_UNLOCK(pf); -} - -void -ixl_init(void *arg) -{ - struct ixl_pf *pf = arg; - - IXL_PF_LOCK(pf); - ixl_init_locked(pf); - IXL_PF_UNLOCK(pf); -} - -/* - * NOTE: Fortville does not support forcing media speeds. Instead, - * use the set_advertise sysctl to set the speeds Fortville - * will advertise or be allowed to operate at. - */ -int -ixl_media_change(struct ifnet * ifp) -{ - struct ixl_vsi *vsi = ifp->if_softc; - struct ifmedia *ifm = &vsi->media; - - INIT_DEBUGOUT("ixl_media_change: begin"); - - if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) - return (EINVAL); - - if_printf(ifp, "Use 'advertise_speed' sysctl to change advertised speeds\n"); - - return (ENODEV); -} - -/********************************************************************* - * Ioctl entry point - * - * ixl_ioctl is called when the user wants to configure the - * interface. - * - * return 0 on success, positive on failure - **********************************************************************/ - -int -ixl_ioctl(struct ifnet * ifp, u_long command, caddr_t data) -{ - struct ixl_vsi *vsi = ifp->if_softc; - struct ixl_pf *pf = vsi->back; - struct ifreq *ifr = (struct ifreq *)data; - struct ifdrv *ifd = (struct ifdrv *)data; -#if defined(INET) || defined(INET6) - struct ifaddr *ifa = (struct ifaddr *)data; - bool avoid_reset = FALSE; -#endif - int error = 0; - - switch (command) { - - case SIOCSIFADDR: - IOCTL_DEBUGOUT("ioctl: SIOCSIFADDR (Set Interface Address)"); -#ifdef INET - if (ifa->ifa_addr->sa_family == AF_INET) - avoid_reset = TRUE; -#endif -#ifdef INET6 - if (ifa->ifa_addr->sa_family == AF_INET6) - avoid_reset = TRUE; -#endif -#if defined(INET) || defined(INET6) - /* - ** Calling init results in link renegotiation, - ** so we avoid doing it when possible. - */ - if (avoid_reset) { - ifp->if_flags |= IFF_UP; - if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) - ixl_init(pf); -#ifdef INET - if (!(ifp->if_flags & IFF_NOARP)) - arp_ifinit(ifp, ifa); -#endif - } else - error = ether_ioctl(ifp, command, data); - break; -#endif - case SIOCSIFMTU: - IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); - if (ifr->ifr_mtu > IXL_MAX_FRAME - - ETHER_HDR_LEN - ETHER_CRC_LEN - ETHER_VLAN_ENCAP_LEN) { - error = EINVAL; - } else { - IXL_PF_LOCK(pf); - ifp->if_mtu = ifr->ifr_mtu; - vsi->max_frame_size = - ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN - + ETHER_VLAN_ENCAP_LEN; - if (ifp->if_drv_flags & IFF_DRV_RUNNING) - ixl_init_locked(pf); - IXL_PF_UNLOCK(pf); - } - break; - case SIOCSIFFLAGS: - IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); - IXL_PF_LOCK(pf); - if (ifp->if_flags & IFF_UP) { - if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) { - if ((ifp->if_flags ^ pf->if_flags) & - (IFF_PROMISC | IFF_ALLMULTI)) { - ixl_set_promisc(vsi); - } - } else { - IXL_PF_UNLOCK(pf); - ixl_init(pf); - IXL_PF_LOCK(pf); - } - } else { - if (ifp->if_drv_flags & IFF_DRV_RUNNING) { - ixl_stop_locked(pf); - } - } - pf->if_flags = ifp->if_flags; - IXL_PF_UNLOCK(pf); - break; - case SIOCSDRVSPEC: - case SIOCGDRVSPEC: - IOCTL_DEBUGOUT("ioctl: SIOCxDRVSPEC (Get/Set Driver-specific " - "Info)\n"); - - /* NVM update command */ - if (ifd->ifd_cmd == I40E_NVM_ACCESS) - error = ixl_handle_nvmupd_cmd(pf, ifd); - else - error = EINVAL; - break; - case SIOCADDMULTI: - IOCTL_DEBUGOUT("ioctl: SIOCADDMULTI"); - if (ifp->if_drv_flags & IFF_DRV_RUNNING) { - IXL_PF_LOCK(pf); - ixl_disable_rings_intr(vsi); - ixl_add_multi(vsi); - ixl_enable_intr(vsi); - IXL_PF_UNLOCK(pf); - } - break; - case SIOCDELMULTI: - IOCTL_DEBUGOUT("ioctl: SIOCDELMULTI"); - if (ifp->if_drv_flags & IFF_DRV_RUNNING) { - IXL_PF_LOCK(pf); - ixl_disable_rings_intr(vsi); - ixl_del_multi(vsi); - ixl_enable_intr(vsi); - IXL_PF_UNLOCK(pf); - } - break; - case SIOCSIFMEDIA: - case SIOCGIFMEDIA: - case SIOCGIFXMEDIA: - IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); - error = ifmedia_ioctl(ifp, ifr, &vsi->media, command); - break; - case SIOCSIFCAP: - { - int mask = ifr->ifr_reqcap ^ ifp->if_capenable; - IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); - - ixl_cap_txcsum_tso(vsi, ifp, mask); - - if (mask & IFCAP_RXCSUM) - ifp->if_capenable ^= IFCAP_RXCSUM; - if (mask & IFCAP_RXCSUM_IPV6) - ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; - if (mask & IFCAP_LRO) - ifp->if_capenable ^= IFCAP_LRO; - if (mask & IFCAP_VLAN_HWTAGGING) - ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; - if (mask & IFCAP_VLAN_HWFILTER) - ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; - if (mask & IFCAP_VLAN_HWTSO) - ifp->if_capenable ^= IFCAP_VLAN_HWTSO; - if (ifp->if_drv_flags & IFF_DRV_RUNNING) { - IXL_PF_LOCK(pf); - ixl_init_locked(pf); - IXL_PF_UNLOCK(pf); - } - VLAN_CAPABILITIES(ifp); - - break; - } -#if __FreeBSD_version >= 1003000 - case SIOCGI2C: - { - struct ifi2creq i2c; - int i; - - IOCTL_DEBUGOUT("ioctl: SIOCGI2C (Get I2C Data)"); - if (!pf->has_i2c) - return (ENOTTY); - - error = copyin(ifr->ifr_data, &i2c, sizeof(i2c)); - if (error != 0) - break; - if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) { - error = EINVAL; - break; - } - if (i2c.len > sizeof(i2c.data)) { - error = EINVAL; - break; - } - - for (i = 0; i < i2c.len; i++) - if (ixl_read_i2c_byte(pf, i2c.offset + i, - i2c.dev_addr, &i2c.data[i])) - return (EIO); - - error = copyout(&i2c, ifr->ifr_data, sizeof(i2c)); - break; - } -#endif - default: - IOCTL_DEBUGOUT("ioctl: UNKNOWN (0x%X)\n", (int)command); - error = ether_ioctl(ifp, command, data); - break; - } - - return (error); -} - int ixl_find_i2c_interface(struct ixl_pf *pf) { @@ -5260,6 +3389,7 @@ return (ext) ? ext_phy_types_str[bit_pos] : phy_types_str[bit_pos]; } +/* TODO: ERJ: I don't this is necessary anymore. */ int ixl_aq_get_link_status(struct ixl_pf *pf, struct i40e_aqc_get_link_status *link_status) { @@ -5490,7 +3620,7 @@ char * ixl_switch_res_type_string(u8 type) { - static char * ixl_switch_res_type_strings[0x14] = { + char * ixl_switch_res_type_strings[0x14] = { "VEB", "VSI", "Perfect Match MAC address", @@ -5585,9 +3715,6 @@ /* ** Caller must init and delete sbuf; this function will clear and ** finish it for caller. -** -** XXX: Cannot use the SEID for this, since there is no longer a -** fixed mapping between SEID and element type. */ char * ixl_switch_element_string(struct sbuf *s, @@ -6074,3 +4201,71 @@ return ixl_set_fec_config(pf, &abilities, I40E_AQ_SET_FEC_AUTO, !!(mode)); } +static int +ixl_sysctl_do_pf_reset(SYSCTL_HANDLER_ARGS) +{ + struct ixl_pf *pf = (struct ixl_pf *)arg1; + int requested = 0, error = 0; + + /* Read in new mode */ + error = sysctl_handle_int(oidp, &requested, 0, req); + if ((error) || (req->newptr == NULL)) + return (error); + + /* Initiate the PF reset later in the admin task */ + atomic_set_32(&pf->state, IXL_PF_STATE_PF_RESET_REQ); + + return (error); +} + +static int +ixl_sysctl_do_core_reset(SYSCTL_HANDLER_ARGS) +{ + struct ixl_pf *pf = (struct ixl_pf *)arg1; + int requested = 0, error = 0; + + /* Read in new mode */ + error = sysctl_handle_int(oidp, &requested, 0, req); + if ((error) || (req->newptr == NULL)) + return (error); + + /* Initiate the CORE reset later in the admin task */ + atomic_set_32(&pf->state, IXL_PF_STATE_CORE_RESET_REQ); + + return (error); +} + +static int +ixl_sysctl_do_global_reset(SYSCTL_HANDLER_ARGS) +{ + struct ixl_pf *pf = (struct ixl_pf *)arg1; + int requested = 0, error = 0; + + /* Read in new mode */ + error = sysctl_handle_int(oidp, &requested, 0, req); + if ((error) || (req->newptr == NULL)) + return (error); + + /* Initiate the CORE reset later in the admin task */ + atomic_set_32(&pf->state, IXL_PF_STATE_GLOB_RESET_REQ); + + return (error); +} + +static int +ixl_sysctl_do_emp_reset(SYSCTL_HANDLER_ARGS) +{ + struct ixl_pf *pf = (struct ixl_pf *)arg1; + int requested = 0, error = 0; + + /* Read in new mode */ + error = sysctl_handle_int(oidp, &requested, 0, req); + if ((error) || (req->newptr == NULL)) + return (error); + + /* Initiate the EMP reset later in the admin task */ + atomic_set_32(&pf->state, IXL_PF_STATE_EMP_RESET_REQ); + + return (error); +} + Index: sys/dev/ixl/ixl_txrx.c =================================================================== --- sys/dev/ixl/ixl_txrx.c +++ sys/dev/ixl/ixl_txrx.c @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2013-2015, Intel Corporation + Copyright (c) 2013-2017, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -51,26 +51,32 @@ #endif /* Local Prototypes */ -static void ixl_rx_checksum(struct mbuf *, u32, u32, u8); -static void ixl_refresh_mbufs(struct ixl_queue *, int); -static int ixl_xmit(struct ixl_queue *, struct mbuf **); -static int ixl_tx_setup_offload(struct ixl_queue *, - struct mbuf *, u32 *, u32 *); -static bool ixl_tso_setup(struct ixl_queue *, struct mbuf *); - -static inline void ixl_rx_discard(struct rx_ring *, int); -static inline void ixl_rx_input(struct rx_ring *, struct ifnet *, - struct mbuf *, u8); - -static inline bool ixl_tso_detect_sparse(struct mbuf *mp); -static int ixl_tx_setup_offload(struct ixl_queue *que, - struct mbuf *mp, u32 *cmd, u32 *off); -static inline u32 ixl_get_tx_head(struct ixl_queue *que); - -#ifdef DEV_NETMAP -#include -int ixl_rx_miss, ixl_rx_miss_bufs, ixl_crcstrip = 1; -#endif /* DEV_NETMAP */ +static void ixl_rx_checksum(if_rxd_info_t ri, u32 status, u32 error, u8 ptype); + +static int ixl_isc_txd_encap(void *arg, if_pkt_info_t pi); +static void ixl_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx); +static int ixl_isc_txd_credits_update_hwb(void *arg, uint16_t txqid, bool clear); +static int ixl_isc_txd_credits_update_dd(void *arg, uint16_t txqid, bool clear); + +static void ixl_isc_rxd_refill(void *arg, if_rxd_update_t iru); +static void ixl_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, + qidx_t pidx); +static int ixl_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, + qidx_t budget); +static int ixl_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri); + +extern int ixl_intr(void *arg); + +struct if_txrx ixl_txrx = { + ixl_isc_txd_encap, + ixl_isc_txd_flush, + ixl_isc_txd_credits_update_dd, + ixl_isc_rxd_available, + ixl_isc_rxd_pkt_get, + ixl_isc_rxd_refill, + ixl_isc_rxd_flush, + ixl_intr +}; /* * @key key is saved into this parameter @@ -89,535 +95,39 @@ bcopy(rss_seed, key, IXL_RSS_KEY_SIZE); } -/* -** Multiqueue Transmit driver -*/ -int -ixl_mq_start(struct ifnet *ifp, struct mbuf *m) -{ - struct ixl_vsi *vsi = ifp->if_softc; - struct ixl_queue *que; - struct tx_ring *txr; - int err, i; -#ifdef RSS - u32 bucket_id; -#endif - - /* - ** Which queue to use: - ** - ** When doing RSS, map it to the same outbound - ** queue as the incoming flow would be mapped to. - ** If everything is setup correctly, it should be - ** the same bucket that the current CPU we're on is. - */ - if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) { -#ifdef RSS - if (rss_hash2bucket(m->m_pkthdr.flowid, - M_HASHTYPE_GET(m), &bucket_id) == 0) { - i = bucket_id % vsi->num_queues; - } else -#endif - i = m->m_pkthdr.flowid % vsi->num_queues; - } else - i = curcpu % vsi->num_queues; - - que = &vsi->queues[i]; - txr = &que->txr; - - err = drbr_enqueue(ifp, txr->br, m); - if (err) - return (err); - if (IXL_TX_TRYLOCK(txr)) { - ixl_mq_start_locked(ifp, txr); - IXL_TX_UNLOCK(txr); - } else - taskqueue_enqueue(que->tq, &que->tx_task); - - return (0); -} - -int -ixl_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr) -{ - struct ixl_queue *que = txr->que; - struct ixl_vsi *vsi = que->vsi; - struct mbuf *next; - int err = 0; - - - if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) || - vsi->link_active == 0) - return (ENETDOWN); - - /* Process the transmit queue */ - while ((next = drbr_peek(ifp, txr->br)) != NULL) { - if ((err = ixl_xmit(que, &next)) != 0) { - if (next == NULL) - drbr_advance(ifp, txr->br); - else - drbr_putback(ifp, txr->br, next); - break; - } - drbr_advance(ifp, txr->br); - /* Send a copy of the frame to the BPF listener */ - ETHER_BPF_MTAP(ifp, next); - if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) - break; - } - - if (txr->avail < IXL_TX_CLEANUP_THRESHOLD) - ixl_txeof(que); - - return (err); -} - -/* - * Called from a taskqueue to drain queued transmit packets. - */ -void -ixl_deferred_mq_start(void *arg, int pending) -{ - struct ixl_queue *que = arg; - struct tx_ring *txr = &que->txr; - struct ixl_vsi *vsi = que->vsi; - struct ifnet *ifp = vsi->ifp; - - IXL_TX_LOCK(txr); - if (!drbr_empty(ifp, txr->br)) - ixl_mq_start_locked(ifp, txr); - IXL_TX_UNLOCK(txr); -} - -/* -** Flush all queue ring buffers -*/ -void -ixl_qflush(struct ifnet *ifp) +static bool +ixl_is_tx_desc_done(struct tx_ring *txr, int idx) { - struct ixl_vsi *vsi = ifp->if_softc; - - for (int i = 0; i < vsi->num_queues; i++) { - struct ixl_queue *que = &vsi->queues[i]; - struct tx_ring *txr = &que->txr; - struct mbuf *m; - IXL_TX_LOCK(txr); - while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) - m_freem(m); - IXL_TX_UNLOCK(txr); - } - if_qflush(ifp); + return (((txr->tx_base[idx].cmd_type_offset_bsz >> I40E_TXD_QW1_DTYPE_SHIFT) + & I40E_TXD_QW1_DTYPE_MASK) == I40E_TX_DESC_DTYPE_DESC_DONE); } +// TODO: Compare this version of iflib with current version in OOT driver /* ** Find mbuf chains passed to the driver ** that are 'sparse', using more than 8 -** mbufs to deliver an mss-size chunk of data +** segments to deliver an mss-size chunk of data */ -static inline bool -ixl_tso_detect_sparse(struct mbuf *mp) -{ - struct mbuf *m; - int num, mss; - - num = 0; - mss = mp->m_pkthdr.tso_segsz; - - /* Exclude first mbuf; assume it contains all headers */ - for (m = mp->m_next; m != NULL; m = m->m_next) { - if (m == NULL) - break; - num++; - mss -= m->m_len % mp->m_pkthdr.tso_segsz; - - if (mss < 1) { - if (num > IXL_SPARSE_CHAIN) - return (true); - num = (mss == 0) ? 0 : 1; - mss += mp->m_pkthdr.tso_segsz; - } - } - - return (false); -} - - -/********************************************************************* - * - * This routine maps the mbufs to tx descriptors, allowing the - * TX engine to transmit the packets. - * - return 0 on success, positive on failure - * - **********************************************************************/ -#define IXL_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS) - static int -ixl_xmit(struct ixl_queue *que, struct mbuf **m_headp) +ixl_tso_detect_sparse(bus_dma_segment_t *segs, int nsegs, int segsz) { - struct ixl_vsi *vsi = que->vsi; - struct i40e_hw *hw = vsi->hw; - struct tx_ring *txr = &que->txr; - struct ixl_tx_buf *buf; - struct i40e_tx_desc *txd = NULL; - struct mbuf *m_head, *m; - int i, j, error, nsegs; - int first, last = 0; - u16 vtag = 0; - u32 cmd, off; - bus_dmamap_t map; - bus_dma_tag_t tag; - bus_dma_segment_t segs[IXL_MAX_TSO_SEGS]; - - cmd = off = 0; - m_head = *m_headp; - - /* - * Important to capture the first descriptor - * used because it will contain the index of - * the one we tell the hardware to report back - */ - first = txr->next_avail; - buf = &txr->buffers[first]; - map = buf->map; - tag = txr->tx_tag; - - if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { - /* Use larger mapping for TSO */ - tag = txr->tso_tag; - if (ixl_tso_detect_sparse(m_head)) { - m = m_defrag(m_head, M_NOWAIT); - if (m == NULL) { - m_freem(*m_headp); - *m_headp = NULL; - return (ENOBUFS); - } - *m_headp = m; - } - } - - /* - * Map the packet for DMA. - */ - error = bus_dmamap_load_mbuf_sg(tag, map, - *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); - - if (error == EFBIG) { - struct mbuf *m; - - m = m_defrag(*m_headp, M_NOWAIT); - if (m == NULL) { - que->mbuf_defrag_failed++; - m_freem(*m_headp); - *m_headp = NULL; - return (ENOBUFS); - } - *m_headp = m; - - /* Try it again */ - error = bus_dmamap_load_mbuf_sg(tag, map, - *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); - - if (error != 0) { - que->tx_dmamap_failed++; - m_freem(*m_headp); - *m_headp = NULL; - return (error); - } - } else if (error != 0) { - que->tx_dmamap_failed++; - m_freem(*m_headp); - *m_headp = NULL; - return (error); - } - - /* Make certain there are enough descriptors */ - if (nsegs > txr->avail - 2) { - txr->no_desc++; - error = ENOBUFS; - goto xmit_fail; - } - m_head = *m_headp; - - /* Set up the TSO/CSUM offload */ - if (m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD) { - error = ixl_tx_setup_offload(que, m_head, &cmd, &off); - if (error) - goto xmit_fail; - } - - cmd |= I40E_TX_DESC_CMD_ICRC; - /* Grab the VLAN tag */ - if (m_head->m_flags & M_VLANTAG) { - cmd |= I40E_TX_DESC_CMD_IL2TAG1; - vtag = htole16(m_head->m_pkthdr.ether_vtag); - } - - i = txr->next_avail; - for (j = 0; j < nsegs; j++) { - bus_size_t seglen; + int i, count, curseg; - buf = &txr->buffers[i]; - buf->tag = tag; /* Keep track of the type tag */ - txd = &txr->base[i]; - seglen = segs[j].ds_len; - - txd->buffer_addr = htole64(segs[j].ds_addr); - txd->cmd_type_offset_bsz = - htole64(I40E_TX_DESC_DTYPE_DATA - | ((u64)cmd << I40E_TXD_QW1_CMD_SHIFT) - | ((u64)off << I40E_TXD_QW1_OFFSET_SHIFT) - | ((u64)seglen << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) - | ((u64)vtag << I40E_TXD_QW1_L2TAG1_SHIFT)); - - last = i; /* descriptor that will get completion IRQ */ - - if (++i == que->num_desc) - i = 0; - - buf->m_head = NULL; - buf->eop_index = -1; + if (nsegs <= IXL_MAX_TX_SEGS-2) + return (0); + for (curseg = count = i = 0; i < nsegs; i++) { + curseg += segs[i].ds_len; + count++; + if (__predict_false(count == IXL_MAX_TX_SEGS-2)) + return (1); + if (curseg > segsz) { + curseg -= segsz; + count = 1; + } + if (curseg == segsz) + curseg = count = 0; } - /* Set the last descriptor for report */ - txd->cmd_type_offset_bsz |= - htole64(((u64)IXL_TXD_CMD << I40E_TXD_QW1_CMD_SHIFT)); - txr->avail -= nsegs; - txr->next_avail = i; - - buf->m_head = m_head; - /* Swap the dma map between the first and last descriptor */ - txr->buffers[first].map = buf->map; - buf->map = map; - bus_dmamap_sync(tag, map, BUS_DMASYNC_PREWRITE); - - /* Set the index of the descriptor that will be marked done */ - buf = &txr->buffers[first]; - buf->eop_index = last; - - bus_dmamap_sync(txr->dma.tag, txr->dma.map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - /* - * Advance the Transmit Descriptor Tail (Tdt), this tells the - * hardware that this frame is available to transmit. - */ - ++txr->total_packets; - wr32(hw, txr->tail, i); - - /* Mark outstanding work */ - atomic_store_rel_32(&txr->watchdog_timer, IXL_WATCHDOG); return (0); - -xmit_fail: - bus_dmamap_unload(tag, buf->map); - return (error); -} - - -/********************************************************************* - * - * Allocate memory for tx_buffer structures. The tx_buffer stores all - * the information needed to transmit a packet on the wire. This is - * called only once at attach, setup is done every reset. - * - **********************************************************************/ -int -ixl_allocate_tx_data(struct ixl_queue *que) -{ - struct tx_ring *txr = &que->txr; - struct ixl_vsi *vsi = que->vsi; - device_t dev = vsi->dev; - struct ixl_tx_buf *buf; - int error = 0; - - /* - * Setup DMA descriptor areas. - */ - if ((error = bus_dma_tag_create(NULL, /* parent */ - 1, 0, /* alignment, bounds */ - BUS_SPACE_MAXADDR, /* lowaddr */ - BUS_SPACE_MAXADDR, /* highaddr */ - NULL, NULL, /* filter, filterarg */ - IXL_TSO_SIZE, /* maxsize */ - IXL_MAX_TX_SEGS, /* nsegments */ - PAGE_SIZE, /* maxsegsize */ - 0, /* flags */ - NULL, /* lockfunc */ - NULL, /* lockfuncarg */ - &txr->tx_tag))) { - device_printf(dev,"Unable to allocate TX DMA tag\n"); - goto fail; - } - - /* Make a special tag for TSO */ - if ((error = bus_dma_tag_create(NULL, /* parent */ - 1, 0, /* alignment, bounds */ - BUS_SPACE_MAXADDR, /* lowaddr */ - BUS_SPACE_MAXADDR, /* highaddr */ - NULL, NULL, /* filter, filterarg */ - IXL_TSO_SIZE, /* maxsize */ - IXL_MAX_TSO_SEGS, /* nsegments */ - PAGE_SIZE, /* maxsegsize */ - 0, /* flags */ - NULL, /* lockfunc */ - NULL, /* lockfuncarg */ - &txr->tso_tag))) { - device_printf(dev,"Unable to allocate TX TSO DMA tag\n"); - goto fail; - } - - if (!(txr->buffers = - (struct ixl_tx_buf *) malloc(sizeof(struct ixl_tx_buf) * - que->num_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { - device_printf(dev, "Unable to allocate tx_buffer memory\n"); - error = ENOMEM; - goto fail; - } - - /* Create the descriptor buffer default dma maps */ - buf = txr->buffers; - for (int i = 0; i < que->num_desc; i++, buf++) { - buf->tag = txr->tx_tag; - error = bus_dmamap_create(buf->tag, 0, &buf->map); - if (error != 0) { - device_printf(dev, "Unable to create TX DMA map\n"); - goto fail; - } - } -fail: - return (error); -} - - -/********************************************************************* - * - * (Re)Initialize a queue transmit ring. - * - called by init, it clears the descriptor ring, - * and frees any stale mbufs - * - **********************************************************************/ -void -ixl_init_tx_ring(struct ixl_queue *que) -{ -#ifdef DEV_NETMAP - struct netmap_adapter *na = NA(que->vsi->ifp); - struct netmap_slot *slot; -#endif /* DEV_NETMAP */ - struct tx_ring *txr = &que->txr; - struct ixl_tx_buf *buf; - - /* Clear the old ring contents */ - IXL_TX_LOCK(txr); - -#ifdef DEV_NETMAP - /* - * (under lock): if in netmap mode, do some consistency - * checks and set slot to entry 0 of the netmap ring. - */ - slot = netmap_reset(na, NR_TX, que->me, 0); -#endif /* DEV_NETMAP */ - - bzero((void *)txr->base, - (sizeof(struct i40e_tx_desc)) * que->num_desc); - - /* Reset indices */ - txr->next_avail = 0; - txr->next_to_clean = 0; - - /* Reset watchdog status */ - txr->watchdog_timer = 0; - -#ifdef IXL_FDIR - /* Initialize flow director */ - txr->atr_rate = ixl_atr_rate; - txr->atr_count = 0; -#endif - /* Free any existing tx mbufs. */ - buf = txr->buffers; - for (int i = 0; i < que->num_desc; i++, buf++) { - if (buf->m_head != NULL) { - bus_dmamap_sync(buf->tag, buf->map, - BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(buf->tag, buf->map); - m_freem(buf->m_head); - buf->m_head = NULL; - } -#ifdef DEV_NETMAP - /* - * In netmap mode, set the map for the packet buffer. - * NOTE: Some drivers (not this one) also need to set - * the physical buffer address in the NIC ring. - * netmap_idx_n2k() maps a nic index, i, into the corresponding - * netmap slot index, si - */ - if (slot) { - int si = netmap_idx_n2k(&na->tx_rings[que->me], i); - netmap_load_map(na, buf->tag, buf->map, NMB(na, slot + si)); - } -#endif /* DEV_NETMAP */ - /* Clear the EOP index */ - buf->eop_index = -1; - } - - /* Set number of descriptors available */ - txr->avail = que->num_desc; - - bus_dmamap_sync(txr->dma.tag, txr->dma.map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - IXL_TX_UNLOCK(txr); -} - - -/********************************************************************* - * - * Free transmit ring related data structures. - * - **********************************************************************/ -void -ixl_free_que_tx(struct ixl_queue *que) -{ - struct tx_ring *txr = &que->txr; - struct ixl_tx_buf *buf; - - INIT_DBG_IF(que->vsi->ifp, "queue %d: begin", que->me); - - for (int i = 0; i < que->num_desc; i++) { - buf = &txr->buffers[i]; - if (buf->m_head != NULL) { - bus_dmamap_sync(buf->tag, buf->map, - BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(buf->tag, - buf->map); - m_freem(buf->m_head); - buf->m_head = NULL; - if (buf->map != NULL) { - bus_dmamap_destroy(buf->tag, - buf->map); - buf->map = NULL; - } - } else if (buf->map != NULL) { - bus_dmamap_unload(buf->tag, - buf->map); - bus_dmamap_destroy(buf->tag, - buf->map); - buf->map = NULL; - } - } - if (txr->br != NULL) - buf_ring_free(txr->br, M_DEVBUF); - if (txr->buffers != NULL) { - free(txr->buffers, M_DEVBUF); - txr->buffers = NULL; - } - if (txr->tx_tag != NULL) { - bus_dma_tag_destroy(txr->tx_tag); - txr->tx_tag = NULL; - } - if (txr->tso_tag != NULL) { - bus_dma_tag_destroy(txr->tso_tag); - txr->tso_tag = NULL; - } - - INIT_DBG_IF(que->vsi->ifp, "queue %d: end", que->me); - return; } /********************************************************************* @@ -626,66 +136,18 @@ * **********************************************************************/ -static int -ixl_tx_setup_offload(struct ixl_queue *que, - struct mbuf *mp, u32 *cmd, u32 *off) +static void +ixl_tx_setup_offload(struct ixl_tx_queue *que, + if_pkt_info_t pi, u32 *cmd, u32 *off) { - struct ether_vlan_header *eh; -#ifdef INET - struct ip *ip = NULL; -#endif - struct tcphdr *th = NULL; -#ifdef INET6 - struct ip6_hdr *ip6; -#endif - int elen, ip_hlen = 0, tcp_hlen; - u16 etype; - u8 ipproto = 0; - bool tso = FALSE; - - /* Set up the TSO context descriptor if required */ - if (mp->m_pkthdr.csum_flags & CSUM_TSO) { - tso = ixl_tso_setup(que, mp); - if (tso) - ++que->tso; - else - return (ENXIO); - } - - /* - * Determine where frame payload starts. - * Jump over vlan headers if already present, - * helpful for QinQ too. - */ - eh = mtod(mp, struct ether_vlan_header *); - if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { - etype = ntohs(eh->evl_proto); - elen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; - } else { - etype = ntohs(eh->evl_encap_proto); - elen = ETHER_HDR_LEN; - } - - switch (etype) { + switch (pi->ipi_etype) { #ifdef INET case ETHERTYPE_IP: - ip = (struct ip *)(mp->m_data + elen); - ip_hlen = ip->ip_hl << 2; - ipproto = ip->ip_p; - th = (struct tcphdr *)((caddr_t)ip + ip_hlen); - /* The IP checksum must be recalculated with TSO */ - if (tso) - *cmd |= I40E_TX_DESC_CMD_IIPT_IPV4_CSUM; - else - *cmd |= I40E_TX_DESC_CMD_IIPT_IPV4; + *cmd |= I40E_TX_DESC_CMD_IIPT_IPV4_CSUM; break; #endif #ifdef INET6 case ETHERTYPE_IPV6: - ip6 = (struct ip6_hdr *)(mp->m_data + elen); - ip_hlen = sizeof(struct ip6_hdr); - ipproto = ip6->ip6_nxt; - th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen); *cmd |= I40E_TX_DESC_CMD_IIPT_IPV6; break; #endif @@ -693,31 +155,26 @@ break; } - *off |= (elen >> 1) << I40E_TX_DESC_LENGTH_MACLEN_SHIFT; - *off |= (ip_hlen >> 2) << I40E_TX_DESC_LENGTH_IPLEN_SHIFT; + *off |= (pi->ipi_ehdrlen >> 1) << I40E_TX_DESC_LENGTH_MACLEN_SHIFT; + *off |= (pi->ipi_ip_hlen >> 2) << I40E_TX_DESC_LENGTH_IPLEN_SHIFT; - switch (ipproto) { + switch (pi->ipi_ipproto) { case IPPROTO_TCP: - tcp_hlen = th->th_off << 2; - if (mp->m_pkthdr.csum_flags & (CSUM_TCP|CSUM_TCP_IPV6)) { + if (pi->ipi_csum_flags & (CSUM_TCP|CSUM_TCP_IPV6)) { *cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP; - *off |= (tcp_hlen >> 2) << + *off |= (pi->ipi_tcp_hlen >> 2) << I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; } -#ifdef IXL_FDIR - ixl_atr(que, th, etype); -#endif break; case IPPROTO_UDP: - if (mp->m_pkthdr.csum_flags & (CSUM_UDP|CSUM_UDP_IPV6)) { + if (pi->ipi_csum_flags & (CSUM_UDP|CSUM_UDP_IPV6)) { *cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP; *off |= (sizeof(struct udphdr) >> 2) << I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; } break; - case IPPROTO_SCTP: - if (mp->m_pkthdr.csum_flags & (CSUM_SCTP|CSUM_SCTP_IPV6)) { + if (pi->ipi_csum_flags & (CSUM_SCTP|CSUM_SCTP_IPV6)) { *cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP; *off |= (sizeof(struct sctphdr) >> 2) << I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; @@ -727,103 +184,32 @@ break; } - return (0); } - /********************************************************************** * * Setup context for hardware segmentation offload (TSO) * **********************************************************************/ -static bool -ixl_tso_setup(struct ixl_queue *que, struct mbuf *mp) +static int +ixl_tso_setup(struct tx_ring *txr, if_pkt_info_t pi) { - struct tx_ring *txr = &que->txr; + if_softc_ctx_t scctx; struct i40e_tx_context_desc *TXD; - struct ixl_tx_buf *buf; u32 cmd, mss, type, tsolen; - u16 etype; - int idx, elen, ip_hlen, tcp_hlen; - struct ether_vlan_header *eh; -#ifdef INET - struct ip *ip; -#endif -#ifdef INET6 - struct ip6_hdr *ip6; -#endif -#if defined(INET6) || defined(INET) - struct tcphdr *th; -#endif + int idx; u64 type_cmd_tso_mss; - /* - * Determine where frame payload starts. - * Jump over vlan headers if already present - */ - eh = mtod(mp, struct ether_vlan_header *); - if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { - elen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; - etype = eh->evl_proto; - } else { - elen = ETHER_HDR_LEN; - etype = eh->evl_encap_proto; - } + // printf("%s: begin\n", __func__); - switch (ntohs(etype)) { -#ifdef INET6 - case ETHERTYPE_IPV6: - ip6 = (struct ip6_hdr *)(mp->m_data + elen); - if (ip6->ip6_nxt != IPPROTO_TCP) - return (ENXIO); - ip_hlen = sizeof(struct ip6_hdr); - th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen); - th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0); - tcp_hlen = th->th_off << 2; - /* - * The corresponding flag is set by the stack in the IPv4 - * TSO case, but not in IPv6 (at least in FreeBSD 10.2). - * So, set it here because the rest of the flow requires it. - */ - mp->m_pkthdr.csum_flags |= CSUM_TCP_IPV6; - break; -#endif -#ifdef INET - case ETHERTYPE_IP: - ip = (struct ip *)(mp->m_data + elen); - if (ip->ip_p != IPPROTO_TCP) - return (ENXIO); - ip->ip_sum = 0; - ip_hlen = ip->ip_hl << 2; - th = (struct tcphdr *)((caddr_t)ip + ip_hlen); - th->th_sum = in_pseudo(ip->ip_src.s_addr, - ip->ip_dst.s_addr, htons(IPPROTO_TCP)); - tcp_hlen = th->th_off << 2; - break; -#endif - default: - printf("%s: CSUM_TSO but no supported IP version (0x%04x)", - __func__, ntohs(etype)); - return FALSE; - } - - /* Ensure we have at least the IP+TCP header in the first mbuf. */ - if (mp->m_len < elen + ip_hlen + sizeof(struct tcphdr)) - return FALSE; - - idx = txr->next_avail; - buf = &txr->buffers[idx]; - TXD = (struct i40e_tx_context_desc *) &txr->base[idx]; - tsolen = mp->m_pkthdr.len - (elen + ip_hlen + tcp_hlen); + idx = pi->ipi_pidx; + TXD = (struct i40e_tx_context_desc *) &txr->tx_base[idx]; + tsolen = pi->ipi_len - (pi->ipi_ehdrlen + pi->ipi_ip_hlen + pi->ipi_tcp_hlen); + scctx = txr->que->vsi->shared; type = I40E_TX_DESC_DTYPE_CONTEXT; cmd = I40E_TX_CTX_DESC_TSO; - /* TSO MSS must not be less than 64 */ - if (mp->m_pkthdr.tso_segsz < IXL_MIN_TSO_MSS) { - que->mss_too_small++; - mp->m_pkthdr.tso_segsz = IXL_MIN_TSO_MSS; - } - mss = mp->m_pkthdr.tso_segsz; + mss = pi->ipi_tso_segsz; type_cmd_tso_mss = ((u64)type << I40E_TXD_CTX_QW1_DTYPE_SHIFT) | ((u64)cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) | @@ -832,617 +218,301 @@ TXD->type_cmd_tso_mss = htole64(type_cmd_tso_mss); TXD->tunneling_params = htole32(0); - buf->m_head = NULL; - buf->eop_index = -1; - if (++idx == que->num_desc) - idx = 0; - - txr->avail--; - txr->next_avail = idx; - - return TRUE; + // XXX: This guy really likes masking numbers + return ((idx + 1) & (scctx->isc_ntxd[0]-1)); } -/* -** ixl_get_tx_head - Retrieve the value from the -** location the HW records its HEAD index -*/ -static inline u32 -ixl_get_tx_head(struct ixl_queue *que) -{ - struct tx_ring *txr = &que->txr; - void *head = &txr->base[que->num_desc]; - return LE32_TO_CPU(*(volatile __le32 *)head); -} - -/********************************************************************** +/********************************************************************* * - * Examine each tx_buffer in the used queue. If the hardware is done - * processing the packet then free associated resources. The - * tx_buffer is put back on the free queue. + * This routine maps the mbufs to tx descriptors, allowing the + * TX engine to transmit the packets. + * - return 0 on success, positive on failure * **********************************************************************/ -bool -ixl_txeof(struct ixl_queue *que) +#define IXL_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS) + +static int +ixl_isc_txd_encap(void *arg, if_pkt_info_t pi) { + struct ixl_vsi *vsi = arg; + if_softc_ctx_t scctx = vsi->shared; + struct ixl_tx_queue *que = &vsi->tx_queues[pi->ipi_qsidx]; struct tx_ring *txr = &que->txr; - u32 first, last, head, done, processed; - struct ixl_tx_buf *buf; - struct i40e_tx_desc *tx_desc, *eop_desc; + int nsegs = pi->ipi_nsegs; + bus_dma_segment_t *segs = pi->ipi_segs; + struct i40e_tx_desc *txd = NULL; + int i, j, mask, pidx_last; + u32 cmd, off, tx_intr; + // device_printf(iflib_get_dev(vsi->ctx), "%s: begin\n", __func__); - mtx_assert(&txr->mtx, MA_OWNED); + cmd = off = 0; + i = pi->ipi_pidx; -#ifdef DEV_NETMAP - // XXX todo: implement moderation - if (netmap_tx_irq(que->vsi->ifp, que->me)) - return FALSE; -#endif /* DEF_NETMAP */ + tx_intr = (pi->ipi_flags & IPI_TX_INTR); + //device_printf(iflib_get_dev(vsi->ctx), "%s: tx_intr %d\n", __func__, tx_intr); - /* These are not the descriptors you seek, move along :) */ - if (txr->avail == que->num_desc) { - atomic_store_rel_32(&txr->watchdog_timer, 0); - return FALSE; - } + /* Set up the TSO/CSUM offload */ + if (pi->ipi_csum_flags & CSUM_OFFLOAD) { + /* Set up the TSO context descriptor if required */ + if (pi->ipi_csum_flags & CSUM_TSO) { + if (ixl_tso_detect_sparse(segs, nsegs, pi->ipi_tso_segsz)) + return (EFBIG); - processed = 0; - first = txr->next_to_clean; - buf = &txr->buffers[first]; - tx_desc = (struct i40e_tx_desc *)&txr->base[first]; - last = buf->eop_index; - if (last == -1) - return FALSE; - eop_desc = (struct i40e_tx_desc *)&txr->base[last]; + i = ixl_tso_setup(txr, pi); + } + ixl_tx_setup_offload(que, pi, &cmd, &off); + } - /* Get the Head WB value */ - head = ixl_get_tx_head(que); + if (pi->ipi_mflags & M_VLANTAG) + cmd |= I40E_TX_DESC_CMD_IL2TAG1; - /* - ** Get the index of the first descriptor - ** BEYOND the EOP and call that 'done'. - ** I do this so the comparison in the - ** inner while loop below can be simple - */ - if (++last == que->num_desc) last = 0; - done = last; + cmd |= (I40E_TX_DESC_CMD_ICRC | I40E_TX_DESC_CMD_RS); + mask = scctx->isc_ntxd[0] - 1; + for (j = 0; j < nsegs; j++) { + bus_size_t seglen; - bus_dmamap_sync(txr->dma.tag, txr->dma.map, - BUS_DMASYNC_POSTREAD); - /* - ** The HEAD index of the ring is written in a - ** defined location, this rather than a done bit - ** is what is used to keep track of what must be - ** 'cleaned'. - */ - while (first != head) { - /* We clean the range of the packet */ - while (first != done) { - ++txr->avail; - ++processed; - - if (buf->m_head) { - txr->bytes += /* for ITR adjustment */ - buf->m_head->m_pkthdr.len; - txr->tx_bytes += /* for TX stats */ - buf->m_head->m_pkthdr.len; - bus_dmamap_sync(buf->tag, - buf->map, - BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(buf->tag, - buf->map); - m_freem(buf->m_head); - buf->m_head = NULL; - buf->map = NULL; - } - buf->eop_index = -1; + txd = &txr->tx_base[i]; + seglen = segs[j].ds_len; - if (++first == que->num_desc) - first = 0; + txd->buffer_addr = htole64(segs[j].ds_addr); + txd->cmd_type_offset_bsz = + htole64(I40E_TX_DESC_DTYPE_DATA + | ((u64)cmd << I40E_TXD_QW1_CMD_SHIFT) + | ((u64)off << I40E_TXD_QW1_OFFSET_SHIFT) + | ((u64)seglen << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) + | ((u64)htole16(pi->ipi_vtag) << I40E_TXD_QW1_L2TAG1_SHIFT)); - buf = &txr->buffers[first]; - tx_desc = &txr->base[first]; - } - ++txr->packets; - /* See if there is more work now */ - last = buf->eop_index; - if (last != -1) { - eop_desc = &txr->base[last]; - /* Get next done point */ - if (++last == que->num_desc) last = 0; - done = last; - } else - break; + pidx_last = i; + i = (i+1) & mask; } - bus_dmamap_sync(txr->dma.tag, txr->dma.map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); + /* Set the last descriptor for report */ + txd->cmd_type_offset_bsz |= + htole64(((u64)IXL_TXD_CMD << I40E_TXD_QW1_CMD_SHIFT)); + /* Add to report status array (if using TX interrupts) */ + if (tx_intr) { + txr->tx_rsq[txr->tx_rs_pidx] = pidx_last; + txr->tx_rs_pidx = (txr->tx_rs_pidx+1) & mask; + // TODO: Should this assert be kept? + MPASS(txr->tx_rs_pidx != txr->tx_rs_cidx); + } + pi->ipi_new_pidx = i; - txr->next_to_clean = first; + ++txr->total_packets; + return (0); +} +static void +ixl_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx) +{ + struct ixl_vsi *vsi = arg; + struct tx_ring *txr = &vsi->tx_queues[txqid].txr; + + // device_printf(iflib_get_dev(vsi->ctx), "%s: begin\n", __func__); /* - * If there are no pending descriptors, clear the timeout. + * Advance the Transmit Descriptor Tail (Tdt), this tells the + * hardware that this frame is available to transmit. */ - if (txr->avail == que->num_desc) { - atomic_store_rel_32(&txr->watchdog_timer, 0); - return FALSE; - } - - return TRUE; + wr32(vsi->hw, txr->tail, pidx); } /********************************************************************* * - * Refresh mbuf buffers for RX descriptor rings - * - now keeps its own state so discards due to resource - * exhaustion are unnecessary, if an mbuf cannot be obtained - * it just returns, keeping its placeholder, thus it can simply - * be recalled to try again. + * (Re)Initialize a queue transmit ring. + * - called by init, it clears the descriptor ring, + * and frees any stale mbufs * **********************************************************************/ -static void -ixl_refresh_mbufs(struct ixl_queue *que, int limit) +void +ixl_init_tx_ring(struct ixl_vsi *vsi, struct ixl_tx_queue *que) { - struct ixl_vsi *vsi = que->vsi; - struct rx_ring *rxr = &que->rxr; - bus_dma_segment_t hseg[1]; - bus_dma_segment_t pseg[1]; - struct ixl_rx_buf *buf; - struct mbuf *mh, *mp; - int i, j, nsegs, error; - bool refreshed = FALSE; - - i = j = rxr->next_refresh; - /* Control the loop with one beyond */ - if (++j == que->num_desc) - j = 0; - - while (j != limit) { - buf = &rxr->buffers[i]; - if (rxr->hdr_split == FALSE) - goto no_split; - - if (buf->m_head == NULL) { - mh = m_gethdr(M_NOWAIT, MT_DATA); - if (mh == NULL) - goto update; - } else - mh = buf->m_head; - - mh->m_pkthdr.len = mh->m_len = MHLEN; - mh->m_len = MHLEN; - mh->m_flags |= M_PKTHDR; - /* Get the memory mapping */ - error = bus_dmamap_load_mbuf_sg(rxr->htag, - buf->hmap, mh, hseg, &nsegs, BUS_DMA_NOWAIT); - if (error != 0) { - printf("Refresh mbufs: hdr dmamap load" - " failure - %d\n", error); - m_free(mh); - buf->m_head = NULL; - goto update; - } - buf->m_head = mh; - bus_dmamap_sync(rxr->htag, buf->hmap, - BUS_DMASYNC_PREREAD); - rxr->base[i].read.hdr_addr = - htole64(hseg[0].ds_addr); - -no_split: - if (buf->m_pack == NULL) { - mp = m_getjcl(M_NOWAIT, MT_DATA, - M_PKTHDR, rxr->mbuf_sz); - if (mp == NULL) - goto update; - } else - mp = buf->m_pack; - - mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz; - /* Get the memory mapping */ - error = bus_dmamap_load_mbuf_sg(rxr->ptag, - buf->pmap, mp, pseg, &nsegs, BUS_DMA_NOWAIT); - if (error != 0) { - printf("Refresh mbufs: payload dmamap load" - " failure - %d\n", error); - m_free(mp); - buf->m_pack = NULL; - goto update; - } - buf->m_pack = mp; - bus_dmamap_sync(rxr->ptag, buf->pmap, - BUS_DMASYNC_PREREAD); - rxr->base[i].read.pkt_addr = - htole64(pseg[0].ds_addr); - /* Used only when doing header split */ - rxr->base[i].read.hdr_addr = 0; - - refreshed = TRUE; - /* Next is precalculated */ - i = j; - rxr->next_refresh = i; - if (++j == que->num_desc) - j = 0; - } -update: - if (refreshed) /* Update hardware tail index */ - wr32(vsi->hw, rxr->tail, rxr->next_refresh); - return; -} + struct tx_ring *txr = &que->txr; + // device_printf(iflib_get_dev(vsi->ctx), "%s: begin\n", __func__); -/********************************************************************* - * - * Allocate memory for rx_buffer structures. Since we use one - * rx_buffer per descriptor, the maximum number of rx_buffer's - * that we'll need is equal to the number of receive descriptors - * that we've defined. - * - **********************************************************************/ -int -ixl_allocate_rx_data(struct ixl_queue *que) -{ - struct rx_ring *rxr = &que->rxr; - struct ixl_vsi *vsi = que->vsi; - device_t dev = vsi->dev; - struct ixl_rx_buf *buf; - int i, bsize, error; - - bsize = sizeof(struct ixl_rx_buf) * que->num_desc; - if (!(rxr->buffers = - (struct ixl_rx_buf *) malloc(bsize, - M_DEVBUF, M_NOWAIT | M_ZERO))) { - device_printf(dev, "Unable to allocate rx_buffer memory\n"); - error = ENOMEM; - return (error); - } + /* Clear the old ring contents */ + bzero((void *)txr->tx_base, + (sizeof(struct i40e_tx_desc)) * vsi->shared->isc_ntxd[0]); - if ((error = bus_dma_tag_create(NULL, /* parent */ - 1, 0, /* alignment, bounds */ - BUS_SPACE_MAXADDR, /* lowaddr */ - BUS_SPACE_MAXADDR, /* highaddr */ - NULL, NULL, /* filter, filterarg */ - MSIZE, /* maxsize */ - 1, /* nsegments */ - MSIZE, /* maxsegsize */ - 0, /* flags */ - NULL, /* lockfunc */ - NULL, /* lockfuncarg */ - &rxr->htag))) { - device_printf(dev, "Unable to create RX DMA htag\n"); - return (error); - } + // TODO: Write max descriptor index instead of 0? + wr32(vsi->hw, txr->tail, 0); + wr32(vsi->hw, I40E_QTX_HEAD(txr->me), 0); +} - if ((error = bus_dma_tag_create(NULL, /* parent */ - 1, 0, /* alignment, bounds */ - BUS_SPACE_MAXADDR, /* lowaddr */ - BUS_SPACE_MAXADDR, /* highaddr */ - NULL, NULL, /* filter, filterarg */ - MJUM16BYTES, /* maxsize */ - 1, /* nsegments */ - MJUM16BYTES, /* maxsegsize */ - 0, /* flags */ - NULL, /* lockfunc */ - NULL, /* lockfuncarg */ - &rxr->ptag))) { - device_printf(dev, "Unable to create RX DMA ptag\n"); - return (error); - } +/* +** ixl_get_tx_head - Retrieve the value from the +** location the HW records its HEAD index +*/ +static inline u32 +ixl_get_tx_head(struct ixl_tx_queue *que) +{ + struct tx_ring *txr = &que->txr; + void *head = &txr->tx_base[que->vsi->shared->isc_ntxd[0]]; - for (i = 0; i < que->num_desc; i++) { - buf = &rxr->buffers[i]; - error = bus_dmamap_create(rxr->htag, - BUS_DMA_NOWAIT, &buf->hmap); - if (error) { - device_printf(dev, "Unable to create RX head map\n"); - break; - } - error = bus_dmamap_create(rxr->ptag, - BUS_DMA_NOWAIT, &buf->pmap); - if (error) { - device_printf(dev, "Unable to create RX pkt map\n"); - break; - } - } + // device_printf(iflib_get_dev(que->vsi->ctx), "%s: begin\n", __func__); - return (error); + return LE32_TO_CPU(*(volatile __le32 *)head); } - -/********************************************************************* - * - * (Re)Initialize the queue receive ring and its buffers. - * - **********************************************************************/ -int -ixl_init_rx_ring(struct ixl_queue *que) +static int +ixl_isc_txd_credits_update_dd(void *arg, uint16_t txqid, bool clear) { - struct rx_ring *rxr = &que->rxr; - struct ixl_vsi *vsi = que->vsi; -#if defined(INET6) || defined(INET) - struct ifnet *ifp = vsi->ifp; - struct lro_ctrl *lro = &rxr->lro; -#endif - struct ixl_rx_buf *buf; - bus_dma_segment_t pseg[1], hseg[1]; - int rsize, nsegs, error = 0; -#ifdef DEV_NETMAP - struct netmap_adapter *na = NA(que->vsi->ifp); - struct netmap_slot *slot; -#endif /* DEV_NETMAP */ - - IXL_RX_LOCK(rxr); -#ifdef DEV_NETMAP - /* same as in ixl_init_tx_ring() */ - slot = netmap_reset(na, NR_RX, que->me, 0); -#endif /* DEV_NETMAP */ - /* Clear the ring contents */ - rsize = roundup2(que->num_desc * - sizeof(union i40e_rx_desc), DBA_ALIGN); - bzero((void *)rxr->base, rsize); - /* Cleanup any existing buffers */ - for (int i = 0; i < que->num_desc; i++) { - buf = &rxr->buffers[i]; - if (buf->m_head != NULL) { - bus_dmamap_sync(rxr->htag, buf->hmap, - BUS_DMASYNC_POSTREAD); - bus_dmamap_unload(rxr->htag, buf->hmap); - buf->m_head->m_flags |= M_PKTHDR; - m_freem(buf->m_head); - } - if (buf->m_pack != NULL) { - bus_dmamap_sync(rxr->ptag, buf->pmap, - BUS_DMASYNC_POSTREAD); - bus_dmamap_unload(rxr->ptag, buf->pmap); - buf->m_pack->m_flags |= M_PKTHDR; - m_freem(buf->m_pack); - } - buf->m_head = NULL; - buf->m_pack = NULL; - } + struct ixl_vsi *vsi = arg; + struct ixl_tx_queue *tx_que = &vsi->tx_queues[txqid]; + if_softc_ctx_t scctx = vsi->shared; + struct tx_ring *txr = &tx_que->txr; - /* header split is off */ - rxr->hdr_split = FALSE; + qidx_t processed = 0; + qidx_t cur, prev, ntxd, rs_cidx; + int32_t delta; + bool is_done; - /* Now replenish the mbufs */ - for (int j = 0; j != que->num_desc; ++j) { - struct mbuf *mh, *mp; + // device_printf(iflib_get_dev(vsi->ctx), "%s: begin\n", __func__); - buf = &rxr->buffers[j]; -#ifdef DEV_NETMAP - /* - * In netmap mode, fill the map and set the buffer - * address in the NIC ring, considering the offset - * between the netmap and NIC rings (see comment in - * ixgbe_setup_transmit_ring() ). No need to allocate - * an mbuf, so end the block with a continue; - */ - if (slot) { - int sj = netmap_idx_n2k(&na->rx_rings[que->me], j); - uint64_t paddr; - void *addr; - - addr = PNMB(na, slot + sj, &paddr); - netmap_load_map(na, rxr->dma.tag, buf->pmap, addr); - /* Update descriptor and the cached value */ - rxr->base[j].read.pkt_addr = htole64(paddr); - rxr->base[j].read.hdr_addr = 0; - continue; - } -#endif /* DEV_NETMAP */ + rs_cidx = txr->tx_rs_cidx; + /* + device_printf(iflib_get_dev(vsi->ctx), "%s: rs_cidx %d, txr->tx_rs_pidx %d\n", __func__, + rs_cidx, txr->tx_rs_pidx); + */ + if (rs_cidx == txr->tx_rs_pidx) + return (0); + cur = txr->tx_rsq[rs_cidx]; + MPASS(cur != QIDX_INVALID); + is_done = ixl_is_tx_desc_done(txr, cur); + + if (clear == false || !is_done) + return (0); + + prev = txr->tx_cidx_processed; + ntxd = scctx->isc_ntxd[0]; + do { + delta = (int32_t)cur - (int32_t)prev; + MPASS(prev == 0 || delta != 0); + if (delta < 0) + delta += ntxd; /* - ** Don't allocate mbufs if not - ** doing header split, its wasteful - */ - if (rxr->hdr_split == FALSE) - goto skip_head; - - /* First the header */ - buf->m_head = m_gethdr(M_NOWAIT, MT_DATA); - if (buf->m_head == NULL) { - error = ENOBUFS; - goto fail; - } - m_adj(buf->m_head, ETHER_ALIGN); - mh = buf->m_head; - mh->m_len = mh->m_pkthdr.len = MHLEN; - mh->m_flags |= M_PKTHDR; - /* Get the memory mapping */ - error = bus_dmamap_load_mbuf_sg(rxr->htag, - buf->hmap, buf->m_head, hseg, - &nsegs, BUS_DMA_NOWAIT); - if (error != 0) /* Nothing elegant to do here */ - goto fail; - bus_dmamap_sync(rxr->htag, - buf->hmap, BUS_DMASYNC_PREREAD); - /* Update descriptor */ - rxr->base[j].read.hdr_addr = htole64(hseg[0].ds_addr); - -skip_head: - /* Now the payload cluster */ - buf->m_pack = m_getjcl(M_NOWAIT, MT_DATA, - M_PKTHDR, rxr->mbuf_sz); - if (buf->m_pack == NULL) { - error = ENOBUFS; - goto fail; - } - mp = buf->m_pack; - mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz; - /* Get the memory mapping */ - error = bus_dmamap_load_mbuf_sg(rxr->ptag, - buf->pmap, mp, pseg, - &nsegs, BUS_DMA_NOWAIT); - if (error != 0) - goto fail; - bus_dmamap_sync(rxr->ptag, - buf->pmap, BUS_DMASYNC_PREREAD); - /* Update descriptor */ - rxr->base[j].read.pkt_addr = htole64(pseg[0].ds_addr); - rxr->base[j].read.hdr_addr = 0; - } + device_printf(iflib_get_dev(vsi->ctx), + "%s: cidx_processed=%u cur=%u clear=%d delta=%d\n", + __FUNCTION__, prev, cur, clear, delta); + */ + processed += delta; + prev = cur; + rs_cidx = (rs_cidx + 1) & (ntxd-1); + if (rs_cidx == txr->tx_rs_pidx) + break; + cur = txr->tx_rsq[rs_cidx]; + MPASS(cur != QIDX_INVALID); + is_done = ixl_is_tx_desc_done(txr, cur); + } while (is_done); + txr->tx_rs_cidx = rs_cidx; + txr->tx_cidx_processed = prev; - /* Setup our descriptor indices */ - rxr->next_check = 0; - rxr->next_refresh = 0; - rxr->lro_enabled = FALSE; - rxr->split = 0; - rxr->bytes = 0; - rxr->discard = FALSE; + // device_printf(iflib_get_dev(vsi->ctx), "%s: processed %d\n", __func__, processed); + return (processed); +} - wr32(vsi->hw, rxr->tail, que->num_desc - 1); - ixl_flush(vsi->hw); +static int +ixl_isc_txd_credits_update_hwb(void *arg, uint16_t txqid, bool clear) +{ + struct ixl_vsi *vsi = arg; + struct ixl_tx_queue *que = &vsi->tx_queues[txqid]; -#if defined(INET6) || defined(INET) - /* - ** Now set up the LRO interface: - */ - if (ifp->if_capenable & IFCAP_LRO) { - int err = tcp_lro_init(lro); - if (err) { - if_printf(ifp, "queue %d: LRO Initialization failed!\n", que->me); - goto fail; - } - INIT_DBG_IF(ifp, "queue %d: RX Soft LRO Initialized", que->me); - rxr->lro_enabled = TRUE; - lro->ifp = vsi->ifp; - } -#endif + int head, credits = 0; - bus_dmamap_sync(rxr->dma.tag, rxr->dma.map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); + /* Get the Head WB value */ + head = ixl_get_tx_head(que); -fail: - IXL_RX_UNLOCK(rxr); - return (error); -} + // TODO: Replace cidx + // credits = head - cidx; + if (credits < 0) + credits += vsi->shared->isc_ntxd[0]; + return (credits); +} /********************************************************************* * - * Free station receive ring data structures + * Refresh mbuf buffers for RX descriptor rings + * - now keeps its own state so discards due to resource + * exhaustion are unnecessary, if an mbuf cannot be obtained + * it just returns, keeping its placeholder, thus it can simply + * be recalled to try again. * **********************************************************************/ -void -ixl_free_que_rx(struct ixl_queue *que) +static void +ixl_isc_rxd_refill(void *arg, if_rxd_update_t iru) { - struct rx_ring *rxr = &que->rxr; - struct ixl_rx_buf *buf; - - INIT_DBG_IF(que->vsi->ifp, "queue %d: begin", que->me); - - /* Cleanup any existing buffers */ - if (rxr->buffers != NULL) { - for (int i = 0; i < que->num_desc; i++) { - buf = &rxr->buffers[i]; - if (buf->m_head != NULL) { - bus_dmamap_sync(rxr->htag, buf->hmap, - BUS_DMASYNC_POSTREAD); - bus_dmamap_unload(rxr->htag, buf->hmap); - buf->m_head->m_flags |= M_PKTHDR; - m_freem(buf->m_head); - } - if (buf->m_pack != NULL) { - bus_dmamap_sync(rxr->ptag, buf->pmap, - BUS_DMASYNC_POSTREAD); - bus_dmamap_unload(rxr->ptag, buf->pmap); - buf->m_pack->m_flags |= M_PKTHDR; - m_freem(buf->m_pack); - } - buf->m_head = NULL; - buf->m_pack = NULL; - if (buf->hmap != NULL) { - bus_dmamap_destroy(rxr->htag, buf->hmap); - buf->hmap = NULL; - } - if (buf->pmap != NULL) { - bus_dmamap_destroy(rxr->ptag, buf->pmap); - buf->pmap = NULL; - } - } - if (rxr->buffers != NULL) { - free(rxr->buffers, M_DEVBUF); - rxr->buffers = NULL; - } - } - - if (rxr->htag != NULL) { - bus_dma_tag_destroy(rxr->htag); - rxr->htag = NULL; - } - if (rxr->ptag != NULL) { - bus_dma_tag_destroy(rxr->ptag); - rxr->ptag = NULL; + struct ixl_vsi *vsi = arg; + if_softc_ctx_t scctx = vsi->shared; + struct rx_ring *rxr = &((vsi->rx_queues[iru->iru_qsidx]).rxr); + uint64_t *paddrs; + uint32_t next_pidx, pidx; + uint16_t count; + int i; + + paddrs = iru->iru_paddrs; + pidx = iru->iru_pidx; + count = iru->iru_count; + + for (i = 0, next_pidx = pidx; i < count; i++) { + rxr->rx_base[next_pidx].read.pkt_addr = htole64(paddrs[i]); + if (++next_pidx == scctx->isc_nrxd[0]) + next_pidx = 0; } - - INIT_DBG_IF(que->vsi->ifp, "queue %d: end", que->me); - return; } -static inline void -ixl_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u8 ptype) +static void +ixl_isc_rxd_flush(void * arg, uint16_t rxqid, uint8_t flid __unused, qidx_t pidx) { + struct ixl_vsi *vsi = arg; + struct rx_ring *rxr = &vsi->rx_queues[rxqid].rxr; -#if defined(INET6) || defined(INET) - /* - * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet - * should be computed by hardware. Also it should not have VLAN tag in - * ethernet header. - */ - if (rxr->lro_enabled && - (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 && - (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) == - (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) { - /* - * Send to the stack if: - ** - LRO not enabled, or - ** - no LRO resources, or - ** - lro enqueue fails - */ - if (rxr->lro.lro_cnt != 0) - if (tcp_lro_rx(&rxr->lro, m, 0) == 0) - return; - } -#endif - IXL_RX_UNLOCK(rxr); - (*ifp->if_input)(ifp, m); - IXL_RX_LOCK(rxr); + wr32(vsi->hw, rxr->tail, pidx); } - -static inline void -ixl_rx_discard(struct rx_ring *rxr, int i) +static int +ixl_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget) { - struct ixl_rx_buf *rbuf; + struct ixl_vsi *vsi = arg; + struct rx_ring *rxr = &vsi->rx_queues[rxqid].rxr; + union i40e_rx_desc *rxd; + u64 qword; + uint32_t status; + int cnt, i, nrxd; - rbuf = &rxr->buffers[i]; + // device_printf(iflib_get_dev(vsi->ctx), "%s: begin\n", __func__); - if (rbuf->fmp != NULL) {/* Partial chain ? */ - rbuf->fmp->m_flags |= M_PKTHDR; - m_freem(rbuf->fmp); - rbuf->fmp = NULL; - } + nrxd = vsi->shared->isc_nrxd[0]; - /* - ** With advanced descriptors the writeback - ** clobbers the buffer addrs, so its easier - ** to just free the existing mbufs and take - ** the normal refresh path to get new buffers - ** and mapping. - */ - if (rbuf->m_head) { - m_free(rbuf->m_head); - rbuf->m_head = NULL; + // Stolen from em + if (budget == 1) { + rxd = &rxr->rx_base[idx]; + qword = le64toh(rxd->wb.qword1.status_error_len); + status = (qword & I40E_RXD_QW1_STATUS_MASK) + >> I40E_RXD_QW1_STATUS_SHIFT; + return !!(status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)); } - - if (rbuf->m_pack) { - m_free(rbuf->m_pack); - rbuf->m_pack = NULL; + + for (cnt = 0, i = idx; cnt < nrxd - 1 && cnt <= budget;) { + rxd = &rxr->rx_base[i]; + qword = le64toh(rxd->wb.qword1.status_error_len); + status = (qword & I40E_RXD_QW1_STATUS_MASK) + >> I40E_RXD_QW1_STATUS_SHIFT; + + if ((status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) == 0) + break; + if (++i == nrxd) + i = 0; + if (status & (1 << I40E_RX_DESC_STATUS_EOF_SHIFT)) + cnt++; } - return; + return (cnt); } -#ifdef RSS /* ** i40e_ptype_to_hash: parse the packet type ** to determine the appropriate hash. @@ -1457,122 +527,96 @@ ex = decoded.outer_frag; if (!decoded.known) - return M_HASHTYPE_OPAQUE_HASH; + return M_HASHTYPE_OPAQUE; if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_L2) - return M_HASHTYPE_OPAQUE_HASH; + return M_HASHTYPE_OPAQUE; /* Note: anything that gets to this point is IP */ if (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6) { switch (decoded.inner_prot) { - case I40E_RX_PTYPE_INNER_PROT_TCP: - if (ex) - return M_HASHTYPE_RSS_TCP_IPV6_EX; - else - return M_HASHTYPE_RSS_TCP_IPV6; - case I40E_RX_PTYPE_INNER_PROT_UDP: - if (ex) - return M_HASHTYPE_RSS_UDP_IPV6_EX; - else - return M_HASHTYPE_RSS_UDP_IPV6; - default: - if (ex) - return M_HASHTYPE_RSS_IPV6_EX; - else - return M_HASHTYPE_RSS_IPV6; + case I40E_RX_PTYPE_INNER_PROT_TCP: + if (ex) + return M_HASHTYPE_RSS_TCP_IPV6_EX; + else + return M_HASHTYPE_RSS_TCP_IPV6; + case I40E_RX_PTYPE_INNER_PROT_UDP: + if (ex) + return M_HASHTYPE_RSS_UDP_IPV6_EX; + else + return M_HASHTYPE_RSS_UDP_IPV6; + default: + if (ex) + return M_HASHTYPE_RSS_IPV6_EX; + else + return M_HASHTYPE_RSS_IPV6; } } if (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV4) { switch (decoded.inner_prot) { - case I40E_RX_PTYPE_INNER_PROT_TCP: - return M_HASHTYPE_RSS_TCP_IPV4; - case I40E_RX_PTYPE_INNER_PROT_UDP: - if (ex) - return M_HASHTYPE_RSS_UDP_IPV4_EX; - else - return M_HASHTYPE_RSS_UDP_IPV4; - default: - return M_HASHTYPE_RSS_IPV4; + case I40E_RX_PTYPE_INNER_PROT_TCP: + return M_HASHTYPE_RSS_TCP_IPV4; + case I40E_RX_PTYPE_INNER_PROT_UDP: + if (ex) + return M_HASHTYPE_RSS_UDP_IPV4_EX; + else + return M_HASHTYPE_RSS_UDP_IPV4; + default: + return M_HASHTYPE_RSS_IPV4; } } /* We should never get here!! */ - return M_HASHTYPE_OPAQUE_HASH; + return M_HASHTYPE_OPAQUE; } -#endif /* RSS */ /********************************************************************* * - * This routine executes in interrupt context. It replenishes - * the mbufs in the descriptor and sends data which has been + * This routine executes in ithread context. It sends data which has been * dma'ed into host memory to upper layer. * - * We loop at most count times if count is > 0, or until done if - * count < 0. - * - * Return TRUE for more work, FALSE for all clean. + * Returns 0 upon success, errno on failure *********************************************************************/ -bool -ixl_rxeof(struct ixl_queue *que, int count) + +static int +ixl_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) { - struct ixl_vsi *vsi = que->vsi; + struct ixl_vsi *vsi = arg; + struct ixl_rx_queue *que = &vsi->rx_queues[ri->iri_qsidx]; struct rx_ring *rxr = &que->rxr; - struct ifnet *ifp = vsi->ifp; -#if defined(INET6) || defined(INET) - struct lro_ctrl *lro = &rxr->lro; -#endif - int i, nextp, processed = 0; union i40e_rx_desc *cur; - struct ixl_rx_buf *rbuf, *nbuf; - - - IXL_RX_LOCK(rxr); - -#ifdef DEV_NETMAP - if (netmap_rx_irq(ifp, que->me, &count)) { - IXL_RX_UNLOCK(rxr); - return (FALSE); - } -#endif /* DEV_NETMAP */ - - for (i = rxr->next_check; count != 0;) { - struct mbuf *sendmp, *mh, *mp; - u32 status, error; - u16 hlen, plen, vtag; - u64 qword; - u8 ptype; - bool eop; - - /* Sync the ring. */ - bus_dmamap_sync(rxr->dma.tag, rxr->dma.map, - BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); - - cur = &rxr->base[i]; + u32 status, error; + u16 hlen, plen, vtag; + u64 qword; + u8 ptype; + bool eop; + int i, cidx; + + /* XXX: No packet split support, so hlen is unused */ + // device_printf(iflib_get_dev(vsi->ctx), "%s: begin\n", __func__); + + cidx = ri->iri_cidx; + i = 0; + do { + cur = &rxr->rx_base[cidx]; qword = le64toh(cur->wb.qword1.status_error_len); status = (qword & I40E_RXD_QW1_STATUS_MASK) - >> I40E_RXD_QW1_STATUS_SHIFT; + >> I40E_RXD_QW1_STATUS_SHIFT; error = (qword & I40E_RXD_QW1_ERROR_MASK) - >> I40E_RXD_QW1_ERROR_SHIFT; + >> I40E_RXD_QW1_ERROR_SHIFT; plen = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) - >> I40E_RXD_QW1_LENGTH_PBUF_SHIFT; + >> I40E_RXD_QW1_LENGTH_PBUF_SHIFT; hlen = (qword & I40E_RXD_QW1_LENGTH_HBUF_MASK) - >> I40E_RXD_QW1_LENGTH_HBUF_SHIFT; + >> I40E_RXD_QW1_LENGTH_HBUF_SHIFT; ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) - >> I40E_RXD_QW1_PTYPE_SHIFT; + >> I40E_RXD_QW1_PTYPE_SHIFT; - if ((status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) == 0) { - ++rxr->not_done; - break; - } - if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) - break; + /* we should never be called without a valid descriptor */ + MPASS((status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) != 0); + + ri->iri_len += plen; + rxr->bytes += plen; - count--; - sendmp = NULL; - nbuf = NULL; cur->wb.qword1.status_error_len = 0; - rbuf = &rxr->buffers[i]; - mh = rbuf->m_head; - mp = rbuf->m_pack; eop = (status & (1 << I40E_RX_DESC_STATUS_EOF_SHIFT)); if (status & (1 << I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)) vtag = le16toh(cur->wb.qword0.lo_dword.l2tag1); @@ -1584,180 +628,36 @@ ** note that only EOP descriptor has valid ** error results. */ - if (eop && (error & (1 << I40E_RX_DESC_ERROR_RXE_SHIFT))) { + if (eop && (error & (1 << I40E_RX_DESC_ERROR_RXE_SHIFT))) { rxr->desc_errs++; - ixl_rx_discard(rxr, i); - goto next_desc; - } - - /* Prefetch the next buffer */ - if (!eop) { - nextp = i + 1; - if (nextp == que->num_desc) - nextp = 0; - nbuf = &rxr->buffers[nextp]; - prefetch(nbuf); - } - - /* - ** The header mbuf is ONLY used when header - ** split is enabled, otherwise we get normal - ** behavior, ie, both header and payload - ** are DMA'd into the payload buffer. - ** - ** Rather than using the fmp/lmp global pointers - ** we now keep the head of a packet chain in the - ** buffer struct and pass this along from one - ** descriptor to the next, until we get EOP. - */ - if (rxr->hdr_split && (rbuf->fmp == NULL)) { - if (hlen > IXL_RX_HDR) - hlen = IXL_RX_HDR; - mh->m_len = hlen; - mh->m_flags |= M_PKTHDR; - mh->m_next = NULL; - mh->m_pkthdr.len = mh->m_len; - /* Null buf pointer so it is refreshed */ - rbuf->m_head = NULL; - /* - ** Check the payload length, this - ** could be zero if its a small - ** packet. - */ - if (plen > 0) { - mp->m_len = plen; - mp->m_next = NULL; - mp->m_flags &= ~M_PKTHDR; - mh->m_next = mp; - mh->m_pkthdr.len += mp->m_len; - /* Null buf pointer so it is refreshed */ - rbuf->m_pack = NULL; - rxr->split++; - } - /* - ** Now create the forward - ** chain so when complete - ** we wont have to. - */ - if (eop == 0) { - /* stash the chain head */ - nbuf->fmp = mh; - /* Make forward chain */ - if (plen) - mp->m_next = nbuf->m_pack; - else - mh->m_next = nbuf->m_pack; - } else { - /* Singlet, prepare to send */ - sendmp = mh; - if (vtag) { - sendmp->m_pkthdr.ether_vtag = vtag; - sendmp->m_flags |= M_VLANTAG; - } - } - } else { - /* - ** Either no header split, or a - ** secondary piece of a fragmented - ** split packet. - */ - mp->m_len = plen; - /* - ** See if there is a stored head - ** that determines what we are - */ - sendmp = rbuf->fmp; - rbuf->m_pack = rbuf->fmp = NULL; - - if (sendmp != NULL) /* secondary frag */ - sendmp->m_pkthdr.len += mp->m_len; - else { - /* first desc of a non-ps chain */ - sendmp = mp; - sendmp->m_flags |= M_PKTHDR; - sendmp->m_pkthdr.len = mp->m_len; - } - /* Pass the head pointer on */ - if (eop == 0) { - nbuf->fmp = sendmp; - sendmp = NULL; - mp->m_next = nbuf->m_pack; - } - } - ++processed; - /* Sending this frame? */ - if (eop) { - sendmp->m_pkthdr.rcvif = ifp; - /* gather stats */ - rxr->rx_packets++; - rxr->rx_bytes += sendmp->m_pkthdr.len; - /* capture data for dynamic ITR adjustment */ - rxr->packets++; - rxr->bytes += sendmp->m_pkthdr.len; - /* Set VLAN tag (field only valid in eop desc) */ - if (vtag) { - sendmp->m_pkthdr.ether_vtag = vtag; - sendmp->m_flags |= M_VLANTAG; - } - if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) - ixl_rx_checksum(sendmp, status, error, ptype); -#ifdef RSS - sendmp->m_pkthdr.flowid = - le32toh(cur->wb.qword0.hi_dword.rss); - M_HASHTYPE_SET(sendmp, ixl_ptype_to_hash(ptype)); -#else - sendmp->m_pkthdr.flowid = que->msix; - M_HASHTYPE_SET(sendmp, M_HASHTYPE_OPAQUE); -#endif - } -next_desc: - bus_dmamap_sync(rxr->dma.tag, rxr->dma.map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - - /* Advance our pointers to the next descriptor. */ - if (++i == que->num_desc) - i = 0; - - /* Now send to the stack or do LRO */ - if (sendmp != NULL) { - rxr->next_check = i; - ixl_rx_input(rxr, ifp, sendmp, ptype); - i = rxr->next_check; - } - - /* Every 8 descriptors we go to refresh mbufs */ - if (processed == 8) { - ixl_refresh_mbufs(que, i); - processed = 0; - } - } - - /* Refresh any remaining buf structs */ - if (ixl_rx_unrefreshed(que)) - ixl_refresh_mbufs(que, i); - - rxr->next_check = i; - -#if defined(INET6) || defined(INET) - /* - * Flush any outstanding LRO work - */ -#if __FreeBSD_version >= 1100105 - tcp_lro_flush_all(lro); -#else - struct lro_entry *queued; - while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) { - SLIST_REMOVE_HEAD(&lro->lro_active, next); - tcp_lro_flush(lro, queued); - } -#endif -#endif /* defined(INET6) || defined(INET) */ - - IXL_RX_UNLOCK(rxr); - return (FALSE); + return (EBADMSG); + } + ri->iri_frags[i].irf_flid = 0; + ri->iri_frags[i].irf_idx = cidx; + ri->iri_frags[i].irf_len = plen; + if (++cidx == vsi->shared->isc_ntxd[0]) + cidx = 0; + i++; + /* even a 16K packet shouldn't consume more than 8 clusters */ + MPASS(i < 9); + } while (!eop); + + /* capture data for dynamic ITR adjustment */ + // TODO: Figure out why these are repeated... + rxr->packets++; + rxr->rx_packets++; + + if ((vsi->ifp->if_capenable & IFCAP_RXCSUM) != 0) + ixl_rx_checksum(ri, status, error, ptype); + ri->iri_flowid = le32toh(cur->wb.qword0.hi_dword.rss); + ri->iri_rsstype = ixl_ptype_to_hash(ptype); + ri->iri_vtag = vtag; + ri->iri_nfrags = i; + if (vtag) + ri->iri_flags |= M_VLANTAG; + return (0); } - /********************************************************************* * * Verify that the hardware indicated that the checksum is valid. @@ -1766,16 +666,15 @@ * *********************************************************************/ static void -ixl_rx_checksum(struct mbuf * mp, u32 status, u32 error, u8 ptype) +ixl_rx_checksum(if_rxd_info_t ri, u32 status, u32 error, u8 ptype) { struct i40e_rx_ptype_decoded decoded; decoded = decode_rx_desc_ptype(ptype); - /* Errors? */ if (error & ((1 << I40E_RX_DESC_ERROR_IPE_SHIFT) | (1 << I40E_RX_DESC_ERROR_L4E_SHIFT))) { - mp->m_pkthdr.csum_flags = 0; + ri->iri_csum_flags = 0; return; } @@ -1784,60 +683,17 @@ decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6) if (status & (1 << I40E_RX_DESC_STATUS_IPV6EXADD_SHIFT)) { - mp->m_pkthdr.csum_flags = 0; + ri->iri_csum_flags = 0; return; } - /* IP Checksum Good */ - mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED; - mp->m_pkthdr.csum_flags |= CSUM_IP_VALID; + ri->iri_csum_flags = CSUM_IP_CHECKED; + ri->iri_csum_flags |= CSUM_IP_VALID; if (status & (1 << I40E_RX_DESC_STATUS_L3L4P_SHIFT)) { - mp->m_pkthdr.csum_flags |= + ri->iri_csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); - mp->m_pkthdr.csum_data |= htons(0xffff); + ri->iri_csum_data |= htons(0xffff); } - return; } - -#if __FreeBSD_version >= 1100000 -uint64_t -ixl_get_counter(if_t ifp, ift_counter cnt) -{ - struct ixl_vsi *vsi; - - vsi = if_getsoftc(ifp); - - switch (cnt) { - case IFCOUNTER_IPACKETS: - return (vsi->ipackets); - case IFCOUNTER_IERRORS: - return (vsi->ierrors); - case IFCOUNTER_OPACKETS: - return (vsi->opackets); - case IFCOUNTER_OERRORS: - return (vsi->oerrors); - case IFCOUNTER_COLLISIONS: - /* Collisions are by standard impossible in 40G/10G Ethernet */ - return (0); - case IFCOUNTER_IBYTES: - return (vsi->ibytes); - case IFCOUNTER_OBYTES: - return (vsi->obytes); - case IFCOUNTER_IMCASTS: - return (vsi->imcasts); - case IFCOUNTER_OMCASTS: - return (vsi->omcasts); - case IFCOUNTER_IQDROPS: - return (vsi->iqdrops); - case IFCOUNTER_OQDROPS: - return (vsi->oqdrops); - case IFCOUNTER_NOPROTO: - return (vsi->noproto); - default: - return (if_get_counter_default(ifp, cnt)); - } -} -#endif - Index: sys/dev/ixl/ixlv.h =================================================================== --- sys/dev/ixl/ixlv.h +++ sys/dev/ixl/ixlv.h @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2013-2015, Intel Corporation + Copyright (c) 2013-2017, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -38,8 +38,7 @@ #include "ixlv_vc_mgr.h" -#define IXLV_AQ_MAX_ERR 30 -#define IXLV_MAX_INIT_WAIT 120 +#define IXLV_AQ_MAX_ERR 200 #define IXLV_MAX_FILTERS 128 #define IXLV_MAX_QUEUES 16 #define IXLV_AQ_TIMEOUT (1 * hz) @@ -79,6 +78,8 @@ "\23I40E_VIRTCHNL_VF_OFFLOAD_RSS_PCTYPE_V2" \ "\24I40E_VIRTCHNL_VF_OFFLOAD_RSS_PF" +static MALLOC_DEFINE(M_IXLV, "ixlv", "ixlv driver allocations"); + /* Driver state */ enum ixlv_state_t { IXLV_START, @@ -144,10 +145,10 @@ u32 qbase; u32 admvec; struct timeout_task timeout; +#ifdef notyet struct task aq_irq; struct task aq_sched; - struct taskqueue *tq; - +#endif struct ixl_vsi vsi; /* Filter lists */ @@ -186,7 +187,6 @@ u8 aq_buffer[IXL_AQ_BUF_SZ]; }; -#define IXLV_CORE_LOCK_ASSERT(sc) mtx_assert(&(sc)->mtx, MA_OWNED) /* ** This checks for a zero mac addr, something that will be likely ** unless the Admin on the Host has created one. @@ -205,6 +205,8 @@ /* ** VF Common function prototypes */ +void ixlv_if_init(if_ctx_t ctx); + int ixlv_send_api_ver(struct ixlv_sc *); int ixlv_verify_api_ver(struct ixlv_sc *); int ixlv_send_vf_config_msg(struct ixlv_sc *); Index: sys/dev/ixl/ixlv_vc_mgr.h =================================================================== --- sys/dev/ixl/ixlv_vc_mgr.h +++ sys/dev/ixl/ixlv_vc_mgr.h @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2013-2015, Intel Corporation + Copyright (c) 2013-2017, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without Index: sys/dev/ixl/ixlvc.c =================================================================== --- sys/dev/ixl/ixlvc.c +++ sys/dev/ixl/ixlvc.c @@ -1,6 +1,6 @@ /****************************************************************************** - Copyright (c) 2013-2015, Intel Corporation + Copyright (c) 2013-2017, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -178,11 +178,8 @@ err = i40e_aq_send_msg_to_pf(hw, op, I40E_SUCCESS, msg, len, NULL); if (err) - device_printf(dev, "Unable to send opcode %s to PF, " - "status %s, aq error %s\n", - ixl_vc_opcode_str(op), - i40e_stat_str(hw, err), - i40e_aq_str(hw, hw->aq.asq_last_status)); + device_printf(dev, "Unable to send opcode %d to PF, " + "error %d, aq status %d\n", op, err, hw->aq.asq_last_status); return err; } @@ -386,7 +383,9 @@ { device_t dev = sc->dev; struct ixl_vsi *vsi = &sc->vsi; - struct ixl_queue *que = vsi->queues; + if_softc_ctx_t scctx = iflib_get_softc_ctx(vsi->ctx); + struct ixl_tx_queue *tx_que = vsi->tx_queues; + struct ixl_rx_queue *rx_que = vsi->rx_queues; struct tx_ring *txr; struct rx_ring *rxr; int len, pairs; @@ -394,7 +393,9 @@ struct i40e_virtchnl_vsi_queue_config_info *vqci; struct i40e_virtchnl_queue_pair_info *vqpi; - pairs = vsi->num_queues; + /* XXX: Linux PF driver wants matching ids in each tx/rx struct, so both TX/RX + * queues of a pair need to be configured */ + pairs = max(vsi->num_tx_queues, vsi->num_rx_queues); len = sizeof(struct i40e_virtchnl_vsi_queue_config_info) + (sizeof(struct i40e_virtchnl_queue_pair_info) * pairs); vqci = malloc(len, M_DEVBUF, M_NOWAIT | M_ZERO); @@ -409,23 +410,25 @@ /* Size check is not needed here - HW max is 16 queue pairs, and we * can fit info for 31 of them into the AQ buffer before it overflows. */ - for (int i = 0; i < pairs; i++, que++, vqpi++) { - txr = &que->txr; - rxr = &que->rxr; + for (int i = 0; i < pairs; i++, tx_que++, rx_que++, vqpi++) { + txr = &tx_que->txr; + rxr = &rx_que->rxr; + vqpi->txq.vsi_id = vqci->vsi_id; vqpi->txq.queue_id = i; - vqpi->txq.ring_len = que->num_desc; - vqpi->txq.dma_ring_addr = txr->dma.pa; + vqpi->txq.ring_len = scctx->isc_ntxd[0]; + vqpi->txq.dma_ring_addr = txr->tx_paddr; /* Enable Head writeback */ vqpi->txq.headwb_enabled = 1; - vqpi->txq.dma_headwb_addr = txr->dma.pa + - (que->num_desc * sizeof(struct i40e_tx_desc)); + vqpi->txq.dma_headwb_addr = txr->tx_paddr + + (scctx->isc_ntxd[0] * sizeof(struct i40e_tx_desc)); vqpi->rxq.vsi_id = vqci->vsi_id; vqpi->rxq.queue_id = i; - vqpi->rxq.ring_len = que->num_desc; - vqpi->rxq.dma_ring_addr = rxr->dma.pa; - vqpi->rxq.max_pkt_size = vsi->max_frame_size; + vqpi->rxq.ring_len = scctx->isc_nrxd[0]; + vqpi->rxq.dma_ring_addr = rxr->rx_paddr; + vqpi->rxq.max_pkt_size = scctx->isc_max_frame_size; + // TODO: Get this value from iflib, somehow vqpi->rxq.databuffer_size = rxr->mbuf_sz; vqpi->rxq.splithdr_enabled = 0; } @@ -446,6 +449,8 @@ struct i40e_virtchnl_queue_select vqs; vqs.vsi_id = sc->vsi_res->vsi_id; + /* XXX: In Linux PF, as long as neither of these is 0, + * every queue in VF VSI is enabled. */ vqs.tx_queues = (1 << sc->vsi_res->num_queue_pairs) - 1; vqs.rx_queues = vqs.tx_queues; ixlv_send_pf_msg(sc, I40E_VIRTCHNL_OP_ENABLE_QUEUES, @@ -463,6 +468,8 @@ struct i40e_virtchnl_queue_select vqs; vqs.vsi_id = sc->vsi_res->vsi_id; + /* XXX: In Linux PF, as long as neither of these is 0, + * every queue in VF VSI is disabled. */ vqs.tx_queues = (1 << sc->vsi_res->num_queue_pairs) - 1; vqs.rx_queues = vqs.tx_queues; ixlv_send_pf_msg(sc, I40E_VIRTCHNL_OP_DISABLE_QUEUES, @@ -474,6 +481,8 @@ ** ** Request that the PF map queues to interrupt vectors. Misc causes, including ** admin queue, are always mapped to vector 0. +** +** XXX: In iflib, only RX queues are mapped to HW interrupts */ void ixlv_map_queues(struct ixlv_sc *sc) @@ -481,27 +490,32 @@ struct i40e_virtchnl_irq_map_info *vm; int i, q, len; struct ixl_vsi *vsi = &sc->vsi; - struct ixl_queue *que = vsi->queues; + struct ixl_rx_queue *rx_que = vsi->rx_queues; + if_softc_ctx_t scctx = vsi->shared; + device_t dev = sc->dev; + + // XXX: What happens if we only get 1 MSI-X vector? + MPASS(scctx->isc_vectors > 1); /* How many queue vectors, adminq uses one */ - q = sc->msix - 1; + // XXX: How do we know how many interrupt vectors we have? + q = scctx->isc_vectors - 1; len = sizeof(struct i40e_virtchnl_irq_map_info) + - (sc->msix * sizeof(struct i40e_virtchnl_vector_map)); + (scctx->isc_vectors * sizeof(struct i40e_virtchnl_vector_map)); vm = malloc(len, M_DEVBUF, M_NOWAIT); if (!vm) { - printf("%s: unable to allocate memory\n", __func__); + device_printf(dev, "%s: unable to allocate memory\n", __func__); ixl_vc_schedule_retry(&sc->vc_mgr); return; } - vm->num_vectors = sc->msix; + vm->num_vectors = scctx->isc_vectors; /* Queue vectors first */ - for (i = 0; i < q; i++, que++) { + for (i = 0; i < q; i++, rx_que++) { vm->vecmap[i].vsi_id = sc->vsi_res->vsi_id; vm->vecmap[i].vector_id = i + 1; /* first is adminq */ - vm->vecmap[i].txq_map = (1 << que->me); - vm->vecmap[i].rxq_map = (1 << que->me); + vm->vecmap[i].rxq_map = (1 << rx_que->rxr.me); vm->vecmap[i].rxitr_idx = 0; vm->vecmap[i].txitr_idx = 1; } @@ -809,8 +823,10 @@ uint64_t tx_discards; tx_discards = es->tx_discards; +#if 0 for (int i = 0; i < vsi->num_queues; i++) tx_discards += sc->vsi.queues[i].txr.br->br_drops; +#endif /* Update ifnet stats */ IXL_SET_IPACKETS(vsi, es->rx_unicast + @@ -873,8 +889,12 @@ ixlv_set_rss_hena(struct ixlv_sc *sc) { struct i40e_virtchnl_rss_hena hena; + struct i40e_hw *hw = &sc->hw; - hena.hena = IXL_DEFAULT_RSS_HENA_X722; + if (hw->mac.type == I40E_MAC_X722_VF) + hena.hena = IXL_DEFAULT_RSS_HENA_X722; + else + hena.hena = IXL_DEFAULT_RSS_HENA_XL710; ixlv_send_pf_msg(sc, I40E_VIRTCHNL_OP_SET_RSS_HENA, (u8 *)&hena, sizeof(hena)); @@ -910,9 +930,9 @@ * num_queues.) */ que_id = rss_get_indirection_to_bucket(i); - que_id = que_id % sc->vsi.num_queues; + que_id = que_id % sc->vsi.num_rx_queues; #else - que_id = i % sc->vsi.num_queues; + que_id = i % sc->vsi.num_rx_queues; #endif lut = que_id & IXL_RSS_VSI_LUT_ENTRY_MASK; rss_lut_msg->lut[i] = lut; @@ -959,9 +979,9 @@ case I40E_VIRTCHNL_EVENT_RESET_IMPENDING: device_printf(dev, "PF initiated reset!\n"); sc->init_state = IXLV_RESET_PENDING; - mtx_unlock(&sc->mtx); - ixlv_init(vsi); - mtx_lock(&sc->mtx); + // mtx_unlock(&sc->mtx); + ixlv_if_init(sc->vsi.ctx); + // mtx_lock(&sc->mtx); break; default: device_printf(dev, "%s: Unknown event %d from AQ\n", @@ -975,8 +995,8 @@ /* Catch-all error response */ if (v_retval) { device_printf(dev, - "%s: AQ returned error %s to our request %s!\n", - __func__, i40e_stat_str(&sc->hw, v_retval), ixl_vc_opcode_str(v_opcode)); + "%s: AQ returned error %d to our request %d!\n", + __func__, v_retval, v_opcode); } #ifdef IXL_DEBUG @@ -1021,7 +1041,7 @@ /* Turn on all interrupts */ ixlv_enable_intr(vsi); /* And inform the stack we're ready */ - vsi->ifp->if_drv_flags |= IFF_DRV_RUNNING; + // vsi->ifp->if_drv_flags |= IFF_DRV_RUNNING; /* TODO: Clear a state flag, so we know we're ready to run init again */ } break; @@ -1058,8 +1078,8 @@ default: #ifdef IXL_DEBUG device_printf(dev, - "%s: Received unexpected message %s from PF.\n", - __func__, ixl_vc_opcode_str(v_opcode)); + "%s: Received unexpected message %d from PF.\n", + __func__, v_opcode); #endif break; } @@ -1158,7 +1178,6 @@ { struct ixl_vc_mgr *mgr = (struct ixl_vc_mgr *)arg; - IXLV_CORE_LOCK_ASSERT(mgr->sc); ixl_vc_process_completion(mgr, I40E_ERR_TIMEOUT); } @@ -1167,7 +1186,6 @@ { struct ixl_vc_mgr *mgr = (struct ixl_vc_mgr *)arg; - IXLV_CORE_LOCK_ASSERT(mgr->sc); ixl_vc_send_current(mgr); } @@ -1210,7 +1228,7 @@ ixl_vc_enqueue(struct ixl_vc_mgr *mgr, struct ixl_vc_cmd *cmd, uint32_t req, ixl_vc_callback_t *callback, void *arg) { - IXLV_CORE_LOCK_ASSERT(mgr->sc); + // IXLV_CORE_LOCK_ASSERT(mgr->sc); if (cmd->flags & IXLV_VC_CMD_FLAG_BUSY) { if (mgr->current == cmd) @@ -1233,7 +1251,7 @@ { struct ixl_vc_cmd *cmd; - IXLV_CORE_LOCK_ASSERT(mgr->sc); + // IXLV_CORE_LOCK_ASSERT(mgr->sc); KASSERT(TAILQ_EMPTY(&mgr->pending) || mgr->current != NULL, ("ixlv: pending commands waiting but no command in progress")); Index: sys/modules/Makefile =================================================================== --- sys/modules/Makefile +++ sys/modules/Makefile @@ -196,7 +196,6 @@ ${_ixv} \ ${_ixgb} \ ${_ixl} \ - ${_ixlv} \ jme \ joy \ kbdmux \ Index: sys/modules/ixl/Makefile =================================================================== --- sys/modules/ixl/Makefile +++ sys/modules/ixl/Makefile @@ -3,8 +3,8 @@ .PATH: ${SRCTOP}/sys/dev/ixl KMOD = if_ixl -SRCS = device_if.h bus_if.h pci_if.h pci_iov_if.h -SRCS += opt_inet.h opt_inet6.h opt_rss.h opt_ixl.h +SRCS = device_if.h bus_if.h pci_if.h pci_iov_if.h ifdi_if.h +SRCS += opt_inet.h opt_inet6.h opt_rss.h opt_ixl.h opt_iflib.h SRCS += if_ixl.c ixl_pf_main.c ixl_pf_qmgr.c ixl_txrx.c ixl_pf_i2c.c i40e_osdep.c SRCS += ixl_pf_iov.c ixl_iw.c Index: sys/modules/ixlv/Makefile =================================================================== --- sys/modules/ixlv/Makefile +++ sys/modules/ixlv/Makefile @@ -3,8 +3,8 @@ .PATH: ${SRCTOP}/sys/dev/ixl KMOD = if_ixlv -SRCS = device_if.h bus_if.h pci_if.h -SRCS += opt_inet.h opt_inet6.h opt_rss.h opt_ixl.h +SRCS = device_if.h bus_if.h pci_if.h ifdi_if.h +SRCS += opt_inet.h opt_inet6.h opt_rss.h opt_ixl.h opt_iflib.h SRCS += if_ixlv.c ixlvc.c ixl_txrx.c i40e_osdep.c # Shared source Index: sys/net/iflib.h =================================================================== --- sys/net/iflib.h +++ sys/net/iflib.h @@ -327,6 +327,9 @@ void iflib_set_mac(if_ctx_t ctx, uint8_t mac[ETHER_ADDR_LEN]); +void iflib_init_locked(if_ctx_t ctx); +void iflib_stop(if_ctx_t ctx); + /* * If the driver can plug cleanly in to newbus use these */ Index: sys/net/iflib.c =================================================================== --- sys/net/iflib.c +++ sys/net/iflib.c @@ -699,12 +699,12 @@ static void iflib_txq_check_drain(iflib_txq_t txq, int budget); static uint32_t iflib_txq_can_drain(struct ifmp_ring *); static int iflib_register(if_ctx_t); -static void iflib_init_locked(if_ctx_t ctx); +//static void iflib_init_locked(if_ctx_t ctx); static void iflib_add_device_sysctl_pre(if_ctx_t ctx); static void iflib_add_device_sysctl_post(if_ctx_t ctx); static void iflib_ifmp_purge(iflib_txq_t txq); static void _iflib_pre_assert(if_softc_ctx_t scctx); -static void iflib_stop(if_ctx_t ctx); +//static void iflib_stop(if_ctx_t ctx); static void iflib_if_init_locked(if_ctx_t ctx); #ifndef __NO_STRICT_ALIGNMENT static struct mbuf * iflib_fixup_rx(struct mbuf *m); @@ -2126,7 +2126,7 @@ CTX_UNLOCK(ctx); } -static void +void iflib_init_locked(if_ctx_t ctx) { if_softc_ctx_t sctx = &ctx->ifc_softc_ctx; @@ -2180,7 +2180,7 @@ } } } - done: +done: if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); IFDI_INTR_ENABLE(ctx); txq = ctx->ifc_txqs; @@ -2213,7 +2213,10 @@ CTX_UNLOCK(ctx); } +#if 0 static void +#endif +void iflib_stop(if_ctx_t ctx) { iflib_txq_t txq = ctx->ifc_txqs; @@ -3726,7 +3729,7 @@ */ if (avoid_reset) { if_setflagbits(ifp, IFF_UP,0); - if (!(if_getdrvflags(ifp)& IFF_DRV_RUNNING)) + if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) reinit = 1; #ifdef INET if (!(if_getflags(ifp) & IFF_NOARP)) @@ -3821,7 +3824,7 @@ #endif setmask |= (mask & IFCAP_FLAGS); - if (setmask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) + if (setmask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) setmask |= (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6); if ((mask & IFCAP_WOL) && (if_getcapabilities(ifp) & IFCAP_WOL) != 0) @@ -3842,7 +3845,7 @@ CTX_UNLOCK(ctx); } break; - } + } case SIOCGPRIVATE_0: case SIOCSDRVSPEC: case SIOCGDRVSPEC: @@ -3986,7 +3989,6 @@ uint16_t main_txq; uint16_t main_rxq; - ctx = malloc(sizeof(* ctx), M_IFLIB, M_WAITOK|M_ZERO); if (sc == NULL) { @@ -5187,6 +5189,8 @@ if (scctx->isc_disable_msix) goto msi; +// Don't need to set busmaster here... +#if 0 /* ** When used in a virtualized environment ** PCI BUSMASTER capability may not be set @@ -5211,6 +5215,7 @@ goto msi; } } +#endif /* * bar == -1 => "trust me I know what I'm doing"