Index: sys/dev/ixl/i40e_osdep.c =================================================================== --- sys/dev/ixl/i40e_osdep.c +++ sys/dev/ixl/i40e_osdep.c @@ -51,14 +51,14 @@ i40e_status i40e_allocate_virt_mem(struct i40e_hw *hw, struct i40e_virt_mem *mem, u32 size) { - mem->va = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO); + mem->va = malloc(size, M_IXL, M_NOWAIT | M_ZERO); return(mem->va == NULL); } i40e_status i40e_free_virt_mem(struct i40e_hw *hw, struct i40e_virt_mem *mem) { - free(mem->va, M_DEVBUF); + free(mem->va, M_IXL); return(0); } Index: sys/dev/ixl/if_ixl.c =================================================================== --- sys/dev/ixl/if_ixl.c +++ sys/dev/ixl/if_ixl.c @@ -1,6721 +1,8 @@ -/****************************************************************************** - Copyright (c) 2013-2015, Intel Corporation - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - 3. Neither the name of the Intel Corporation nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. +#include "opt_iflib.h" -******************************************************************************/ -/*$FreeBSD$*/ - -#ifndef IXL_STANDALONE_BUILD -#include "opt_inet.h" -#include "opt_inet6.h" -#include "opt_rss.h" -#endif - -#include "ixl.h" -#include "ixl_pf.h" - -#ifdef RSS -#include -#endif - -/********************************************************************* - * Driver version - *********************************************************************/ -char ixl_driver_version[] = "1.4.3"; - -/********************************************************************* - * PCI Device ID Table - * - * Used by probe to select devices to load on - * Last field stores an index into ixl_strings - * Last entry must be all 0s - * - * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } - *********************************************************************/ - -static ixl_vendor_info_t ixl_vendor_info_array[] = -{ - {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_XL710, 0, 0, 0}, - {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_B, 0, 0, 0}, - {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_C, 0, 0, 0}, - {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_A, 0, 0, 0}, - {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_B, 0, 0, 0}, - {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_C, 0, 0, 0}, - {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T, 0, 0, 0}, - {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T4, 0, 0, 0}, -#ifdef X722_SUPPORT - {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_X722, 0, 0, 0}, - {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_1G_BASE_T_X722, 0, 0, 0}, - {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T_X722, 0, 0, 0}, -#endif - /* required last entry */ - {0, 0, 0, 0, 0} -}; - -/********************************************************************* - * Table of branding strings - *********************************************************************/ - -static char *ixl_strings[] = { - "Intel(R) Ethernet Connection XL710 Driver" -}; - - -/********************************************************************* - * Function prototypes - *********************************************************************/ -static int ixl_probe(device_t); -static int ixl_attach(device_t); -static int ixl_detach(device_t); -static int ixl_shutdown(device_t); -static int ixl_get_hw_capabilities(struct ixl_pf *); -static void ixl_cap_txcsum_tso(struct ixl_vsi *, struct ifnet *, int); -static int ixl_ioctl(struct ifnet *, u_long, caddr_t); -static void ixl_init(void *); -static void ixl_init_locked(struct ixl_pf *); -static void ixl_stop(struct ixl_pf *); -static void ixl_media_status(struct ifnet *, struct ifmediareq *); -static int ixl_media_change(struct ifnet *); -static void ixl_update_link_status(struct ixl_pf *); -static int ixl_allocate_pci_resources(struct ixl_pf *); -static u16 ixl_get_bus_info(struct i40e_hw *, device_t); -static int ixl_setup_stations(struct ixl_pf *); -static int ixl_switch_config(struct ixl_pf *); -static int ixl_initialize_vsi(struct ixl_vsi *); -static int ixl_assign_vsi_msix(struct ixl_pf *); -static int ixl_assign_vsi_legacy(struct ixl_pf *); -static int ixl_init_msix(struct ixl_pf *); -static void ixl_configure_msix(struct ixl_pf *); -static void ixl_configure_itr(struct ixl_pf *); -static void ixl_configure_legacy(struct ixl_pf *); -static void ixl_init_taskqueues(struct ixl_pf *); -static void ixl_free_taskqueues(struct ixl_pf *); -static void ixl_free_pci_resources(struct ixl_pf *); -static void ixl_local_timer(void *); -static int ixl_setup_interface(device_t, struct ixl_vsi *); -static void ixl_link_event(struct ixl_pf *, struct i40e_arq_event_info *); -static void ixl_config_rss(struct ixl_vsi *); -static void ixl_set_queue_rx_itr(struct ixl_queue *); -static void ixl_set_queue_tx_itr(struct ixl_queue *); -static int ixl_set_advertised_speeds(struct ixl_pf *, int); - -static int ixl_enable_rings(struct ixl_vsi *); -static int ixl_disable_rings(struct ixl_vsi *); -static void ixl_enable_intr(struct ixl_vsi *); -static void ixl_disable_intr(struct ixl_vsi *); -static void ixl_disable_rings_intr(struct ixl_vsi *); - -static void ixl_enable_adminq(struct i40e_hw *); -static void ixl_disable_adminq(struct i40e_hw *); -static void ixl_enable_queue(struct i40e_hw *, int); -static void ixl_disable_queue(struct i40e_hw *, int); -static void ixl_enable_legacy(struct i40e_hw *); -static void ixl_disable_legacy(struct i40e_hw *); - -static void ixl_set_promisc(struct ixl_vsi *); -static void ixl_add_multi(struct ixl_vsi *); -static void ixl_del_multi(struct ixl_vsi *); -static void ixl_register_vlan(void *, struct ifnet *, u16); -static void ixl_unregister_vlan(void *, struct ifnet *, u16); -static void ixl_setup_vlan_filters(struct ixl_vsi *); - -static void ixl_init_filters(struct ixl_vsi *); -static void ixl_reconfigure_filters(struct ixl_vsi *vsi); -static void ixl_add_filter(struct ixl_vsi *, u8 *, s16 vlan); -static void ixl_del_filter(struct ixl_vsi *, u8 *, s16 vlan); -static void ixl_add_hw_filters(struct ixl_vsi *, int, int); -static void ixl_del_hw_filters(struct ixl_vsi *, int); -static struct ixl_mac_filter * - ixl_find_filter(struct ixl_vsi *, u8 *, s16); -static void ixl_add_mc_filter(struct ixl_vsi *, u8 *); -static void ixl_free_mac_filters(struct ixl_vsi *vsi); - - -/* Sysctl debug interface */ -#ifdef IXL_DEBUG_SYSCTL -static int ixl_debug_info(SYSCTL_HANDLER_ARGS); -static void ixl_print_debug_info(struct ixl_pf *); -#endif - -/* The MSI/X Interrupt handlers */ -static void ixl_intr(void *); -static void ixl_msix_que(void *); -static void ixl_msix_adminq(void *); -static void ixl_handle_mdd_event(struct ixl_pf *); - -/* Deferred interrupt tasklets */ -static void ixl_do_adminq(void *, int); - -/* Sysctl handlers */ -static int ixl_set_flowcntl(SYSCTL_HANDLER_ARGS); -static int ixl_set_advertise(SYSCTL_HANDLER_ARGS); -static int ixl_current_speed(SYSCTL_HANDLER_ARGS); -static int ixl_sysctl_show_fw(SYSCTL_HANDLER_ARGS); - -/* Statistics */ -static void ixl_add_hw_stats(struct ixl_pf *); -static void ixl_add_sysctls_mac_stats(struct sysctl_ctx_list *, - struct sysctl_oid_list *, struct i40e_hw_port_stats *); -static void ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *, - struct sysctl_oid_list *, - struct i40e_eth_stats *); -static void ixl_update_stats_counters(struct ixl_pf *); -static void ixl_update_eth_stats(struct ixl_vsi *); -static void ixl_update_vsi_stats(struct ixl_vsi *); -static void ixl_pf_reset_stats(struct ixl_pf *); -static void ixl_vsi_reset_stats(struct ixl_vsi *); -static void ixl_stat_update48(struct i40e_hw *, u32, u32, bool, - u64 *, u64 *); -static void ixl_stat_update32(struct i40e_hw *, u32, bool, - u64 *, u64 *); - -#ifdef IXL_DEBUG_SYSCTL -static int ixl_sysctl_link_status(SYSCTL_HANDLER_ARGS); -static int ixl_sysctl_phy_abilities(SYSCTL_HANDLER_ARGS); -static int ixl_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS); -static int ixl_sysctl_hw_res_alloc(SYSCTL_HANDLER_ARGS); -static int ixl_sysctl_switch_config(SYSCTL_HANDLER_ARGS); -#endif - -#ifdef PCI_IOV -static int ixl_adminq_err_to_errno(enum i40e_admin_queue_err err); - -static int ixl_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t*); -static void ixl_iov_uninit(device_t dev); -static int ixl_add_vf(device_t dev, uint16_t vfnum, const nvlist_t*); - -static void ixl_handle_vf_msg(struct ixl_pf *, - struct i40e_arq_event_info *); -static void ixl_handle_vflr(void *arg, int pending); - -static void ixl_reset_vf(struct ixl_pf *pf, struct ixl_vf *vf); -static void ixl_reinit_vf(struct ixl_pf *pf, struct ixl_vf *vf); -#endif - -/********************************************************************* - * FreeBSD Device Interface Entry Points - *********************************************************************/ - -static device_method_t ixl_methods[] = { - /* Device interface */ - DEVMETHOD(device_probe, ixl_probe), - DEVMETHOD(device_attach, ixl_attach), - DEVMETHOD(device_detach, ixl_detach), - DEVMETHOD(device_shutdown, ixl_shutdown), -#ifdef PCI_IOV - DEVMETHOD(pci_iov_init, ixl_iov_init), - DEVMETHOD(pci_iov_uninit, ixl_iov_uninit), - DEVMETHOD(pci_iov_add_vf, ixl_add_vf), -#endif - {0, 0} -}; - -static driver_t ixl_driver = { - "ixl", ixl_methods, sizeof(struct ixl_pf), -}; - -devclass_t ixl_devclass; -DRIVER_MODULE(ixl, pci, ixl_driver, ixl_devclass, 0, 0); - -MODULE_DEPEND(ixl, pci, 1, 1, 1); -MODULE_DEPEND(ixl, ether, 1, 1, 1); -#ifdef DEV_NETMAP -MODULE_DEPEND(ixl, netmap, 1, 1, 1); -#endif /* DEV_NETMAP */ - -/* -** Global reset mutex -*/ -static struct mtx ixl_reset_mtx; - -/* -** TUNEABLE PARAMETERS: -*/ - -static SYSCTL_NODE(_hw, OID_AUTO, ixl, CTLFLAG_RD, 0, - "IXL driver parameters"); - -/* - * MSIX should be the default for best performance, - * but this allows it to be forced off for testing. - */ -static int ixl_enable_msix = 1; -TUNABLE_INT("hw.ixl.enable_msix", &ixl_enable_msix); -SYSCTL_INT(_hw_ixl, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &ixl_enable_msix, 0, - "Enable MSI-X interrupts"); - -/* -** Number of descriptors per ring: -** - TX and RX are the same size -*/ -static int ixl_ringsz = DEFAULT_RING; -TUNABLE_INT("hw.ixl.ringsz", &ixl_ringsz); -SYSCTL_INT(_hw_ixl, OID_AUTO, ring_size, CTLFLAG_RDTUN, - &ixl_ringsz, 0, "Descriptor Ring Size"); - -/* -** This can be set manually, if left as 0 the -** number of queues will be calculated based -** on cpus and msix vectors available. -*/ -int ixl_max_queues = 0; -TUNABLE_INT("hw.ixl.max_queues", &ixl_max_queues); -SYSCTL_INT(_hw_ixl, OID_AUTO, max_queues, CTLFLAG_RDTUN, - &ixl_max_queues, 0, "Number of Queues"); - -/* -** Controls for Interrupt Throttling -** - true/false for dynamic adjustment -** - default values for static ITR -*/ -int ixl_dynamic_rx_itr = 0; -TUNABLE_INT("hw.ixl.dynamic_rx_itr", &ixl_dynamic_rx_itr); -SYSCTL_INT(_hw_ixl, OID_AUTO, dynamic_rx_itr, CTLFLAG_RDTUN, - &ixl_dynamic_rx_itr, 0, "Dynamic RX Interrupt Rate"); - -int ixl_dynamic_tx_itr = 0; -TUNABLE_INT("hw.ixl.dynamic_tx_itr", &ixl_dynamic_tx_itr); -SYSCTL_INT(_hw_ixl, OID_AUTO, dynamic_tx_itr, CTLFLAG_RDTUN, - &ixl_dynamic_tx_itr, 0, "Dynamic TX Interrupt Rate"); - -int ixl_rx_itr = IXL_ITR_8K; -TUNABLE_INT("hw.ixl.rx_itr", &ixl_rx_itr); -SYSCTL_INT(_hw_ixl, OID_AUTO, rx_itr, CTLFLAG_RDTUN, - &ixl_rx_itr, 0, "RX Interrupt Rate"); - -int ixl_tx_itr = IXL_ITR_4K; -TUNABLE_INT("hw.ixl.tx_itr", &ixl_tx_itr); -SYSCTL_INT(_hw_ixl, OID_AUTO, tx_itr, CTLFLAG_RDTUN, - &ixl_tx_itr, 0, "TX Interrupt Rate"); - -#ifdef IXL_FDIR -static int ixl_enable_fdir = 1; -TUNABLE_INT("hw.ixl.enable_fdir", &ixl_enable_fdir); -/* Rate at which we sample */ -int ixl_atr_rate = 20; -TUNABLE_INT("hw.ixl.atr_rate", &ixl_atr_rate); -#endif - -#ifdef DEV_NETMAP -#define NETMAP_IXL_MAIN /* only bring in one part of the netmap code */ -#include -#endif /* DEV_NETMAP */ - -static char *ixl_fc_string[6] = { - "None", - "Rx", - "Tx", - "Full", - "Priority", - "Default" -}; - -static MALLOC_DEFINE(M_IXL, "ixl", "ixl driver allocations"); - -static uint8_t ixl_bcast_addr[ETHER_ADDR_LEN] = - {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; - -/********************************************************************* - * Device identification routine - * - * ixl_probe determines if the driver should be loaded on - * the hardware based on PCI vendor/device id of the device. - * - * return BUS_PROBE_DEFAULT on success, positive on failure - *********************************************************************/ - -static int -ixl_probe(device_t dev) -{ - ixl_vendor_info_t *ent; - - u16 pci_vendor_id, pci_device_id; - u16 pci_subvendor_id, pci_subdevice_id; - char device_name[256]; - static bool lock_init = FALSE; - - INIT_DEBUGOUT("ixl_probe: begin"); - - pci_vendor_id = pci_get_vendor(dev); - if (pci_vendor_id != I40E_INTEL_VENDOR_ID) - return (ENXIO); - - pci_device_id = pci_get_device(dev); - pci_subvendor_id = pci_get_subvendor(dev); - pci_subdevice_id = pci_get_subdevice(dev); - - ent = ixl_vendor_info_array; - while (ent->vendor_id != 0) { - if ((pci_vendor_id == ent->vendor_id) && - (pci_device_id == ent->device_id) && - - ((pci_subvendor_id == ent->subvendor_id) || - (ent->subvendor_id == 0)) && - - ((pci_subdevice_id == ent->subdevice_id) || - (ent->subdevice_id == 0))) { - sprintf(device_name, "%s, Version - %s", - ixl_strings[ent->index], - ixl_driver_version); - device_set_desc_copy(dev, device_name); - /* One shot mutex init */ - if (lock_init == FALSE) { - lock_init = TRUE; - mtx_init(&ixl_reset_mtx, - "ixl_reset", - "IXL RESET Lock", MTX_DEF); - } - return (BUS_PROBE_DEFAULT); - } - ent++; - } - return (ENXIO); -} - -/********************************************************************* - * Device initialization routine - * - * The attach entry point is called when the driver is being loaded. - * This routine identifies the type of hardware, allocates all resources - * and initializes the hardware. - * - * return 0 on success, positive on failure - *********************************************************************/ - -static int -ixl_attach(device_t dev) -{ - struct ixl_pf *pf; - struct i40e_hw *hw; - struct ixl_vsi *vsi; - u16 bus; - int error = 0; -#ifdef PCI_IOV - nvlist_t *pf_schema, *vf_schema; - int iov_error; -#endif - - INIT_DEBUGOUT("ixl_attach: begin"); - - /* Allocate, clear, and link in our primary soft structure */ - pf = device_get_softc(dev); - pf->dev = pf->osdep.dev = dev; - hw = &pf->hw; - - /* - ** Note this assumes we have a single embedded VSI, - ** this could be enhanced later to allocate multiple - */ - vsi = &pf->vsi; - vsi->dev = pf->dev; - - /* Core Lock Init*/ - IXL_PF_LOCK_INIT(pf, device_get_nameunit(dev)); - - /* Set up the timer callout */ - callout_init_mtx(&pf->timer, &pf->pf_mtx, 0); - - /* Set up sysctls */ - SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW, - pf, 0, ixl_set_flowcntl, "I", "Flow Control"); - - SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW, - pf, 0, ixl_set_advertise, "I", "Advertised Speed"); - - SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "current_speed", CTLTYPE_STRING | CTLFLAG_RD, - pf, 0, ixl_current_speed, "A", "Current Port Speed"); - - SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "fw_version", CTLTYPE_STRING | CTLFLAG_RD, - pf, 0, ixl_sysctl_show_fw, "A", "Firmware version"); - - SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "rx_itr", CTLFLAG_RW, - &ixl_rx_itr, IXL_ITR_8K, "RX ITR"); - - SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "dynamic_rx_itr", CTLFLAG_RW, - &ixl_dynamic_rx_itr, 0, "Dynamic RX ITR"); - - SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "tx_itr", CTLFLAG_RW, - &ixl_tx_itr, IXL_ITR_4K, "TX ITR"); - - SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "dynamic_tx_itr", CTLFLAG_RW, - &ixl_dynamic_tx_itr, 0, "Dynamic TX ITR"); - -#ifdef IXL_DEBUG_SYSCTL - SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, pf, 0, - ixl_debug_info, "I", "Debug Information"); - - /* Debug shared-code message level */ - SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "debug_mask", CTLFLAG_RW, - &pf->hw.debug_mask, 0, "Debug Message Level"); - - SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "vc_debug_level", CTLFLAG_RW, &pf->vc_debug_lvl, - 0, "PF/VF Virtual Channel debug level"); - - SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "link_status", CTLTYPE_STRING | CTLFLAG_RD, - pf, 0, ixl_sysctl_link_status, "A", "Current Link Status"); - - SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "phy_abilities", CTLTYPE_STRING | CTLFLAG_RD, - pf, 0, ixl_sysctl_phy_abilities, "A", "PHY Abilities"); - - SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "filter_list", CTLTYPE_STRING | CTLFLAG_RD, - pf, 0, ixl_sysctl_sw_filter_list, "A", "SW Filter List"); - - SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "hw_res_alloc", CTLTYPE_STRING | CTLFLAG_RD, - pf, 0, ixl_sysctl_hw_res_alloc, "A", "HW Resource Allocation"); - - SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "switch_config", CTLTYPE_STRING | CTLFLAG_RD, - pf, 0, ixl_sysctl_switch_config, "A", "HW Switch Configuration"); -#endif - - /* Save off the PCI information */ - hw->vendor_id = pci_get_vendor(dev); - hw->device_id = pci_get_device(dev); - hw->revision_id = pci_read_config(dev, PCIR_REVID, 1); - hw->subsystem_vendor_id = - pci_read_config(dev, PCIR_SUBVEND_0, 2); - hw->subsystem_device_id = - pci_read_config(dev, PCIR_SUBDEV_0, 2); - - hw->bus.device = pci_get_slot(dev); - hw->bus.func = pci_get_function(dev); - - pf->vc_debug_lvl = 1; - - /* Do PCI setup - map BAR0, etc */ - if (ixl_allocate_pci_resources(pf)) { - device_printf(dev, "Allocation of PCI resources failed\n"); - error = ENXIO; - goto err_out; - } - - /* Establish a clean starting point */ - i40e_clear_hw(hw); - error = i40e_pf_reset(hw); - if (error) { - device_printf(dev,"PF reset failure %x\n", error); - error = EIO; - goto err_out; - } - - /* Set admin queue parameters */ - hw->aq.num_arq_entries = IXL_AQ_LEN; - hw->aq.num_asq_entries = IXL_AQ_LEN; - hw->aq.arq_buf_size = IXL_AQ_BUFSZ; - hw->aq.asq_buf_size = IXL_AQ_BUFSZ; - - /* Initialize the shared code */ - error = i40e_init_shared_code(hw); - if (error) { - device_printf(dev,"Unable to initialize the shared code\n"); - error = EIO; - goto err_out; - } - - /* Set up the admin queue */ - error = i40e_init_adminq(hw); - if (error) { - device_printf(dev, "The driver for the device stopped " - "because the NVM image is newer than expected.\n" - "You must install the most recent version of " - " the network driver.\n"); - goto err_out; - } - device_printf(dev, "%s\n", ixl_fw_version_str(hw)); - - if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR && - hw->aq.api_min_ver > I40E_FW_API_VERSION_MINOR) - device_printf(dev, "The driver for the device detected " - "a newer version of the NVM image than expected.\n" - "Please install the most recent version of the network driver.\n"); - else if (hw->aq.api_maj_ver < I40E_FW_API_VERSION_MAJOR || - hw->aq.api_min_ver < (I40E_FW_API_VERSION_MINOR - 1)) - device_printf(dev, "The driver for the device detected " - "an older version of the NVM image than expected.\n" - "Please update the NVM image.\n"); - - /* Clear PXE mode */ - i40e_clear_pxe_mode(hw); - - /* Get capabilities from the device */ - error = ixl_get_hw_capabilities(pf); - if (error) { - device_printf(dev, "HW capabilities failure!\n"); - goto err_get_cap; - } - - /* Set up host memory cache */ - error = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, - hw->func_caps.num_rx_qp, 0, 0); - if (error) { - device_printf(dev, "init_lan_hmc failed: %d\n", error); - goto err_get_cap; - } - - error = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); - if (error) { - device_printf(dev, "configure_lan_hmc failed: %d\n", error); - goto err_mac_hmc; - } - - /* Disable LLDP from the firmware */ - i40e_aq_stop_lldp(hw, TRUE, NULL); - - i40e_get_mac_addr(hw, hw->mac.addr); - error = i40e_validate_mac_addr(hw->mac.addr); - if (error) { - device_printf(dev, "validate_mac_addr failed: %d\n", error); - goto err_mac_hmc; - } - bcopy(hw->mac.addr, hw->mac.perm_addr, ETHER_ADDR_LEN); - i40e_get_port_mac_addr(hw, hw->mac.port_addr); - - /* Set up VSI and queues */ - if (ixl_setup_stations(pf) != 0) { - device_printf(dev, "setup stations failed!\n"); - error = ENOMEM; - goto err_mac_hmc; - } - - /* Initialize mac filter list for VSI */ - SLIST_INIT(&vsi->ftl); - - /* Set up interrupt routing here */ - if (pf->msix > 1) - error = ixl_assign_vsi_msix(pf); - else - error = ixl_assign_vsi_legacy(pf); - if (error) - goto err_mac_hmc; - - if (((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver < 33)) || - (hw->aq.fw_maj_ver < 4)) { - i40e_msec_delay(75); - error = i40e_aq_set_link_restart_an(hw, TRUE, NULL); - if (error) - device_printf(dev, "link restart failed, aq_err=%d\n", - pf->hw.aq.asq_last_status); - } - - /* Determine link state */ - i40e_aq_get_link_info(hw, TRUE, NULL, NULL); - i40e_get_link_status(hw, &pf->link_up); - - /* Setup OS specific network interface */ - if (ixl_setup_interface(dev, vsi) != 0) { - device_printf(dev, "interface setup failed!\n"); - error = EIO; - goto err_late; - } - - error = ixl_switch_config(pf); - if (error) { - device_printf(dev, "Initial switch config failed: %d\n", error); - goto err_late; - } - - /* Limit phy interrupts to link and modules failure */ - error = i40e_aq_set_phy_int_mask(hw, ~(I40E_AQ_EVENT_LINK_UPDOWN | - I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL); - if (error) - device_printf(dev, "set phy mask failed: %d\n", error); - - /* Get the bus configuration and set the shared code */ - bus = ixl_get_bus_info(hw, dev); - i40e_set_pci_config_data(hw, bus); - - /* Initialize taskqueues */ - ixl_init_taskqueues(pf); - - /* Initialize statistics */ - ixl_pf_reset_stats(pf); - ixl_update_stats_counters(pf); - ixl_add_hw_stats(pf); - - /* Register for VLAN events */ - vsi->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, - ixl_register_vlan, vsi, EVENTHANDLER_PRI_FIRST); - vsi->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, - ixl_unregister_vlan, vsi, EVENTHANDLER_PRI_FIRST); - -#ifdef PCI_IOV - /* SR-IOV is only supported when MSI-X is in use. */ - if (pf->msix > 1) { - pf_schema = pci_iov_schema_alloc_node(); - vf_schema = pci_iov_schema_alloc_node(); - pci_iov_schema_add_unicast_mac(vf_schema, "mac-addr", 0, NULL); - pci_iov_schema_add_bool(vf_schema, "mac-anti-spoof", - IOV_SCHEMA_HASDEFAULT, TRUE); - pci_iov_schema_add_bool(vf_schema, "allow-set-mac", - IOV_SCHEMA_HASDEFAULT, FALSE); - pci_iov_schema_add_bool(vf_schema, "allow-promisc", - IOV_SCHEMA_HASDEFAULT, FALSE); - - iov_error = pci_iov_attach(dev, pf_schema, vf_schema); - if (iov_error != 0) - device_printf(dev, - "Failed to initialize SR-IOV (error=%d)\n", - iov_error); - } -#endif - -#ifdef DEV_NETMAP - ixl_netmap_attach(vsi); -#endif /* DEV_NETMAP */ - INIT_DEBUGOUT("ixl_attach: end"); - return (0); - -err_late: - if (vsi->ifp != NULL) - if_free(vsi->ifp); -err_mac_hmc: - i40e_shutdown_lan_hmc(hw); -err_get_cap: - i40e_shutdown_adminq(hw); -err_out: - ixl_free_pci_resources(pf); - ixl_free_vsi(vsi); - IXL_PF_LOCK_DESTROY(pf); - return (error); -} - -/********************************************************************* - * Device removal routine - * - * The detach entry point is called when the driver is being removed. - * This routine stops the adapter and deallocates all the resources - * that were allocated for driver operation. - * - * return 0 on success, positive on failure - *********************************************************************/ - -static int -ixl_detach(device_t dev) -{ - struct ixl_pf *pf = device_get_softc(dev); - struct i40e_hw *hw = &pf->hw; - struct ixl_vsi *vsi = &pf->vsi; - i40e_status status; -#ifdef PCI_IOV - int error; -#endif - - INIT_DEBUGOUT("ixl_detach: begin"); - - /* Make sure VLANS are not using driver */ - if (vsi->ifp->if_vlantrunk != NULL) { - device_printf(dev,"Vlan in use, detach first\n"); - return (EBUSY); - } - -#ifdef PCI_IOV - error = pci_iov_detach(dev); - if (error != 0) { - device_printf(dev, "SR-IOV in use; detach first.\n"); - return (error); - } -#endif - - ether_ifdetach(vsi->ifp); - if (vsi->ifp->if_drv_flags & IFF_DRV_RUNNING) { - IXL_PF_LOCK(pf); - ixl_stop(pf); - IXL_PF_UNLOCK(pf); - } - - ixl_free_taskqueues(pf); - - /* Shutdown LAN HMC */ - status = i40e_shutdown_lan_hmc(hw); - if (status) - device_printf(dev, - "Shutdown LAN HMC failed with code %d\n", status); - - /* Shutdown admin queue */ - status = i40e_shutdown_adminq(hw); - if (status) - device_printf(dev, - "Shutdown Admin queue failed with code %d\n", status); - - /* Unregister VLAN events */ - if (vsi->vlan_attach != NULL) - EVENTHANDLER_DEREGISTER(vlan_config, vsi->vlan_attach); - if (vsi->vlan_detach != NULL) - EVENTHANDLER_DEREGISTER(vlan_unconfig, vsi->vlan_detach); - - callout_drain(&pf->timer); -#ifdef DEV_NETMAP - netmap_detach(vsi->ifp); -#endif /* DEV_NETMAP */ - ixl_free_pci_resources(pf); - bus_generic_detach(dev); - if_free(vsi->ifp); - ixl_free_vsi(vsi); - IXL_PF_LOCK_DESTROY(pf); - return (0); -} - -/********************************************************************* - * - * Shutdown entry point - * - **********************************************************************/ - -static int -ixl_shutdown(device_t dev) -{ - struct ixl_pf *pf = device_get_softc(dev); - IXL_PF_LOCK(pf); - ixl_stop(pf); - IXL_PF_UNLOCK(pf); - return (0); -} - - -/********************************************************************* - * - * Get the hardware capabilities - * - **********************************************************************/ - -static int -ixl_get_hw_capabilities(struct ixl_pf *pf) -{ - struct i40e_aqc_list_capabilities_element_resp *buf; - struct i40e_hw *hw = &pf->hw; - device_t dev = pf->dev; - int error, len; - u16 needed; - bool again = TRUE; - - len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp); -retry: - if (!(buf = (struct i40e_aqc_list_capabilities_element_resp *) - malloc(len, M_DEVBUF, M_NOWAIT | M_ZERO))) { - device_printf(dev, "Unable to allocate cap memory\n"); - return (ENOMEM); - } - - /* This populates the hw struct */ - error = i40e_aq_discover_capabilities(hw, buf, len, - &needed, i40e_aqc_opc_list_func_capabilities, NULL); - free(buf, M_DEVBUF); - if ((pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) && - (again == TRUE)) { - /* retry once with a larger buffer */ - again = FALSE; - len = needed; - goto retry; - } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) { - device_printf(dev, "capability discovery failed: %d\n", - pf->hw.aq.asq_last_status); - return (ENODEV); - } - - /* Capture this PF's starting queue pair */ - pf->qbase = hw->func_caps.base_queue; - -#ifdef IXL_DEBUG - device_printf(dev,"pf_id=%d, num_vfs=%d, msix_pf=%d, " - "msix_vf=%d, fd_g=%d, fd_b=%d, tx_qp=%d rx_qp=%d qbase=%d\n", - hw->pf_id, hw->func_caps.num_vfs, - hw->func_caps.num_msix_vectors, - hw->func_caps.num_msix_vectors_vf, - hw->func_caps.fd_filters_guaranteed, - hw->func_caps.fd_filters_best_effort, - hw->func_caps.num_tx_qp, - hw->func_caps.num_rx_qp, - hw->func_caps.base_queue); -#endif - return (error); -} - -static void -ixl_cap_txcsum_tso(struct ixl_vsi *vsi, struct ifnet *ifp, int mask) -{ - device_t dev = vsi->dev; - - /* Enable/disable TXCSUM/TSO4 */ - if (!(ifp->if_capenable & IFCAP_TXCSUM) - && !(ifp->if_capenable & IFCAP_TSO4)) { - if (mask & IFCAP_TXCSUM) { - ifp->if_capenable |= IFCAP_TXCSUM; - /* enable TXCSUM, restore TSO if previously enabled */ - if (vsi->flags & IXL_FLAGS_KEEP_TSO4) { - vsi->flags &= ~IXL_FLAGS_KEEP_TSO4; - ifp->if_capenable |= IFCAP_TSO4; - } - } - else if (mask & IFCAP_TSO4) { - ifp->if_capenable |= (IFCAP_TXCSUM | IFCAP_TSO4); - vsi->flags &= ~IXL_FLAGS_KEEP_TSO4; - device_printf(dev, - "TSO4 requires txcsum, enabling both...\n"); - } - } else if((ifp->if_capenable & IFCAP_TXCSUM) - && !(ifp->if_capenable & IFCAP_TSO4)) { - if (mask & IFCAP_TXCSUM) - ifp->if_capenable &= ~IFCAP_TXCSUM; - else if (mask & IFCAP_TSO4) - ifp->if_capenable |= IFCAP_TSO4; - } else if((ifp->if_capenable & IFCAP_TXCSUM) - && (ifp->if_capenable & IFCAP_TSO4)) { - if (mask & IFCAP_TXCSUM) { - vsi->flags |= IXL_FLAGS_KEEP_TSO4; - ifp->if_capenable &= ~(IFCAP_TXCSUM | IFCAP_TSO4); - device_printf(dev, - "TSO4 requires txcsum, disabling both...\n"); - } else if (mask & IFCAP_TSO4) - ifp->if_capenable &= ~IFCAP_TSO4; - } - - /* Enable/disable TXCSUM_IPV6/TSO6 */ - if (!(ifp->if_capenable & IFCAP_TXCSUM_IPV6) - && !(ifp->if_capenable & IFCAP_TSO6)) { - if (mask & IFCAP_TXCSUM_IPV6) { - ifp->if_capenable |= IFCAP_TXCSUM_IPV6; - if (vsi->flags & IXL_FLAGS_KEEP_TSO6) { - vsi->flags &= ~IXL_FLAGS_KEEP_TSO6; - ifp->if_capenable |= IFCAP_TSO6; - } - } else if (mask & IFCAP_TSO6) { - ifp->if_capenable |= (IFCAP_TXCSUM_IPV6 | IFCAP_TSO6); - vsi->flags &= ~IXL_FLAGS_KEEP_TSO6; - device_printf(dev, - "TSO6 requires txcsum6, enabling both...\n"); - } - } else if((ifp->if_capenable & IFCAP_TXCSUM_IPV6) - && !(ifp->if_capenable & IFCAP_TSO6)) { - if (mask & IFCAP_TXCSUM_IPV6) - ifp->if_capenable &= ~IFCAP_TXCSUM_IPV6; - else if (mask & IFCAP_TSO6) - ifp->if_capenable |= IFCAP_TSO6; - } else if ((ifp->if_capenable & IFCAP_TXCSUM_IPV6) - && (ifp->if_capenable & IFCAP_TSO6)) { - if (mask & IFCAP_TXCSUM_IPV6) { - vsi->flags |= IXL_FLAGS_KEEP_TSO6; - ifp->if_capenable &= ~(IFCAP_TXCSUM_IPV6 | IFCAP_TSO6); - device_printf(dev, - "TSO6 requires txcsum6, disabling both...\n"); - } else if (mask & IFCAP_TSO6) - ifp->if_capenable &= ~IFCAP_TSO6; - } -} - -/********************************************************************* - * Ioctl entry point - * - * ixl_ioctl is called when the user wants to configure the - * interface. - * - * return 0 on success, positive on failure - **********************************************************************/ - -static int -ixl_ioctl(struct ifnet * ifp, u_long command, caddr_t data) -{ - struct ixl_vsi *vsi = ifp->if_softc; - struct ixl_pf *pf = vsi->back; - struct ifreq *ifr = (struct ifreq *) data; -#if defined(INET) || defined(INET6) - struct ifaddr *ifa = (struct ifaddr *)data; - bool avoid_reset = FALSE; -#endif - int error = 0; - - switch (command) { - - case SIOCSIFADDR: -#ifdef INET - if (ifa->ifa_addr->sa_family == AF_INET) - avoid_reset = TRUE; -#endif -#ifdef INET6 - if (ifa->ifa_addr->sa_family == AF_INET6) - avoid_reset = TRUE; -#endif -#if defined(INET) || defined(INET6) - /* - ** Calling init results in link renegotiation, - ** so we avoid doing it when possible. - */ - if (avoid_reset) { - ifp->if_flags |= IFF_UP; - if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) - ixl_init(pf); -#ifdef INET - if (!(ifp->if_flags & IFF_NOARP)) - arp_ifinit(ifp, ifa); -#endif - } else - error = ether_ioctl(ifp, command, data); - break; -#endif - case SIOCSIFMTU: - IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); - if (ifr->ifr_mtu > IXL_MAX_FRAME - - ETHER_HDR_LEN - ETHER_CRC_LEN - ETHER_VLAN_ENCAP_LEN) { - error = EINVAL; - } else { - IXL_PF_LOCK(pf); - ifp->if_mtu = ifr->ifr_mtu; - vsi->max_frame_size = - ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN - + ETHER_VLAN_ENCAP_LEN; - ixl_init_locked(pf); - IXL_PF_UNLOCK(pf); - } - break; - case SIOCSIFFLAGS: - IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); - IXL_PF_LOCK(pf); - if (ifp->if_flags & IFF_UP) { - if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) { - if ((ifp->if_flags ^ pf->if_flags) & - (IFF_PROMISC | IFF_ALLMULTI)) { - ixl_set_promisc(vsi); - } - } else - ixl_init_locked(pf); - } else - if (ifp->if_drv_flags & IFF_DRV_RUNNING) - ixl_stop(pf); - pf->if_flags = ifp->if_flags; - IXL_PF_UNLOCK(pf); - break; - case SIOCADDMULTI: - IOCTL_DEBUGOUT("ioctl: SIOCADDMULTI"); - if (ifp->if_drv_flags & IFF_DRV_RUNNING) { - IXL_PF_LOCK(pf); - ixl_disable_intr(vsi); - ixl_add_multi(vsi); - ixl_enable_intr(vsi); - IXL_PF_UNLOCK(pf); - } - break; - case SIOCDELMULTI: - IOCTL_DEBUGOUT("ioctl: SIOCDELMULTI"); - if (ifp->if_drv_flags & IFF_DRV_RUNNING) { - IXL_PF_LOCK(pf); - ixl_disable_intr(vsi); - ixl_del_multi(vsi); - ixl_enable_intr(vsi); - IXL_PF_UNLOCK(pf); - } - break; - case SIOCSIFMEDIA: - case SIOCGIFMEDIA: -#ifdef IFM_ETH_XTYPE - case SIOCGIFXMEDIA: -#endif - IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); - error = ifmedia_ioctl(ifp, ifr, &vsi->media, command); - break; - case SIOCSIFCAP: - { - int mask = ifr->ifr_reqcap ^ ifp->if_capenable; - IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); - - ixl_cap_txcsum_tso(vsi, ifp, mask); - - if (mask & IFCAP_RXCSUM) - ifp->if_capenable ^= IFCAP_RXCSUM; - if (mask & IFCAP_RXCSUM_IPV6) - ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; - if (mask & IFCAP_LRO) - ifp->if_capenable ^= IFCAP_LRO; - if (mask & IFCAP_VLAN_HWTAGGING) - ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; - if (mask & IFCAP_VLAN_HWFILTER) - ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; - if (mask & IFCAP_VLAN_HWTSO) - ifp->if_capenable ^= IFCAP_VLAN_HWTSO; - if (ifp->if_drv_flags & IFF_DRV_RUNNING) { - IXL_PF_LOCK(pf); - ixl_init_locked(pf); - IXL_PF_UNLOCK(pf); - } - VLAN_CAPABILITIES(ifp); - - break; - } - - default: - IOCTL_DEBUGOUT("ioctl: UNKNOWN (0x%X)\n", (int)command); - error = ether_ioctl(ifp, command, data); - break; - } - - return (error); -} - - -/********************************************************************* - * Init entry point - * - * This routine is used in two ways. It is used by the stack as - * init entry point in network interface structure. It is also used - * by the driver as a hw/sw initialization routine to get to a - * consistent state. - * - * return 0 on success, positive on failure - **********************************************************************/ - -static void -ixl_init_locked(struct ixl_pf *pf) -{ - struct i40e_hw *hw = &pf->hw; - struct ixl_vsi *vsi = &pf->vsi; - struct ifnet *ifp = vsi->ifp; - device_t dev = pf->dev; - struct i40e_filter_control_settings filter; - u8 tmpaddr[ETHER_ADDR_LEN]; - int ret; - - mtx_assert(&pf->pf_mtx, MA_OWNED); - INIT_DEBUGOUT("ixl_init: begin"); - ixl_stop(pf); - - /* Get the latest mac address... User might use a LAA */ - bcopy(IF_LLADDR(vsi->ifp), tmpaddr, - I40E_ETH_LENGTH_OF_ADDRESS); - if (!cmp_etheraddr(hw->mac.addr, tmpaddr) && - (i40e_validate_mac_addr(tmpaddr) == I40E_SUCCESS)) { - ixl_del_filter(vsi, hw->mac.addr, IXL_VLAN_ANY); - bcopy(tmpaddr, hw->mac.addr, - I40E_ETH_LENGTH_OF_ADDRESS); - ret = i40e_aq_mac_address_write(hw, - I40E_AQC_WRITE_TYPE_LAA_ONLY, - hw->mac.addr, NULL); - if (ret) { - device_printf(dev, "LLA address" - "change failed!!\n"); - return; - } else { - ixl_add_filter(vsi, hw->mac.addr, IXL_VLAN_ANY); - } - } - - /* Set the various hardware offload abilities */ - ifp->if_hwassist = 0; - if (ifp->if_capenable & IFCAP_TSO) - ifp->if_hwassist |= CSUM_TSO; - if (ifp->if_capenable & IFCAP_TXCSUM) - ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); - if (ifp->if_capenable & IFCAP_TXCSUM_IPV6) - ifp->if_hwassist |= (CSUM_TCP_IPV6 | CSUM_UDP_IPV6); - - /* Set up the device filtering */ - bzero(&filter, sizeof(filter)); - filter.enable_ethtype = TRUE; - filter.enable_macvlan = TRUE; -#ifdef IXL_FDIR - filter.enable_fdir = TRUE; -#endif - filter.hash_lut_size = I40E_HASH_LUT_SIZE_512; - if (i40e_set_filter_control(hw, &filter)) - device_printf(dev, "set_filter_control() failed\n"); - - /* Set up RSS */ - ixl_config_rss(vsi); - - /* - ** Prepare the VSI: rings, hmc contexts, etc... - */ - if (ixl_initialize_vsi(vsi)) { - device_printf(dev, "initialize vsi failed!!\n"); - return; - } - - /* Add protocol filters to list */ - ixl_init_filters(vsi); - - /* Setup vlan's if needed */ - ixl_setup_vlan_filters(vsi); - - /* Start the local timer */ - callout_reset(&pf->timer, hz, ixl_local_timer, pf); - - /* Set up MSI/X routing and the ITR settings */ - if (ixl_enable_msix) { - ixl_configure_msix(pf); - ixl_configure_itr(pf); - } else - ixl_configure_legacy(pf); - - ixl_enable_rings(vsi); - - i40e_aq_set_default_vsi(hw, vsi->seid, NULL); - - ixl_reconfigure_filters(vsi); - - /* Set MTU in hardware*/ - int aq_error = i40e_aq_set_mac_config(hw, vsi->max_frame_size, - TRUE, 0, NULL); - if (aq_error) - device_printf(vsi->dev, - "aq_set_mac_config in init error, code %d\n", - aq_error); - - /* And now turn on interrupts */ - ixl_enable_intr(vsi); - - /* Now inform the stack we're ready */ - ifp->if_drv_flags |= IFF_DRV_RUNNING; - ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; - - return; -} - -static void -ixl_init(void *arg) -{ - struct ixl_pf *pf = arg; - - IXL_PF_LOCK(pf); - ixl_init_locked(pf); - IXL_PF_UNLOCK(pf); - return; -} - -/* -** -** MSIX Interrupt Handlers and Tasklets -** -*/ -static void -ixl_handle_que(void *context, int pending) -{ - struct ixl_queue *que = context; - struct ixl_vsi *vsi = que->vsi; - struct i40e_hw *hw = vsi->hw; - struct tx_ring *txr = &que->txr; - struct ifnet *ifp = vsi->ifp; - bool more; - - if (ifp->if_drv_flags & IFF_DRV_RUNNING) { - more = ixl_rxeof(que, IXL_RX_LIMIT); - IXL_TX_LOCK(txr); - ixl_txeof(que); - if (!drbr_empty(ifp, txr->br)) - ixl_mq_start_locked(ifp, txr); - IXL_TX_UNLOCK(txr); - if (more) { - taskqueue_enqueue(que->tq, &que->task); - return; - } - } - - /* Reenable this interrupt - hmmm */ - ixl_enable_queue(hw, que->me); - return; -} - - -/********************************************************************* - * - * Legacy Interrupt Service routine - * - **********************************************************************/ -void -ixl_intr(void *arg) -{ - struct ixl_pf *pf = arg; - struct i40e_hw *hw = &pf->hw; - struct ixl_vsi *vsi = &pf->vsi; - struct ixl_queue *que = vsi->queues; - struct ifnet *ifp = vsi->ifp; - struct tx_ring *txr = &que->txr; - u32 reg, icr0, mask; - bool more_tx, more_rx; - - ++que->irqs; - - /* Protect against spurious interrupts */ - if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) - return; - - icr0 = rd32(hw, I40E_PFINT_ICR0); - - reg = rd32(hw, I40E_PFINT_DYN_CTL0); - reg = reg | I40E_PFINT_DYN_CTL0_CLEARPBA_MASK; - wr32(hw, I40E_PFINT_DYN_CTL0, reg); - - mask = rd32(hw, I40E_PFINT_ICR0_ENA); - -#ifdef PCI_IOV - if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) - taskqueue_enqueue(pf->tq, &pf->vflr_task); -#endif - - if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) { - taskqueue_enqueue(pf->tq, &pf->adminq); - return; - } - - more_rx = ixl_rxeof(que, IXL_RX_LIMIT); - - IXL_TX_LOCK(txr); - more_tx = ixl_txeof(que); - if (!drbr_empty(vsi->ifp, txr->br)) - more_tx = 1; - IXL_TX_UNLOCK(txr); - - /* re-enable other interrupt causes */ - wr32(hw, I40E_PFINT_ICR0_ENA, mask); - - /* And now the queues */ - reg = rd32(hw, I40E_QINT_RQCTL(0)); - reg |= I40E_QINT_RQCTL_CAUSE_ENA_MASK; - wr32(hw, I40E_QINT_RQCTL(0), reg); - - reg = rd32(hw, I40E_QINT_TQCTL(0)); - reg |= I40E_QINT_TQCTL_CAUSE_ENA_MASK; - reg &= ~I40E_PFINT_ICR0_INTEVENT_MASK; - wr32(hw, I40E_QINT_TQCTL(0), reg); - - ixl_enable_legacy(hw); - - return; -} - - -/********************************************************************* - * - * MSIX VSI Interrupt Service routine - * - **********************************************************************/ -void -ixl_msix_que(void *arg) -{ - struct ixl_queue *que = arg; - struct ixl_vsi *vsi = que->vsi; - struct i40e_hw *hw = vsi->hw; - struct tx_ring *txr = &que->txr; - bool more_tx, more_rx; - - /* Protect against spurious interrupts */ - if (!(vsi->ifp->if_drv_flags & IFF_DRV_RUNNING)) - return; - - ++que->irqs; - - more_rx = ixl_rxeof(que, IXL_RX_LIMIT); - - IXL_TX_LOCK(txr); - more_tx = ixl_txeof(que); - /* - ** Make certain that if the stack - ** has anything queued the task gets - ** scheduled to handle it. - */ - if (!drbr_empty(vsi->ifp, txr->br)) - more_tx = 1; - IXL_TX_UNLOCK(txr); - - ixl_set_queue_rx_itr(que); - ixl_set_queue_tx_itr(que); - - if (more_tx || more_rx) - taskqueue_enqueue(que->tq, &que->task); - else - ixl_enable_queue(hw, que->me); - - return; -} - - -/********************************************************************* - * - * MSIX Admin Queue Interrupt Service routine - * - **********************************************************************/ -static void -ixl_msix_adminq(void *arg) -{ - struct ixl_pf *pf = arg; - struct i40e_hw *hw = &pf->hw; - u32 reg, mask; - - ++pf->admin_irq; - - reg = rd32(hw, I40E_PFINT_ICR0); - mask = rd32(hw, I40E_PFINT_ICR0_ENA); - - /* Check on the cause */ - if (reg & I40E_PFINT_ICR0_ADMINQ_MASK) - mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK; - - if (reg & I40E_PFINT_ICR0_MAL_DETECT_MASK) { - ixl_handle_mdd_event(pf); - mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; - } - -#ifdef PCI_IOV - if (reg & I40E_PFINT_ICR0_VFLR_MASK) { - mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK; - taskqueue_enqueue(pf->tq, &pf->vflr_task); - } -#endif - - reg = rd32(hw, I40E_PFINT_DYN_CTL0); - reg = reg | I40E_PFINT_DYN_CTL0_CLEARPBA_MASK; - wr32(hw, I40E_PFINT_DYN_CTL0, reg); - - taskqueue_enqueue(pf->tq, &pf->adminq); - return; -} - -/********************************************************************* - * - * Media Ioctl callback - * - * This routine is called whenever the user queries the status of - * the interface using ifconfig. - * - **********************************************************************/ -static void -ixl_media_status(struct ifnet * ifp, struct ifmediareq * ifmr) -{ - struct ixl_vsi *vsi = ifp->if_softc; - struct ixl_pf *pf = vsi->back; - struct i40e_hw *hw = &pf->hw; - - INIT_DEBUGOUT("ixl_media_status: begin"); - IXL_PF_LOCK(pf); - - hw->phy.get_link_info = TRUE; - i40e_get_link_status(hw, &pf->link_up); - ixl_update_link_status(pf); - - ifmr->ifm_status = IFM_AVALID; - ifmr->ifm_active = IFM_ETHER; - - if (!pf->link_up) { - IXL_PF_UNLOCK(pf); - return; - } - - ifmr->ifm_status |= IFM_ACTIVE; - /* Hardware is always full-duplex */ - ifmr->ifm_active |= IFM_FDX; - - switch (hw->phy.link_info.phy_type) { - /* 100 M */ - case I40E_PHY_TYPE_100BASE_TX: - ifmr->ifm_active |= IFM_100_TX; - break; - /* 1 G */ - case I40E_PHY_TYPE_1000BASE_T: - ifmr->ifm_active |= IFM_1000_T; - break; - case I40E_PHY_TYPE_1000BASE_SX: - ifmr->ifm_active |= IFM_1000_SX; - break; - case I40E_PHY_TYPE_1000BASE_LX: - ifmr->ifm_active |= IFM_1000_LX; - break; - /* 10 G */ - case I40E_PHY_TYPE_10GBASE_SFPP_CU: - ifmr->ifm_active |= IFM_10G_TWINAX; - break; - case I40E_PHY_TYPE_10GBASE_SR: - ifmr->ifm_active |= IFM_10G_SR; - break; - case I40E_PHY_TYPE_10GBASE_LR: - ifmr->ifm_active |= IFM_10G_LR; - break; - case I40E_PHY_TYPE_10GBASE_T: - ifmr->ifm_active |= IFM_10G_T; - break; - /* 40 G */ - case I40E_PHY_TYPE_40GBASE_CR4: - case I40E_PHY_TYPE_40GBASE_CR4_CU: - ifmr->ifm_active |= IFM_40G_CR4; - break; - case I40E_PHY_TYPE_40GBASE_SR4: - ifmr->ifm_active |= IFM_40G_SR4; - break; - case I40E_PHY_TYPE_40GBASE_LR4: - ifmr->ifm_active |= IFM_40G_LR4; - break; -#ifndef IFM_ETH_XTYPE - case I40E_PHY_TYPE_1000BASE_KX: - ifmr->ifm_active |= IFM_1000_CX; - break; - case I40E_PHY_TYPE_10GBASE_CR1_CU: - case I40E_PHY_TYPE_10GBASE_CR1: - ifmr->ifm_active |= IFM_10G_TWINAX; - break; - case I40E_PHY_TYPE_10GBASE_KX4: - ifmr->ifm_active |= IFM_10G_CX4; - break; - case I40E_PHY_TYPE_10GBASE_KR: - ifmr->ifm_active |= IFM_10G_SR; - break; - case I40E_PHY_TYPE_40GBASE_KR4: - case I40E_PHY_TYPE_XLPPI: - ifmr->ifm_active |= IFM_40G_SR4; - break; -#else - case I40E_PHY_TYPE_1000BASE_KX: - ifmr->ifm_active |= IFM_1000_KX; - break; - /* ERJ: What's the difference between these? */ - case I40E_PHY_TYPE_10GBASE_CR1_CU: - case I40E_PHY_TYPE_10GBASE_CR1: - ifmr->ifm_active |= IFM_10G_CR1; - break; - case I40E_PHY_TYPE_10GBASE_KX4: - ifmr->ifm_active |= IFM_10G_KX4; - break; - case I40E_PHY_TYPE_10GBASE_KR: - ifmr->ifm_active |= IFM_10G_KR; - break; - case I40E_PHY_TYPE_20GBASE_KR2: - ifmr->ifm_active |= IFM_20G_KR2; - break; - case I40E_PHY_TYPE_40GBASE_KR4: - ifmr->ifm_active |= IFM_40G_KR4; - break; - case I40E_PHY_TYPE_XLPPI: - ifmr->ifm_active |= IFM_40G_XLPPI; - break; -#endif - default: - ifmr->ifm_active |= IFM_UNKNOWN; - break; - } - /* Report flow control status as well */ - if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_TX) - ifmr->ifm_active |= IFM_ETH_TXPAUSE; - if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_RX) - ifmr->ifm_active |= IFM_ETH_RXPAUSE; - - IXL_PF_UNLOCK(pf); - - return; -} - -/********************************************************************* - * - * Media Ioctl callback - * - * This routine is called when the user changes speed/duplex using - * media/mediopt option with ifconfig. - * - **********************************************************************/ -static int -ixl_media_change(struct ifnet * ifp) -{ - struct ixl_vsi *vsi = ifp->if_softc; - struct ifmedia *ifm = &vsi->media; - - INIT_DEBUGOUT("ixl_media_change: begin"); - - if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) - return (EINVAL); - - if_printf(ifp, "Media change is currently not supported.\n"); - - return (ENODEV); -} - - -#ifdef IXL_FDIR -/* -** ATR: Application Targetted Receive - creates a filter -** based on TX flow info that will keep the receive -** portion of the flow on the same queue. Based on the -** implementation this is only available for TCP connections -*/ -void -ixl_atr(struct ixl_queue *que, struct tcphdr *th, int etype) -{ - struct ixl_vsi *vsi = que->vsi; - struct tx_ring *txr = &que->txr; - struct i40e_filter_program_desc *FDIR; - u32 ptype, dtype; - int idx; - - /* check if ATR is enabled and sample rate */ - if ((!ixl_enable_fdir) || (!txr->atr_rate)) - return; - /* - ** We sample all TCP SYN/FIN packets, - ** or at the selected sample rate - */ - txr->atr_count++; - if (((th->th_flags & (TH_FIN | TH_SYN)) == 0) && - (txr->atr_count < txr->atr_rate)) - return; - txr->atr_count = 0; - - /* Get a descriptor to use */ - idx = txr->next_avail; - FDIR = (struct i40e_filter_program_desc *) &txr->base[idx]; - if (++idx == que->num_desc) - idx = 0; - txr->avail--; - txr->next_avail = idx; - - ptype = (que->me << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) & - I40E_TXD_FLTR_QW0_QINDEX_MASK; - - ptype |= (etype == ETHERTYPE_IP) ? - (I40E_FILTER_PCTYPE_NONF_IPV4_TCP << - I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) : - (I40E_FILTER_PCTYPE_NONF_IPV6_TCP << - I40E_TXD_FLTR_QW0_PCTYPE_SHIFT); - - ptype |= vsi->id << I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT; - - dtype = I40E_TX_DESC_DTYPE_FILTER_PROG; - - /* - ** We use the TCP TH_FIN as a trigger to remove - ** the filter, otherwise its an update. - */ - dtype |= (th->th_flags & TH_FIN) ? - (I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE << - I40E_TXD_FLTR_QW1_PCMD_SHIFT) : - (I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE << - I40E_TXD_FLTR_QW1_PCMD_SHIFT); - - dtype |= I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX << - I40E_TXD_FLTR_QW1_DEST_SHIFT; - - dtype |= I40E_FILTER_PROGRAM_DESC_FD_STATUS_FD_ID << - I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT; - - FDIR->qindex_flex_ptype_vsi = htole32(ptype); - FDIR->dtype_cmd_cntindex = htole32(dtype); - return; -} -#endif - - -static void -ixl_set_promisc(struct ixl_vsi *vsi) -{ - struct ifnet *ifp = vsi->ifp; - struct i40e_hw *hw = vsi->hw; - int err, mcnt = 0; - bool uni = FALSE, multi = FALSE; - - if (ifp->if_flags & IFF_ALLMULTI) - multi = TRUE; - else { /* Need to count the multicast addresses */ - struct ifmultiaddr *ifma; - if_maddr_rlock(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { - if (ifma->ifma_addr->sa_family != AF_LINK) - continue; - if (mcnt == MAX_MULTICAST_ADDR) - break; - mcnt++; - } - if_maddr_runlock(ifp); - } - - if (mcnt >= MAX_MULTICAST_ADDR) - multi = TRUE; - if (ifp->if_flags & IFF_PROMISC) - uni = TRUE; - - err = i40e_aq_set_vsi_unicast_promiscuous(hw, - vsi->seid, uni, NULL); - err = i40e_aq_set_vsi_multicast_promiscuous(hw, - vsi->seid, multi, NULL); - return; -} - -/********************************************************************* - * Filter Routines - * - * Routines for multicast and vlan filter management. - * - *********************************************************************/ -static void -ixl_add_multi(struct ixl_vsi *vsi) -{ - struct ifmultiaddr *ifma; - struct ifnet *ifp = vsi->ifp; - struct i40e_hw *hw = vsi->hw; - int mcnt = 0, flags; - - IOCTL_DEBUGOUT("ixl_add_multi: begin"); - - if_maddr_rlock(ifp); - /* - ** First just get a count, to decide if we - ** we simply use multicast promiscuous. - */ - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { - if (ifma->ifma_addr->sa_family != AF_LINK) - continue; - mcnt++; - } - if_maddr_runlock(ifp); - - if (__predict_false(mcnt >= MAX_MULTICAST_ADDR)) { - /* delete existing MC filters */ - ixl_del_hw_filters(vsi, mcnt); - i40e_aq_set_vsi_multicast_promiscuous(hw, - vsi->seid, TRUE, NULL); - return; - } - - mcnt = 0; - if_maddr_rlock(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { - if (ifma->ifma_addr->sa_family != AF_LINK) - continue; - ixl_add_mc_filter(vsi, - (u8*)LLADDR((struct sockaddr_dl *) ifma->ifma_addr)); - mcnt++; - } - if_maddr_runlock(ifp); - if (mcnt > 0) { - flags = (IXL_FILTER_ADD | IXL_FILTER_USED | IXL_FILTER_MC); - ixl_add_hw_filters(vsi, flags, mcnt); - } - - IOCTL_DEBUGOUT("ixl_add_multi: end"); - return; -} - -static void -ixl_del_multi(struct ixl_vsi *vsi) -{ - struct ifnet *ifp = vsi->ifp; - struct ifmultiaddr *ifma; - struct ixl_mac_filter *f; - int mcnt = 0; - bool match = FALSE; - - IOCTL_DEBUGOUT("ixl_del_multi: begin"); - - /* Search for removed multicast addresses */ - if_maddr_rlock(ifp); - SLIST_FOREACH(f, &vsi->ftl, next) { - if ((f->flags & IXL_FILTER_USED) && (f->flags & IXL_FILTER_MC)) { - match = FALSE; - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { - if (ifma->ifma_addr->sa_family != AF_LINK) - continue; - u8 *mc_addr = (u8 *)LLADDR((struct sockaddr_dl *)ifma->ifma_addr); - if (cmp_etheraddr(f->macaddr, mc_addr)) { - match = TRUE; - break; - } - } - if (match == FALSE) { - f->flags |= IXL_FILTER_DEL; - mcnt++; - } - } - } - if_maddr_runlock(ifp); - - if (mcnt > 0) - ixl_del_hw_filters(vsi, mcnt); -} - - -/********************************************************************* - * Timer routine - * - * This routine checks for link status,updates statistics, - * and runs the watchdog check. - * - **********************************************************************/ - -static void -ixl_local_timer(void *arg) -{ - struct ixl_pf *pf = arg; - struct i40e_hw *hw = &pf->hw; - struct ixl_vsi *vsi = &pf->vsi; - struct ixl_queue *que = vsi->queues; - device_t dev = pf->dev; - int hung = 0; - u32 mask; - - mtx_assert(&pf->pf_mtx, MA_OWNED); - - /* Fire off the adminq task */ - taskqueue_enqueue(pf->tq, &pf->adminq); - - /* Update stats */ - ixl_update_stats_counters(pf); - - /* - ** Check status of the queues - */ - mask = (I40E_PFINT_DYN_CTLN_INTENA_MASK | - I40E_PFINT_DYN_CTLN_SWINT_TRIG_MASK); - - for (int i = 0; i < vsi->num_queues; i++,que++) { - /* Any queues with outstanding work get a sw irq */ - if (que->busy) - wr32(hw, I40E_PFINT_DYN_CTLN(que->me), mask); - /* - ** Each time txeof runs without cleaning, but there - ** are uncleaned descriptors it increments busy. If - ** we get to 5 we declare it hung. - */ - if (que->busy == IXL_QUEUE_HUNG) { - ++hung; - /* Mark the queue as inactive */ - vsi->active_queues &= ~((u64)1 << que->me); - continue; - } else { - /* Check if we've come back from hung */ - if ((vsi->active_queues & ((u64)1 << que->me)) == 0) - vsi->active_queues |= ((u64)1 << que->me); - } - if (que->busy >= IXL_MAX_TX_BUSY) { -#ifdef IXL_DEBUG - device_printf(dev,"Warning queue %d " - "appears to be hung!\n", i); -#endif - que->busy = IXL_QUEUE_HUNG; - ++hung; - } - } - /* Only reinit if all queues show hung */ - if (hung == vsi->num_queues) - goto hung; - - callout_reset(&pf->timer, hz, ixl_local_timer, pf); - return; - -hung: - device_printf(dev, "Local Timer: HANG DETECT - Resetting!!\n"); - ixl_init_locked(pf); -} - -/* -** Note: this routine updates the OS on the link state -** the real check of the hardware only happens with -** a link interrupt. -*/ -static void -ixl_update_link_status(struct ixl_pf *pf) -{ - struct ixl_vsi *vsi = &pf->vsi; - struct i40e_hw *hw = &pf->hw; - struct ifnet *ifp = vsi->ifp; - device_t dev = pf->dev; - - if (pf->link_up){ - if (vsi->link_active == FALSE) { - pf->fc = hw->fc.current_mode; - if (bootverbose) { - device_printf(dev,"Link is up %d Gbps %s," - " Flow Control: %s\n", - ((pf->link_speed == - I40E_LINK_SPEED_40GB)? 40:10), - "Full Duplex", ixl_fc_string[pf->fc]); - } - vsi->link_active = TRUE; - /* - ** Warn user if link speed on NPAR enabled - ** partition is not at least 10GB - */ - if (hw->func_caps.npar_enable && - (hw->phy.link_info.link_speed == - I40E_LINK_SPEED_1GB || - hw->phy.link_info.link_speed == - I40E_LINK_SPEED_100MB)) - device_printf(dev, "The partition detected" - "link speed that is less than 10Gbps\n"); - if_link_state_change(ifp, LINK_STATE_UP); - } - } else { /* Link down */ - if (vsi->link_active == TRUE) { - if (bootverbose) - device_printf(dev,"Link is Down\n"); - if_link_state_change(ifp, LINK_STATE_DOWN); - vsi->link_active = FALSE; - } - } - - return; -} - -/********************************************************************* - * - * This routine disables all traffic on the adapter by issuing a - * global reset on the MAC and deallocates TX/RX buffers. - * - **********************************************************************/ - -static void -ixl_stop(struct ixl_pf *pf) -{ - struct ixl_vsi *vsi = &pf->vsi; - struct ifnet *ifp = vsi->ifp; - - mtx_assert(&pf->pf_mtx, MA_OWNED); - - INIT_DEBUGOUT("ixl_stop: begin\n"); - if (pf->num_vfs == 0) - ixl_disable_intr(vsi); - else - ixl_disable_rings_intr(vsi); - ixl_disable_rings(vsi); - - /* Tell the stack that the interface is no longer active */ - ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); - - /* Stop the local timer */ - callout_stop(&pf->timer); - - return; -} - - -/********************************************************************* - * - * Setup MSIX Interrupt resources and handlers for the VSI - * - **********************************************************************/ -static int -ixl_assign_vsi_legacy(struct ixl_pf *pf) -{ - device_t dev = pf->dev; - struct ixl_vsi *vsi = &pf->vsi; - struct ixl_queue *que = vsi->queues; - int error, rid = 0; - - if (pf->msix == 1) - rid = 1; - pf->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, - &rid, RF_SHAREABLE | RF_ACTIVE); - if (pf->res == NULL) { - device_printf(dev,"Unable to allocate" - " bus resource: vsi legacy/msi interrupt\n"); - return (ENXIO); - } - - /* Set the handler function */ - error = bus_setup_intr(dev, pf->res, - INTR_TYPE_NET | INTR_MPSAFE, NULL, - ixl_intr, pf, &pf->tag); - if (error) { - pf->res = NULL; - device_printf(dev, "Failed to register legacy/msi handler"); - return (error); - } - bus_describe_intr(dev, pf->res, pf->tag, "irq0"); - TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que); - TASK_INIT(&que->task, 0, ixl_handle_que, que); - que->tq = taskqueue_create_fast("ixl_que", M_NOWAIT, - taskqueue_thread_enqueue, &que->tq); - taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que", - device_get_nameunit(dev)); - TASK_INIT(&pf->adminq, 0, ixl_do_adminq, pf); - -#ifdef PCI_IOV - TASK_INIT(&pf->vflr_task, 0, ixl_handle_vflr, pf); -#endif - - pf->tq = taskqueue_create_fast("ixl_adm", M_NOWAIT, - taskqueue_thread_enqueue, &pf->tq); - taskqueue_start_threads(&pf->tq, 1, PI_NET, "%s adminq", - device_get_nameunit(dev)); - - return (0); -} - -static void -ixl_init_taskqueues(struct ixl_pf *pf) -{ - struct ixl_vsi *vsi = &pf->vsi; - struct ixl_queue *que = vsi->queues; - device_t dev = pf->dev; -#ifdef RSS - int cpu_id; - cpuset_t cpu_mask; -#endif - - /* Tasklet for Admin Queue */ - TASK_INIT(&pf->adminq, 0, ixl_do_adminq, pf); -#ifdef PCI_IOV - /* VFLR Tasklet */ - TASK_INIT(&pf->vflr_task, 0, ixl_handle_vflr, pf); -#endif - - /* Create and start PF taskqueue */ - pf->tq = taskqueue_create_fast("ixl_adm", M_NOWAIT, - taskqueue_thread_enqueue, &pf->tq); - taskqueue_start_threads(&pf->tq, 1, PI_NET, "%s adminq", - device_get_nameunit(dev)); - - /* Create queue tasks and start queue taskqueues */ - for (int i = 0; i < vsi->num_queues; i++, que++) { - TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que); - TASK_INIT(&que->task, 0, ixl_handle_que, que); - que->tq = taskqueue_create_fast("ixl_que", M_NOWAIT, - taskqueue_thread_enqueue, &que->tq); -#ifdef RSS - cpu_id = rss_getcpu(i % rss_getnumbuckets()); - CPU_SETOF(cpu_id, &cpu_mask); - taskqueue_start_threads_cpuset(&que->tq, 1, PI_NET, - &cpu_mask, "%s (bucket %d)", - device_get_nameunit(dev), cpu_id); -#else - taskqueue_start_threads(&que->tq, 1, PI_NET, - "%s (que %d)", device_get_nameunit(dev), que->me); -#endif - } -} - -static void -ixl_free_taskqueues(struct ixl_pf *pf) -{ - struct ixl_vsi *vsi = &pf->vsi; - struct ixl_queue *que = vsi->queues; - - if (pf->tq) - taskqueue_free(pf->tq); - for (int i = 0; i < vsi->num_queues; i++, que++) { - if (que->tq) - taskqueue_free(que->tq); - } -} - -/********************************************************************* - * - * Setup MSIX Interrupt resources and handlers for the VSI - * - **********************************************************************/ -static int -ixl_assign_vsi_msix(struct ixl_pf *pf) -{ - device_t dev = pf->dev; - struct ixl_vsi *vsi = &pf->vsi; - struct ixl_queue *que = vsi->queues; - struct tx_ring *txr; - int error, rid, vector = 0; - - /* Admin Que is vector 0*/ - rid = vector + 1; - pf->res = bus_alloc_resource_any(dev, - SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); - if (!pf->res) { - device_printf(dev,"Unable to allocate" - " bus resource: Adminq interrupt [%d]\n", rid); - return (ENXIO); - } - /* Set the adminq vector and handler */ - error = bus_setup_intr(dev, pf->res, - INTR_TYPE_NET | INTR_MPSAFE, NULL, - ixl_msix_adminq, pf, &pf->tag); - if (error) { - pf->res = NULL; - device_printf(dev, "Failed to register Admin que handler"); - return (error); - } - bus_describe_intr(dev, pf->res, pf->tag, "aq"); - pf->admvec = vector; - ++vector; - - /* Now set up the stations */ - for (int i = 0; i < vsi->num_queues; i++, vector++, que++) { - int cpu_id = i; - rid = vector + 1; - txr = &que->txr; - que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, - RF_SHAREABLE | RF_ACTIVE); - if (que->res == NULL) { - device_printf(dev,"Unable to allocate" - " bus resource: que interrupt [%d]\n", vector); - return (ENXIO); - } - /* Set the handler function */ - error = bus_setup_intr(dev, que->res, - INTR_TYPE_NET | INTR_MPSAFE, NULL, - ixl_msix_que, que, &que->tag); - if (error) { - que->res = NULL; - device_printf(dev, "Failed to register que handler"); - return (error); - } - bus_describe_intr(dev, que->res, que->tag, "q%d", i); - /* Bind the vector to a CPU */ -#ifdef RSS - cpu_id = rss_getcpu(i % rss_getnumbuckets()); -#endif - bus_bind_intr(dev, que->res, cpu_id); - que->msix = vector; - } - - return (0); -} - - -/* - * Allocate MSI/X vectors - */ -static int -ixl_init_msix(struct ixl_pf *pf) -{ - device_t dev = pf->dev; - int rid, want, vectors, queues, available; - - /* Override by tuneable */ - if (ixl_enable_msix == 0) - goto msi; - - /* - ** When used in a virtualized environment - ** PCI BUSMASTER capability may not be set - ** so explicity set it here and rewrite - ** the ENABLE in the MSIX control register - ** at this point to cause the host to - ** successfully initialize us. - */ - { - u16 pci_cmd_word; - int msix_ctrl; - pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); - pci_cmd_word |= PCIM_CMD_BUSMASTEREN; - pci_write_config(dev, PCIR_COMMAND, pci_cmd_word, 2); - pci_find_cap(dev, PCIY_MSIX, &rid); - rid += PCIR_MSIX_CTRL; - msix_ctrl = pci_read_config(dev, rid, 2); - msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE; - pci_write_config(dev, rid, msix_ctrl, 2); - } - - /* First try MSI/X */ - rid = PCIR_BAR(IXL_BAR); - pf->msix_mem = bus_alloc_resource_any(dev, - SYS_RES_MEMORY, &rid, RF_ACTIVE); - if (!pf->msix_mem) { - /* May not be enabled */ - device_printf(pf->dev, - "Unable to map MSIX table \n"); - goto msi; - } - - available = pci_msix_count(dev); - if (available == 0) { /* system has msix disabled */ - bus_release_resource(dev, SYS_RES_MEMORY, - rid, pf->msix_mem); - pf->msix_mem = NULL; - goto msi; - } - - /* Figure out a reasonable auto config value */ - queues = (mp_ncpus > (available - 1)) ? (available - 1) : mp_ncpus; - - /* Override with hardcoded value if it's less than autoconfig count */ - if ((ixl_max_queues != 0) && (ixl_max_queues <= queues)) - queues = ixl_max_queues; - else if ((ixl_max_queues != 0) && (ixl_max_queues > queues)) - device_printf(dev, "ixl_max_queues > # of cpus, using " - "autoconfig amount...\n"); - /* Or limit maximum auto-configured queues to 8 */ - else if ((ixl_max_queues == 0) && (queues > 8)) - queues = 8; - -#ifdef RSS - /* If we're doing RSS, clamp at the number of RSS buckets */ - if (queues > rss_getnumbuckets()) - queues = rss_getnumbuckets(); -#endif - - /* - ** Want one vector (RX/TX pair) per queue - ** plus an additional for the admin queue. - */ - want = queues + 1; - if (want <= available) /* Have enough */ - vectors = want; - else { - device_printf(pf->dev, - "MSIX Configuration Problem, " - "%d vectors available but %d wanted!\n", - available, want); - return (0); /* Will go to Legacy setup */ - } - - if (pci_alloc_msix(dev, &vectors) == 0) { - device_printf(pf->dev, - "Using MSIX interrupts with %d vectors\n", vectors); - pf->msix = vectors; - pf->vsi.num_queues = queues; -#ifdef RSS - /* - * If we're doing RSS, the number of queues needs to - * match the number of RSS buckets that are configured. - * - * + If there's more queues than RSS buckets, we'll end - * up with queues that get no traffic. - * - * + If there's more RSS buckets than queues, we'll end - * up having multiple RSS buckets map to the same queue, - * so there'll be some contention. - */ - if (queues != rss_getnumbuckets()) { - device_printf(dev, - "%s: queues (%d) != RSS buckets (%d)" - "; performance will be impacted.\n", - __func__, queues, rss_getnumbuckets()); - } -#endif - return (vectors); - } -msi: - vectors = pci_msi_count(dev); - pf->vsi.num_queues = 1; - pf->msix = 1; - ixl_max_queues = 1; - ixl_enable_msix = 0; - if (vectors == 1 && pci_alloc_msi(dev, &vectors) == 0) - device_printf(pf->dev,"Using an MSI interrupt\n"); - else { - pf->msix = 0; - device_printf(pf->dev,"Using a Legacy interrupt\n"); - } - return (vectors); -} - - -/* - * Plumb MSI/X vectors - */ -static void -ixl_configure_msix(struct ixl_pf *pf) -{ - struct i40e_hw *hw = &pf->hw; - struct ixl_vsi *vsi = &pf->vsi; - u32 reg; - u16 vector = 1; - - /* First set up the adminq - vector 0 */ - wr32(hw, I40E_PFINT_ICR0_ENA, 0); /* disable all */ - rd32(hw, I40E_PFINT_ICR0); /* read to clear */ - - reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | - I40E_PFINT_ICR0_ENA_GRST_MASK | - I40E_PFINT_ICR0_HMC_ERR_MASK | - I40E_PFINT_ICR0_ENA_ADMINQ_MASK | - I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | - I40E_PFINT_ICR0_ENA_VFLR_MASK | - I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK; - wr32(hw, I40E_PFINT_ICR0_ENA, reg); - - wr32(hw, I40E_PFINT_LNKLST0, 0x7FF); - wr32(hw, I40E_PFINT_ITR0(IXL_RX_ITR), 0x003E); - - wr32(hw, I40E_PFINT_DYN_CTL0, - I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK | - I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK); - - wr32(hw, I40E_PFINT_STAT_CTL0, 0); - - /* Next configure the queues */ - for (int i = 0; i < vsi->num_queues; i++, vector++) { - wr32(hw, I40E_PFINT_DYN_CTLN(i), 0); - /* First queue type is RX / type 0 */ - wr32(hw, I40E_PFINT_LNKLSTN(i), i); - - reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK | - (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | - (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | - (i << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | - (I40E_QUEUE_TYPE_TX << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT); - wr32(hw, I40E_QINT_RQCTL(i), reg); - - reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK | - (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | - (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | - (IXL_QUEUE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | - (I40E_QUEUE_TYPE_RX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); - wr32(hw, I40E_QINT_TQCTL(i), reg); - } -} - -/* - * Configure for MSI single vector operation - */ -static void -ixl_configure_legacy(struct ixl_pf *pf) -{ - struct i40e_hw *hw = &pf->hw; - u32 reg; - - - wr32(hw, I40E_PFINT_ITR0(0), 0); - wr32(hw, I40E_PFINT_ITR0(1), 0); - - - /* Setup "other" causes */ - reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK - | I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK - | I40E_PFINT_ICR0_ENA_GRST_MASK - | I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK - | I40E_PFINT_ICR0_ENA_GPIO_MASK - | I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK - | I40E_PFINT_ICR0_ENA_HMC_ERR_MASK - | I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK - | I40E_PFINT_ICR0_ENA_VFLR_MASK - | I40E_PFINT_ICR0_ENA_ADMINQ_MASK - ; - wr32(hw, I40E_PFINT_ICR0_ENA, reg); - - /* SW_ITR_IDX = 0, but don't change INTENA */ - wr32(hw, I40E_PFINT_DYN_CTL0, - I40E_PFINT_DYN_CTLN_SW_ITR_INDX_MASK | - I40E_PFINT_DYN_CTLN_INTENA_MSK_MASK); - /* SW_ITR_IDX = 0, OTHER_ITR_IDX = 0 */ - wr32(hw, I40E_PFINT_STAT_CTL0, 0); - - /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */ - wr32(hw, I40E_PFINT_LNKLST0, 0); - - /* Associate the queue pair to the vector and enable the q int */ - reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK - | (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) - | (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); - wr32(hw, I40E_QINT_RQCTL(0), reg); - - reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK - | (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) - | (IXL_QUEUE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); - wr32(hw, I40E_QINT_TQCTL(0), reg); - - /* Next enable the queue pair */ - reg = rd32(hw, I40E_QTX_ENA(0)); - reg |= I40E_QTX_ENA_QENA_REQ_MASK; - wr32(hw, I40E_QTX_ENA(0), reg); - - reg = rd32(hw, I40E_QRX_ENA(0)); - reg |= I40E_QRX_ENA_QENA_REQ_MASK; - wr32(hw, I40E_QRX_ENA(0), reg); -} - - -/* - * Set the Initial ITR state - */ -static void -ixl_configure_itr(struct ixl_pf *pf) -{ - struct i40e_hw *hw = &pf->hw; - struct ixl_vsi *vsi = &pf->vsi; - struct ixl_queue *que = vsi->queues; - - vsi->rx_itr_setting = ixl_rx_itr; - if (ixl_dynamic_rx_itr) - vsi->rx_itr_setting |= IXL_ITR_DYNAMIC; - vsi->tx_itr_setting = ixl_tx_itr; - if (ixl_dynamic_tx_itr) - vsi->tx_itr_setting |= IXL_ITR_DYNAMIC; - - for (int i = 0; i < vsi->num_queues; i++, que++) { - struct tx_ring *txr = &que->txr; - struct rx_ring *rxr = &que->rxr; - - wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, i), - vsi->rx_itr_setting); - rxr->itr = vsi->rx_itr_setting; - rxr->latency = IXL_AVE_LATENCY; - wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, i), - vsi->tx_itr_setting); - txr->itr = vsi->tx_itr_setting; - txr->latency = IXL_AVE_LATENCY; - } -} - - -static int -ixl_allocate_pci_resources(struct ixl_pf *pf) -{ - int rid; - device_t dev = pf->dev; - - rid = PCIR_BAR(0); - pf->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, - &rid, RF_ACTIVE); - - if (!(pf->pci_mem)) { - device_printf(dev,"Unable to allocate bus resource: memory\n"); - return (ENXIO); - } - - pf->osdep.mem_bus_space_tag = - rman_get_bustag(pf->pci_mem); - pf->osdep.mem_bus_space_handle = - rman_get_bushandle(pf->pci_mem); - pf->osdep.mem_bus_space_size = rman_get_size(pf->pci_mem); - pf->osdep.flush_reg = I40E_GLGEN_STAT; - pf->hw.hw_addr = (u8 *) &pf->osdep.mem_bus_space_handle; - - pf->hw.back = &pf->osdep; - - /* - ** Now setup MSI or MSI/X, should - ** return us the number of supported - ** vectors. (Will be 1 for MSI) - */ - pf->msix = ixl_init_msix(pf); - return (0); -} - -static void -ixl_free_pci_resources(struct ixl_pf * pf) -{ - struct ixl_vsi *vsi = &pf->vsi; - struct ixl_queue *que = vsi->queues; - device_t dev = pf->dev; - int rid, memrid; - - memrid = PCIR_BAR(IXL_BAR); - - /* We may get here before stations are setup */ - if ((!ixl_enable_msix) || (que == NULL)) - goto early; - - /* - ** Release all msix VSI resources: - */ - for (int i = 0; i < vsi->num_queues; i++, que++) { - rid = que->msix + 1; - if (que->tag != NULL) { - bus_teardown_intr(dev, que->res, que->tag); - que->tag = NULL; - } - if (que->res != NULL) - bus_release_resource(dev, SYS_RES_IRQ, rid, que->res); - } - -early: - /* Clean the AdminQ interrupt last */ - if (pf->admvec) /* we are doing MSIX */ - rid = pf->admvec + 1; - else - (pf->msix != 0) ? (rid = 1):(rid = 0); - - if (pf->tag != NULL) { - bus_teardown_intr(dev, pf->res, pf->tag); - pf->tag = NULL; - } - if (pf->res != NULL) - bus_release_resource(dev, SYS_RES_IRQ, rid, pf->res); - - if (pf->msix) - pci_release_msi(dev); - - if (pf->msix_mem != NULL) - bus_release_resource(dev, SYS_RES_MEMORY, - memrid, pf->msix_mem); - - if (pf->pci_mem != NULL) - bus_release_resource(dev, SYS_RES_MEMORY, - PCIR_BAR(0), pf->pci_mem); - - return; -} - -static void -ixl_add_ifmedia(struct ixl_vsi *vsi, u32 phy_type) -{ - /* Display supported media types */ - if (phy_type & (1 << I40E_PHY_TYPE_100BASE_TX)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_100_TX, 0, NULL); - - if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_T)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_T, 0, NULL); - if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_SX)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_SX, 0, NULL); - if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_LX)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_LX, 0, NULL); - - if (phy_type & (1 << I40E_PHY_TYPE_XAUI) || - phy_type & (1 << I40E_PHY_TYPE_XFI) || - phy_type & (1 << I40E_PHY_TYPE_10GBASE_SFPP_CU)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX, 0, NULL); - - if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_SR)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SR, 0, NULL); - if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_LR)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_LR, 0, NULL); - if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_T)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_T, 0, NULL); - - if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_CR4) || - phy_type & (1 << I40E_PHY_TYPE_40GBASE_CR4_CU) || - phy_type & (1 << I40E_PHY_TYPE_40GBASE_AOC) || - phy_type & (1 << I40E_PHY_TYPE_XLAUI) || - phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_CR4, 0, NULL); - if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_SR4)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_SR4, 0, NULL); - if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_LR4)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_LR4, 0, NULL); - -#ifndef IFM_ETH_XTYPE - if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_KX)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_CX, 0, NULL); - - if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1_CU) || - phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1) || - phy_type & (1 << I40E_PHY_TYPE_10GBASE_AOC) || - phy_type & (1 << I40E_PHY_TYPE_SFI)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX, 0, NULL); - if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KX4)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_CX4, 0, NULL); - if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KR)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SR, 0, NULL); - - if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_SR4, 0, NULL); - if (phy_type & (1 << I40E_PHY_TYPE_XLPPI)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_CR4, 0, NULL); -#else - if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_KX)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_KX, 0, NULL); - - if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1_CU) - || phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_CR1, 0, NULL); - if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_AOC)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX_LONG, 0, NULL); - if (phy_type & (1 << I40E_PHY_TYPE_SFI)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SFI, 0, NULL); - if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KX4)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_KX4, 0, NULL); - if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KR)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_KR, 0, NULL); - - if (phy_type & (1 << I40E_PHY_TYPE_20GBASE_KR2)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_20G_KR2, 0, NULL); - - if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_KR4, 0, NULL); - if (phy_type & (1 << I40E_PHY_TYPE_XLPPI)) - ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_XLPPI, 0, NULL); -#endif -} - -/********************************************************************* - * - * Setup networking device structure and register an interface. - * - **********************************************************************/ -static int -ixl_setup_interface(device_t dev, struct ixl_vsi *vsi) -{ - struct ifnet *ifp; - struct i40e_hw *hw = vsi->hw; - struct ixl_queue *que = vsi->queues; - struct i40e_aq_get_phy_abilities_resp abilities; - enum i40e_status_code aq_error = 0; - - INIT_DEBUGOUT("ixl_setup_interface: begin"); - - ifp = vsi->ifp = if_alloc(IFT_ETHER); - if (ifp == NULL) { - device_printf(dev, "can not allocate ifnet structure\n"); - return (-1); - } - if_initname(ifp, device_get_name(dev), device_get_unit(dev)); - ifp->if_mtu = ETHERMTU; - ifp->if_baudrate = IF_Gbps(40); - ifp->if_init = ixl_init; - ifp->if_softc = vsi; - ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; - ifp->if_ioctl = ixl_ioctl; - -#if __FreeBSD_version >= 1100036 - if_setgetcounterfn(ifp, ixl_get_counter); -#endif - - ifp->if_transmit = ixl_mq_start; - - ifp->if_qflush = ixl_qflush; - - ifp->if_snd.ifq_maxlen = que->num_desc - 2; - - vsi->max_frame_size = - ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN - + ETHER_VLAN_ENCAP_LEN; - - /* - * Tell the upper layer(s) we support long frames. - */ - ifp->if_hdrlen = sizeof(struct ether_vlan_header); - - ifp->if_capabilities |= IFCAP_HWCSUM; - ifp->if_capabilities |= IFCAP_HWCSUM_IPV6; - ifp->if_capabilities |= IFCAP_TSO; - ifp->if_capabilities |= IFCAP_JUMBO_MTU; - ifp->if_capabilities |= IFCAP_LRO; - - /* VLAN capabilties */ - ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING - | IFCAP_VLAN_HWTSO - | IFCAP_VLAN_MTU - | IFCAP_VLAN_HWCSUM; - ifp->if_capenable = ifp->if_capabilities; - - /* - ** Don't turn this on by default, if vlans are - ** created on another pseudo device (eg. lagg) - ** then vlan events are not passed thru, breaking - ** operation, but with HW FILTER off it works. If - ** using vlans directly on the ixl driver you can - ** enable this and get full hardware tag filtering. - */ - ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; - - /* - * Specify the media types supported by this adapter and register - * callbacks to update media and link information - */ - ifmedia_init(&vsi->media, IFM_IMASK, ixl_media_change, - ixl_media_status); - - aq_error = i40e_aq_get_phy_capabilities(hw, - FALSE, TRUE, &abilities, NULL); - /* May need delay to detect fiber correctly */ - if (aq_error == I40E_ERR_UNKNOWN_PHY) { - i40e_msec_delay(200); - aq_error = i40e_aq_get_phy_capabilities(hw, FALSE, - TRUE, &abilities, NULL); - } - if (aq_error) { - if (aq_error == I40E_ERR_UNKNOWN_PHY) - device_printf(dev, "Unknown PHY type detected!\n"); - else - device_printf(dev, - "Error getting supported media types, err %d," - " AQ error %d\n", aq_error, hw->aq.asq_last_status); - return (0); - } - - ixl_add_ifmedia(vsi, abilities.phy_type); - - /* Use autoselect media by default */ - ifmedia_add(&vsi->media, IFM_ETHER | IFM_AUTO, 0, NULL); - ifmedia_set(&vsi->media, IFM_ETHER | IFM_AUTO); - - ether_ifattach(ifp, hw->mac.addr); - - return (0); -} - -/* -** Run when the Admin Queue gets a -** link transition interrupt. -*/ -static void -ixl_link_event(struct ixl_pf *pf, struct i40e_arq_event_info *e) -{ - struct i40e_hw *hw = &pf->hw; - struct i40e_aqc_get_link_status *status = - (struct i40e_aqc_get_link_status *)&e->desc.params.raw; - bool check; - - hw->phy.get_link_info = TRUE; - i40e_get_link_status(hw, &check); - pf->link_up = check; -#ifdef IXL_DEBUG - printf("Link is %s\n", check ? "up":"down"); -#endif - /* Report if Unqualified modules are found */ - if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) && - (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) && - (!(status->link_info & I40E_AQ_LINK_UP))) - device_printf(pf->dev, "Link failed because " - "an unqualified module was detected\n"); - - return; -} - -/********************************************************************* - * - * Get Firmware Switch configuration - * - this will need to be more robust when more complex - * switch configurations are enabled. - * - **********************************************************************/ -static int -ixl_switch_config(struct ixl_pf *pf) -{ - struct i40e_hw *hw = &pf->hw; - struct ixl_vsi *vsi = &pf->vsi; - device_t dev = vsi->dev; - struct i40e_aqc_get_switch_config_resp *sw_config; - u8 aq_buf[I40E_AQ_LARGE_BUF]; - int ret; - u16 next = 0; - - memset(&aq_buf, 0, sizeof(aq_buf)); - sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf; - ret = i40e_aq_get_switch_config(hw, sw_config, - sizeof(aq_buf), &next, NULL); - if (ret) { - device_printf(dev,"aq_get_switch_config failed (ret=%d)!!\n", - ret); - return (ret); - } -#ifdef IXL_DEBUG - device_printf(dev, - "Switch config: header reported: %d in structure, %d total\n", - sw_config->header.num_reported, sw_config->header.num_total); - for (int i = 0; i < sw_config->header.num_reported; i++) { - device_printf(dev, - "%d: type=%d seid=%d uplink=%d downlink=%d\n", i, - sw_config->element[i].element_type, - sw_config->element[i].seid, - sw_config->element[i].uplink_seid, - sw_config->element[i].downlink_seid); - } -#endif - /* Simplified due to a single VSI at the moment */ - vsi->uplink_seid = sw_config->element[0].uplink_seid; - vsi->downlink_seid = sw_config->element[0].downlink_seid; - vsi->seid = sw_config->element[0].seid; - return (ret); -} - -/********************************************************************* - * - * Initialize the VSI: this handles contexts, which means things - * like the number of descriptors, buffer size, - * plus we init the rings thru this function. - * - **********************************************************************/ -static int -ixl_initialize_vsi(struct ixl_vsi *vsi) -{ - struct ixl_pf *pf = vsi->back; - struct ixl_queue *que = vsi->queues; - device_t dev = vsi->dev; - struct i40e_hw *hw = vsi->hw; - struct i40e_vsi_context ctxt; - int err = 0; - - memset(&ctxt, 0, sizeof(ctxt)); - ctxt.seid = vsi->seid; - if (pf->veb_seid != 0) - ctxt.uplink_seid = pf->veb_seid; - ctxt.pf_num = hw->pf_id; - err = i40e_aq_get_vsi_params(hw, &ctxt, NULL); - if (err) { - device_printf(dev,"get vsi params failed %x!!\n", err); - return (err); - } -#ifdef IXL_DEBUG - printf("get_vsi_params: seid: %d, uplinkseid: %d, vsi_number: %d, " - "vsis_allocated: %d, vsis_unallocated: %d, flags: 0x%x, " - "pfnum: %d, vfnum: %d, stat idx: %d, enabled: %d\n", ctxt.seid, - ctxt.uplink_seid, ctxt.vsi_number, - ctxt.vsis_allocated, ctxt.vsis_unallocated, - ctxt.flags, ctxt.pf_num, ctxt.vf_num, - ctxt.info.stat_counter_idx, ctxt.info.up_enable_bits); -#endif - /* - ** Set the queue and traffic class bits - ** - when multiple traffic classes are supported - ** this will need to be more robust. - */ - ctxt.info.valid_sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; - ctxt.info.mapping_flags |= I40E_AQ_VSI_QUE_MAP_CONTIG; - /* In contig mode, que_mapping[0] is first queue index used by this VSI */ - ctxt.info.queue_mapping[0] = 0; - /* - * This VSI will only use traffic class 0; start traffic class 0's - * queue allocation at queue 0, and assign it 64 (2^6) queues (though - * the driver may not use all of them). - */ - ctxt.info.tc_mapping[0] = ((0 << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) - & I40E_AQ_VSI_TC_QUE_OFFSET_MASK) | - ((6 << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT) - & I40E_AQ_VSI_TC_QUE_NUMBER_MASK); - - /* Set VLAN receive stripping mode */ - ctxt.info.valid_sections |= I40E_AQ_VSI_PROP_VLAN_VALID; - ctxt.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL; - if (vsi->ifp->if_capenable & IFCAP_VLAN_HWTAGGING) - ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH; - else - ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_NOTHING; - - /* Keep copy of VSI info in VSI for statistic counters */ - memcpy(&vsi->info, &ctxt.info, sizeof(ctxt.info)); - - /* Reset VSI statistics */ - ixl_vsi_reset_stats(vsi); - vsi->hw_filters_add = 0; - vsi->hw_filters_del = 0; - - ctxt.flags = htole16(I40E_AQ_VSI_TYPE_PF); - - err = i40e_aq_update_vsi_params(hw, &ctxt, NULL); - if (err) { - device_printf(dev,"update vsi params failed %x!!\n", - hw->aq.asq_last_status); - return (err); - } - - for (int i = 0; i < vsi->num_queues; i++, que++) { - struct tx_ring *txr = &que->txr; - struct rx_ring *rxr = &que->rxr; - struct i40e_hmc_obj_txq tctx; - struct i40e_hmc_obj_rxq rctx; - u32 txctl; - u16 size; - - - /* Setup the HMC TX Context */ - size = que->num_desc * sizeof(struct i40e_tx_desc); - memset(&tctx, 0, sizeof(struct i40e_hmc_obj_txq)); - tctx.new_context = 1; - tctx.base = (txr->dma.pa/IXL_TX_CTX_BASE_UNITS); - tctx.qlen = que->num_desc; - tctx.fc_ena = 0; - tctx.rdylist = vsi->info.qs_handle[0]; /* index is TC */ - /* Enable HEAD writeback */ - tctx.head_wb_ena = 1; - tctx.head_wb_addr = txr->dma.pa + - (que->num_desc * sizeof(struct i40e_tx_desc)); - tctx.rdylist_act = 0; - err = i40e_clear_lan_tx_queue_context(hw, i); - if (err) { - device_printf(dev, "Unable to clear TX context\n"); - break; - } - err = i40e_set_lan_tx_queue_context(hw, i, &tctx); - if (err) { - device_printf(dev, "Unable to set TX context\n"); - break; - } - /* Associate the ring with this PF */ - txctl = I40E_QTX_CTL_PF_QUEUE; - txctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) & - I40E_QTX_CTL_PF_INDX_MASK); - wr32(hw, I40E_QTX_CTL(i), txctl); - ixl_flush(hw); - - /* Do ring (re)init */ - ixl_init_tx_ring(que); - - /* Next setup the HMC RX Context */ - if (vsi->max_frame_size <= MCLBYTES) - rxr->mbuf_sz = MCLBYTES; - else - rxr->mbuf_sz = MJUMPAGESIZE; - - u16 max_rxmax = rxr->mbuf_sz * hw->func_caps.rx_buf_chain_len; - - /* Set up an RX context for the HMC */ - memset(&rctx, 0, sizeof(struct i40e_hmc_obj_rxq)); - rctx.dbuff = rxr->mbuf_sz >> I40E_RXQ_CTX_DBUFF_SHIFT; - /* ignore header split for now */ - rctx.hbuff = 0 >> I40E_RXQ_CTX_HBUFF_SHIFT; - rctx.rxmax = (vsi->max_frame_size < max_rxmax) ? - vsi->max_frame_size : max_rxmax; - rctx.dtype = 0; - rctx.dsize = 1; /* do 32byte descriptors */ - rctx.hsplit_0 = 0; /* no HDR split initially */ - rctx.base = (rxr->dma.pa/IXL_RX_CTX_BASE_UNITS); - rctx.qlen = que->num_desc; - rctx.tphrdesc_ena = 1; - rctx.tphwdesc_ena = 1; - rctx.tphdata_ena = 0; - rctx.tphhead_ena = 0; - rctx.lrxqthresh = 2; - rctx.crcstrip = 1; - rctx.l2tsel = 1; - rctx.showiv = 1; - rctx.fc_ena = 0; - rctx.prefena = 1; - - err = i40e_clear_lan_rx_queue_context(hw, i); - if (err) { - device_printf(dev, - "Unable to clear RX context %d\n", i); - break; - } - err = i40e_set_lan_rx_queue_context(hw, i, &rctx); - if (err) { - device_printf(dev, "Unable to set RX context %d\n", i); - break; - } - err = ixl_init_rx_ring(que); - if (err) { - device_printf(dev, "Fail in init_rx_ring %d\n", i); - break; - } -#ifdef DEV_NETMAP - /* preserve queue */ - if (vsi->ifp->if_capenable & IFCAP_NETMAP) { - struct netmap_adapter *na = NA(vsi->ifp); - struct netmap_kring *kring = &na->rx_rings[i]; - int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring); - wr32(vsi->hw, I40E_QRX_TAIL(que->me), t); - } else -#endif /* DEV_NETMAP */ - wr32(vsi->hw, I40E_QRX_TAIL(que->me), que->num_desc - 1); - } - return (err); -} - - -/********************************************************************* - * - * Free all VSI structs. - * - **********************************************************************/ -void -ixl_free_vsi(struct ixl_vsi *vsi) -{ - struct ixl_pf *pf = (struct ixl_pf *)vsi->back; - struct ixl_queue *que = vsi->queues; - - /* Free station queues */ - for (int i = 0; i < vsi->num_queues; i++, que++) { - struct tx_ring *txr = &que->txr; - struct rx_ring *rxr = &que->rxr; - - if (!mtx_initialized(&txr->mtx)) /* uninitialized */ - continue; - IXL_TX_LOCK(txr); - ixl_free_que_tx(que); - if (txr->base) - i40e_free_dma_mem(&pf->hw, &txr->dma); - IXL_TX_UNLOCK(txr); - IXL_TX_LOCK_DESTROY(txr); - - if (!mtx_initialized(&rxr->mtx)) /* uninitialized */ - continue; - IXL_RX_LOCK(rxr); - ixl_free_que_rx(que); - if (rxr->base) - i40e_free_dma_mem(&pf->hw, &rxr->dma); - IXL_RX_UNLOCK(rxr); - IXL_RX_LOCK_DESTROY(rxr); - - } - free(vsi->queues, M_DEVBUF); - - /* Free VSI filter list */ - ixl_free_mac_filters(vsi); -} - -static void -ixl_free_mac_filters(struct ixl_vsi *vsi) -{ - struct ixl_mac_filter *f; - - while (!SLIST_EMPTY(&vsi->ftl)) { - f = SLIST_FIRST(&vsi->ftl); - SLIST_REMOVE_HEAD(&vsi->ftl, next); - free(f, M_DEVBUF); - } -} - - -/********************************************************************* - * - * Allocate memory for the VSI (virtual station interface) and their - * associated queues, rings and the descriptors associated with each, - * called only once at attach. - * - **********************************************************************/ -static int -ixl_setup_stations(struct ixl_pf *pf) -{ - device_t dev = pf->dev; - struct ixl_vsi *vsi; - struct ixl_queue *que; - struct tx_ring *txr; - struct rx_ring *rxr; - int rsize, tsize; - int error = I40E_SUCCESS; - - vsi = &pf->vsi; - vsi->back = (void *)pf; - vsi->hw = &pf->hw; - vsi->id = 0; - vsi->num_vlans = 0; - vsi->back = pf; - - /* Get memory for the station queues */ - if (!(vsi->queues = - (struct ixl_queue *) malloc(sizeof(struct ixl_queue) * - vsi->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { - device_printf(dev, "Unable to allocate queue memory\n"); - error = ENOMEM; - goto early; - } - - for (int i = 0; i < vsi->num_queues; i++) { - que = &vsi->queues[i]; - que->num_desc = ixl_ringsz; - que->me = i; - que->vsi = vsi; - /* mark the queue as active */ - vsi->active_queues |= (u64)1 << que->me; - txr = &que->txr; - txr->que = que; - txr->tail = I40E_QTX_TAIL(que->me); - - /* Initialize the TX lock */ - snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", - device_get_nameunit(dev), que->me); - mtx_init(&txr->mtx, txr->mtx_name, NULL, MTX_DEF); - /* Create the TX descriptor ring */ - tsize = roundup2((que->num_desc * - sizeof(struct i40e_tx_desc)) + - sizeof(u32), DBA_ALIGN); - if (i40e_allocate_dma_mem(&pf->hw, - &txr->dma, i40e_mem_reserved, tsize, DBA_ALIGN)) { - device_printf(dev, - "Unable to allocate TX Descriptor memory\n"); - error = ENOMEM; - goto fail; - } - txr->base = (struct i40e_tx_desc *)txr->dma.va; - bzero((void *)txr->base, tsize); - /* Now allocate transmit soft structs for the ring */ - if (ixl_allocate_tx_data(que)) { - device_printf(dev, - "Critical Failure setting up TX structures\n"); - error = ENOMEM; - goto fail; - } - /* Allocate a buf ring */ - txr->br = buf_ring_alloc(4096, M_DEVBUF, - M_WAITOK, &txr->mtx); - if (txr->br == NULL) { - device_printf(dev, - "Critical Failure setting up TX buf ring\n"); - error = ENOMEM; - goto fail; - } - - /* - * Next the RX queues... - */ - rsize = roundup2(que->num_desc * - sizeof(union i40e_rx_desc), DBA_ALIGN); - rxr = &que->rxr; - rxr->que = que; - rxr->tail = I40E_QRX_TAIL(que->me); - - /* Initialize the RX side lock */ - snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", - device_get_nameunit(dev), que->me); - mtx_init(&rxr->mtx, rxr->mtx_name, NULL, MTX_DEF); - - if (i40e_allocate_dma_mem(&pf->hw, - &rxr->dma, i40e_mem_reserved, rsize, 4096)) { - device_printf(dev, - "Unable to allocate RX Descriptor memory\n"); - error = ENOMEM; - goto fail; - } - rxr->base = (union i40e_rx_desc *)rxr->dma.va; - bzero((void *)rxr->base, rsize); - - /* Allocate receive soft structs for the ring*/ - if (ixl_allocate_rx_data(que)) { - device_printf(dev, - "Critical Failure setting up receive structs\n"); - error = ENOMEM; - goto fail; - } - } - - return (0); - -fail: - for (int i = 0; i < vsi->num_queues; i++) { - que = &vsi->queues[i]; - rxr = &que->rxr; - txr = &que->txr; - if (rxr->base) - i40e_free_dma_mem(&pf->hw, &rxr->dma); - if (txr->base) - i40e_free_dma_mem(&pf->hw, &txr->dma); - } - -early: - return (error); -} - -/* -** Provide a update to the queue RX -** interrupt moderation value. -*/ -static void -ixl_set_queue_rx_itr(struct ixl_queue *que) -{ - struct ixl_vsi *vsi = que->vsi; - struct i40e_hw *hw = vsi->hw; - struct rx_ring *rxr = &que->rxr; - u16 rx_itr; - u16 rx_latency = 0; - int rx_bytes; - - - /* Idle, do nothing */ - if (rxr->bytes == 0) - return; - - if (ixl_dynamic_rx_itr) { - rx_bytes = rxr->bytes/rxr->itr; - rx_itr = rxr->itr; - - /* Adjust latency range */ - switch (rxr->latency) { - case IXL_LOW_LATENCY: - if (rx_bytes > 10) { - rx_latency = IXL_AVE_LATENCY; - rx_itr = IXL_ITR_20K; - } - break; - case IXL_AVE_LATENCY: - if (rx_bytes > 20) { - rx_latency = IXL_BULK_LATENCY; - rx_itr = IXL_ITR_8K; - } else if (rx_bytes <= 10) { - rx_latency = IXL_LOW_LATENCY; - rx_itr = IXL_ITR_100K; - } - break; - case IXL_BULK_LATENCY: - if (rx_bytes <= 20) { - rx_latency = IXL_AVE_LATENCY; - rx_itr = IXL_ITR_20K; - } - break; - } - - rxr->latency = rx_latency; - - if (rx_itr != rxr->itr) { - /* do an exponential smoothing */ - rx_itr = (10 * rx_itr * rxr->itr) / - ((9 * rx_itr) + rxr->itr); - rxr->itr = rx_itr & IXL_MAX_ITR; - wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, - que->me), rxr->itr); - } - } else { /* We may have have toggled to non-dynamic */ - if (vsi->rx_itr_setting & IXL_ITR_DYNAMIC) - vsi->rx_itr_setting = ixl_rx_itr; - /* Update the hardware if needed */ - if (rxr->itr != vsi->rx_itr_setting) { - rxr->itr = vsi->rx_itr_setting; - wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, - que->me), rxr->itr); - } - } - rxr->bytes = 0; - rxr->packets = 0; - return; -} - - -/* -** Provide a update to the queue TX -** interrupt moderation value. -*/ -static void -ixl_set_queue_tx_itr(struct ixl_queue *que) -{ - struct ixl_vsi *vsi = que->vsi; - struct i40e_hw *hw = vsi->hw; - struct tx_ring *txr = &que->txr; - u16 tx_itr; - u16 tx_latency = 0; - int tx_bytes; - - - /* Idle, do nothing */ - if (txr->bytes == 0) - return; - - if (ixl_dynamic_tx_itr) { - tx_bytes = txr->bytes/txr->itr; - tx_itr = txr->itr; - - switch (txr->latency) { - case IXL_LOW_LATENCY: - if (tx_bytes > 10) { - tx_latency = IXL_AVE_LATENCY; - tx_itr = IXL_ITR_20K; - } - break; - case IXL_AVE_LATENCY: - if (tx_bytes > 20) { - tx_latency = IXL_BULK_LATENCY; - tx_itr = IXL_ITR_8K; - } else if (tx_bytes <= 10) { - tx_latency = IXL_LOW_LATENCY; - tx_itr = IXL_ITR_100K; - } - break; - case IXL_BULK_LATENCY: - if (tx_bytes <= 20) { - tx_latency = IXL_AVE_LATENCY; - tx_itr = IXL_ITR_20K; - } - break; - } - - txr->latency = tx_latency; - - if (tx_itr != txr->itr) { - /* do an exponential smoothing */ - tx_itr = (10 * tx_itr * txr->itr) / - ((9 * tx_itr) + txr->itr); - txr->itr = tx_itr & IXL_MAX_ITR; - wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, - que->me), txr->itr); - } - - } else { /* We may have have toggled to non-dynamic */ - if (vsi->tx_itr_setting & IXL_ITR_DYNAMIC) - vsi->tx_itr_setting = ixl_tx_itr; - /* Update the hardware if needed */ - if (txr->itr != vsi->tx_itr_setting) { - txr->itr = vsi->tx_itr_setting; - wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, - que->me), txr->itr); - } - } - txr->bytes = 0; - txr->packets = 0; - return; -} - -#define QUEUE_NAME_LEN 32 - -static void -ixl_add_vsi_sysctls(struct ixl_pf *pf, struct ixl_vsi *vsi, - struct sysctl_ctx_list *ctx, const char *sysctl_name) -{ - struct sysctl_oid *tree; - struct sysctl_oid_list *child; - struct sysctl_oid_list *vsi_list; - - tree = device_get_sysctl_tree(pf->dev); - child = SYSCTL_CHILDREN(tree); - vsi->vsi_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, sysctl_name, - CTLFLAG_RD, NULL, "VSI Number"); - vsi_list = SYSCTL_CHILDREN(vsi->vsi_node); - - ixl_add_sysctls_eth_stats(ctx, vsi_list, &vsi->eth_stats); -} - -static void -ixl_add_hw_stats(struct ixl_pf *pf) -{ - device_t dev = pf->dev; - struct ixl_vsi *vsi = &pf->vsi; - struct ixl_queue *queues = vsi->queues; - struct i40e_hw_port_stats *pf_stats = &pf->stats; - - struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); - struct sysctl_oid *tree = device_get_sysctl_tree(dev); - struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree); - struct sysctl_oid_list *vsi_list; - - struct sysctl_oid *queue_node; - struct sysctl_oid_list *queue_list; - - struct tx_ring *txr; - struct rx_ring *rxr; - char queue_namebuf[QUEUE_NAME_LEN]; - - /* Driver statistics */ - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events", - CTLFLAG_RD, &pf->watchdog_events, - "Watchdog timeouts"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "admin_irq", - CTLFLAG_RD, &pf->admin_irq, - "Admin Queue IRQ Handled"); - - ixl_add_vsi_sysctls(pf, &pf->vsi, ctx, "pf"); - vsi_list = SYSCTL_CHILDREN(pf->vsi.vsi_node); - - /* Queue statistics */ - for (int q = 0; q < vsi->num_queues; q++) { - snprintf(queue_namebuf, QUEUE_NAME_LEN, "que%d", q); - queue_node = SYSCTL_ADD_NODE(ctx, vsi_list, - OID_AUTO, queue_namebuf, CTLFLAG_RD, NULL, "Queue #"); - queue_list = SYSCTL_CHILDREN(queue_node); - - txr = &(queues[q].txr); - rxr = &(queues[q].rxr); - - SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "mbuf_defrag_failed", - CTLFLAG_RD, &(queues[q].mbuf_defrag_failed), - "m_defrag() failed"); - SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "dropped", - CTLFLAG_RD, &(queues[q].dropped_pkts), - "Driver dropped packets"); - SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs", - CTLFLAG_RD, &(queues[q].irqs), - "irqs on this queue"); - SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tso_tx", - CTLFLAG_RD, &(queues[q].tso), - "TSO"); - SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_dma_setup", - CTLFLAG_RD, &(queues[q].tx_dma_setup), - "Driver tx dma failure in xmit"); - SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail", - CTLFLAG_RD, &(txr->no_desc), - "Queue No Descriptor Available"); - SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets", - CTLFLAG_RD, &(txr->total_packets), - "Queue Packets Transmitted"); - SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_bytes", - CTLFLAG_RD, &(txr->tx_bytes), - "Queue Bytes Transmitted"); - SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets", - CTLFLAG_RD, &(rxr->rx_packets), - "Queue Packets Received"); - SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes", - CTLFLAG_RD, &(rxr->rx_bytes), - "Queue Bytes Received"); - } - - /* MAC stats */ - ixl_add_sysctls_mac_stats(ctx, child, pf_stats); -} - -static void -ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *ctx, - struct sysctl_oid_list *child, - struct i40e_eth_stats *eth_stats) -{ - struct ixl_sysctl_info ctls[] = - { - {ð_stats->rx_bytes, "good_octets_rcvd", "Good Octets Received"}, - {ð_stats->rx_unicast, "ucast_pkts_rcvd", - "Unicast Packets Received"}, - {ð_stats->rx_multicast, "mcast_pkts_rcvd", - "Multicast Packets Received"}, - {ð_stats->rx_broadcast, "bcast_pkts_rcvd", - "Broadcast Packets Received"}, - {ð_stats->rx_discards, "rx_discards", "Discarded RX packets"}, - {ð_stats->tx_bytes, "good_octets_txd", "Good Octets Transmitted"}, - {ð_stats->tx_unicast, "ucast_pkts_txd", "Unicast Packets Transmitted"}, - {ð_stats->tx_multicast, "mcast_pkts_txd", - "Multicast Packets Transmitted"}, - {ð_stats->tx_broadcast, "bcast_pkts_txd", - "Broadcast Packets Transmitted"}, - // end - {0,0,0} - }; - - struct ixl_sysctl_info *entry = ctls; - while (entry->stat != NULL) - { - SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, entry->name, - CTLFLAG_RD, entry->stat, - entry->description); - entry++; - } -} - -static void -ixl_add_sysctls_mac_stats(struct sysctl_ctx_list *ctx, - struct sysctl_oid_list *child, - struct i40e_hw_port_stats *stats) -{ - struct sysctl_oid *stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac", - CTLFLAG_RD, NULL, "Mac Statistics"); - struct sysctl_oid_list *stat_list = SYSCTL_CHILDREN(stat_node); - - struct i40e_eth_stats *eth_stats = &stats->eth; - ixl_add_sysctls_eth_stats(ctx, stat_list, eth_stats); - - struct ixl_sysctl_info ctls[] = - { - {&stats->crc_errors, "crc_errors", "CRC Errors"}, - {&stats->illegal_bytes, "illegal_bytes", "Illegal Byte Errors"}, - {&stats->mac_local_faults, "local_faults", "MAC Local Faults"}, - {&stats->mac_remote_faults, "remote_faults", "MAC Remote Faults"}, - {&stats->rx_length_errors, "rx_length_errors", "Receive Length Errors"}, - /* Packet Reception Stats */ - {&stats->rx_size_64, "rx_frames_64", "64 byte frames received"}, - {&stats->rx_size_127, "rx_frames_65_127", "65-127 byte frames received"}, - {&stats->rx_size_255, "rx_frames_128_255", "128-255 byte frames received"}, - {&stats->rx_size_511, "rx_frames_256_511", "256-511 byte frames received"}, - {&stats->rx_size_1023, "rx_frames_512_1023", "512-1023 byte frames received"}, - {&stats->rx_size_1522, "rx_frames_1024_1522", "1024-1522 byte frames received"}, - {&stats->rx_size_big, "rx_frames_big", "1523-9522 byte frames received"}, - {&stats->rx_undersize, "rx_undersize", "Undersized packets received"}, - {&stats->rx_fragments, "rx_fragmented", "Fragmented packets received"}, - {&stats->rx_oversize, "rx_oversized", "Oversized packets received"}, - {&stats->rx_jabber, "rx_jabber", "Received Jabber"}, - {&stats->checksum_error, "checksum_errors", "Checksum Errors"}, - /* Packet Transmission Stats */ - {&stats->tx_size_64, "tx_frames_64", "64 byte frames transmitted"}, - {&stats->tx_size_127, "tx_frames_65_127", "65-127 byte frames transmitted"}, - {&stats->tx_size_255, "tx_frames_128_255", "128-255 byte frames transmitted"}, - {&stats->tx_size_511, "tx_frames_256_511", "256-511 byte frames transmitted"}, - {&stats->tx_size_1023, "tx_frames_512_1023", "512-1023 byte frames transmitted"}, - {&stats->tx_size_1522, "tx_frames_1024_1522", "1024-1522 byte frames transmitted"}, - {&stats->tx_size_big, "tx_frames_big", "1523-9522 byte frames transmitted"}, - /* Flow control */ - {&stats->link_xon_tx, "xon_txd", "Link XON transmitted"}, - {&stats->link_xon_rx, "xon_recvd", "Link XON received"}, - {&stats->link_xoff_tx, "xoff_txd", "Link XOFF transmitted"}, - {&stats->link_xoff_rx, "xoff_recvd", "Link XOFF received"}, - /* End */ - {0,0,0} - }; - - struct ixl_sysctl_info *entry = ctls; - while (entry->stat != NULL) - { - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, entry->name, - CTLFLAG_RD, entry->stat, - entry->description); - entry++; - } -} - - -/* -** ixl_config_rss - setup RSS -** - note this is done for the single vsi -*/ -static void ixl_config_rss(struct ixl_vsi *vsi) -{ - struct ixl_pf *pf = (struct ixl_pf *)vsi->back; - struct i40e_hw *hw = vsi->hw; - u32 lut = 0; - u64 set_hena = 0, hena; - int i, j, que_id; -#ifdef RSS - u32 rss_hash_config; - u32 rss_seed[IXL_KEYSZ]; -#else - u32 rss_seed[IXL_KEYSZ] = {0x41b01687, - 0x183cfd8c, 0xce880440, 0x580cbc3c, - 0x35897377, 0x328b25e1, 0x4fa98922, - 0xb7d90c14, 0xd5bad70d, 0xcd15a2c1}; -#endif - -#ifdef RSS - /* Fetch the configured RSS key */ - rss_getkey((uint8_t *) &rss_seed); -#endif - - /* Fill out hash function seed */ - for (i = 0; i < IXL_KEYSZ; i++) - wr32(hw, I40E_PFQF_HKEY(i), rss_seed[i]); - - /* Enable PCTYPES for RSS: */ -#ifdef RSS - rss_hash_config = rss_gethashconfig(); - if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4) - set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER); - if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4) - set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP); - if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4) - set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP); - if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6) - set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER); - if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX) - set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6); - if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6) - set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP); - if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6) - set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP); -#else - set_hena = - ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP) | - ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP) | - ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) | - ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) | - ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV4) | - ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP) | - ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP) | - ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) | - ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) | - ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6) | - ((u64)1 << I40E_FILTER_PCTYPE_L2_PAYLOAD); -#endif - hena = (u64)rd32(hw, I40E_PFQF_HENA(0)) | - ((u64)rd32(hw, I40E_PFQF_HENA(1)) << 32); - hena |= set_hena; - wr32(hw, I40E_PFQF_HENA(0), (u32)hena); - wr32(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32)); - - /* Populate the LUT with max no. of queues in round robin fashion */ - for (i = j = 0; i < pf->hw.func_caps.rss_table_size; i++, j++) { - if (j == vsi->num_queues) - j = 0; -#ifdef RSS - /* - * Fetch the RSS bucket id for the given indirection entry. - * Cap it at the number of configured buckets (which is - * num_queues.) - */ - que_id = rss_get_indirection_to_bucket(i); - que_id = que_id % vsi->num_queues; -#else - que_id = j; -#endif - /* lut = 4-byte sliding window of 4 lut entries */ - lut = (lut << 8) | (que_id & - ((0x1 << pf->hw.func_caps.rss_table_entry_width) - 1)); - /* On i = 3, we have 4 entries in lut; write to the register */ - if ((i & 3) == 3) - wr32(hw, I40E_PFQF_HLUT(i >> 2), lut); - } - ixl_flush(hw); -} - - -/* -** This routine is run via an vlan config EVENT, -** it enables us to use the HW Filter table since -** we can get the vlan id. This just creates the -** entry in the soft version of the VFTA, init will -** repopulate the real table. -*/ -static void -ixl_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) -{ - struct ixl_vsi *vsi = ifp->if_softc; - struct i40e_hw *hw = vsi->hw; - struct ixl_pf *pf = (struct ixl_pf *)vsi->back; - - if (ifp->if_softc != arg) /* Not our event */ - return; - - if ((vtag == 0) || (vtag > 4095)) /* Invalid */ - return; - - IXL_PF_LOCK(pf); - ++vsi->num_vlans; - ixl_add_filter(vsi, hw->mac.addr, vtag); - IXL_PF_UNLOCK(pf); -} - -/* -** This routine is run via an vlan -** unconfig EVENT, remove our entry -** in the soft vfta. -*/ -static void -ixl_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) -{ - struct ixl_vsi *vsi = ifp->if_softc; - struct i40e_hw *hw = vsi->hw; - struct ixl_pf *pf = (struct ixl_pf *)vsi->back; - - if (ifp->if_softc != arg) - return; - - if ((vtag == 0) || (vtag > 4095)) /* Invalid */ - return; - - IXL_PF_LOCK(pf); - --vsi->num_vlans; - ixl_del_filter(vsi, hw->mac.addr, vtag); - IXL_PF_UNLOCK(pf); -} - -/* -** This routine updates vlan filters, called by init -** it scans the filter table and then updates the hw -** after a soft reset. -*/ -static void -ixl_setup_vlan_filters(struct ixl_vsi *vsi) -{ - struct ixl_mac_filter *f; - int cnt = 0, flags; - - if (vsi->num_vlans == 0) - return; - /* - ** Scan the filter list for vlan entries, - ** mark them for addition and then call - ** for the AQ update. - */ - SLIST_FOREACH(f, &vsi->ftl, next) { - if (f->flags & IXL_FILTER_VLAN) { - f->flags |= - (IXL_FILTER_ADD | - IXL_FILTER_USED); - cnt++; - } - } - if (cnt == 0) { - printf("setup vlan: no filters found!\n"); - return; - } - flags = IXL_FILTER_VLAN; - flags |= (IXL_FILTER_ADD | IXL_FILTER_USED); - ixl_add_hw_filters(vsi, flags, cnt); - return; -} - -/* -** Initialize filter list and add filters that the hardware -** needs to know about. -*/ -static void -ixl_init_filters(struct ixl_vsi *vsi) -{ - /* Add broadcast address */ - ixl_add_filter(vsi, ixl_bcast_addr, IXL_VLAN_ANY); -} - -/* -** This routine adds mulicast filters -*/ -static void -ixl_add_mc_filter(struct ixl_vsi *vsi, u8 *macaddr) -{ - struct ixl_mac_filter *f; - - /* Does one already exist */ - f = ixl_find_filter(vsi, macaddr, IXL_VLAN_ANY); - if (f != NULL) - return; - - f = ixl_get_filter(vsi); - if (f == NULL) { - printf("WARNING: no filter available!!\n"); - return; - } - bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN); - f->vlan = IXL_VLAN_ANY; - f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED - | IXL_FILTER_MC); - - return; -} - -static void -ixl_reconfigure_filters(struct ixl_vsi *vsi) -{ - - ixl_add_hw_filters(vsi, IXL_FILTER_USED, vsi->num_macs); -} - -/* -** This routine adds macvlan filters -*/ -static void -ixl_add_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan) -{ - struct ixl_mac_filter *f, *tmp; - struct ixl_pf *pf; - device_t dev; - - DEBUGOUT("ixl_add_filter: begin"); - - pf = vsi->back; - dev = pf->dev; - - /* Does one already exist */ - f = ixl_find_filter(vsi, macaddr, vlan); - if (f != NULL) - return; - /* - ** Is this the first vlan being registered, if so we - ** need to remove the ANY filter that indicates we are - ** not in a vlan, and replace that with a 0 filter. - */ - if ((vlan != IXL_VLAN_ANY) && (vsi->num_vlans == 1)) { - tmp = ixl_find_filter(vsi, macaddr, IXL_VLAN_ANY); - if (tmp != NULL) { - ixl_del_filter(vsi, macaddr, IXL_VLAN_ANY); - ixl_add_filter(vsi, macaddr, 0); - } - } - - f = ixl_get_filter(vsi); - if (f == NULL) { - device_printf(dev, "WARNING: no filter available!!\n"); - return; - } - bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN); - f->vlan = vlan; - f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED); - if (f->vlan != IXL_VLAN_ANY) - f->flags |= IXL_FILTER_VLAN; - else - vsi->num_macs++; - - ixl_add_hw_filters(vsi, f->flags, 1); - return; -} - -static void -ixl_del_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan) -{ - struct ixl_mac_filter *f; - - f = ixl_find_filter(vsi, macaddr, vlan); - if (f == NULL) - return; - - f->flags |= IXL_FILTER_DEL; - ixl_del_hw_filters(vsi, 1); - vsi->num_macs--; - - /* Check if this is the last vlan removal */ - if (vlan != IXL_VLAN_ANY && vsi->num_vlans == 0) { - /* Switch back to a non-vlan filter */ - ixl_del_filter(vsi, macaddr, 0); - ixl_add_filter(vsi, macaddr, IXL_VLAN_ANY); - } - return; -} - -/* -** Find the filter with both matching mac addr and vlan id -*/ -static struct ixl_mac_filter * -ixl_find_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan) -{ - struct ixl_mac_filter *f; - bool match = FALSE; - - SLIST_FOREACH(f, &vsi->ftl, next) { - if (!cmp_etheraddr(f->macaddr, macaddr)) - continue; - if (f->vlan == vlan) { - match = TRUE; - break; - } - } - - if (!match) - f = NULL; - return (f); -} - -/* -** This routine takes additions to the vsi filter -** table and creates an Admin Queue call to create -** the filters in the hardware. -*/ -static void -ixl_add_hw_filters(struct ixl_vsi *vsi, int flags, int cnt) -{ - struct i40e_aqc_add_macvlan_element_data *a, *b; - struct ixl_mac_filter *f; - struct ixl_pf *pf; - struct i40e_hw *hw; - device_t dev; - int err, j = 0; - - pf = vsi->back; - dev = pf->dev; - hw = &pf->hw; - IXL_PF_LOCK_ASSERT(pf); - - a = malloc(sizeof(struct i40e_aqc_add_macvlan_element_data) * cnt, - M_DEVBUF, M_NOWAIT | M_ZERO); - if (a == NULL) { - device_printf(dev, "add_hw_filters failed to get memory\n"); - return; - } - - /* - ** Scan the filter list, each time we find one - ** we add it to the admin queue array and turn off - ** the add bit. - */ - SLIST_FOREACH(f, &vsi->ftl, next) { - if (f->flags == flags) { - b = &a[j]; // a pox on fvl long names :) - bcopy(f->macaddr, b->mac_addr, ETHER_ADDR_LEN); - if (f->vlan == IXL_VLAN_ANY) { - b->vlan_tag = 0; - b->flags = I40E_AQC_MACVLAN_ADD_IGNORE_VLAN; - } else { - b->vlan_tag = f->vlan; - b->flags = 0; - } - b->flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH; - f->flags &= ~IXL_FILTER_ADD; - j++; - } - if (j == cnt) - break; - } - if (j > 0) { - err = i40e_aq_add_macvlan(hw, vsi->seid, a, j, NULL); - if (err) - device_printf(dev, "aq_add_macvlan err %d, " - "aq_error %d\n", err, hw->aq.asq_last_status); - else - vsi->hw_filters_add += j; - } - free(a, M_DEVBUF); - return; -} - -/* -** This routine takes removals in the vsi filter -** table and creates an Admin Queue call to delete -** the filters in the hardware. -*/ -static void -ixl_del_hw_filters(struct ixl_vsi *vsi, int cnt) -{ - struct i40e_aqc_remove_macvlan_element_data *d, *e; - struct ixl_pf *pf; - struct i40e_hw *hw; - device_t dev; - struct ixl_mac_filter *f, *f_temp; - int err, j = 0; - - DEBUGOUT("ixl_del_hw_filters: begin\n"); - - pf = vsi->back; - hw = &pf->hw; - dev = pf->dev; - - d = malloc(sizeof(struct i40e_aqc_remove_macvlan_element_data) * cnt, - M_DEVBUF, M_NOWAIT | M_ZERO); - if (d == NULL) { - printf("del hw filter failed to get memory\n"); - return; - } - - SLIST_FOREACH_SAFE(f, &vsi->ftl, next, f_temp) { - if (f->flags & IXL_FILTER_DEL) { - e = &d[j]; // a pox on fvl long names :) - bcopy(f->macaddr, e->mac_addr, ETHER_ADDR_LEN); - e->vlan_tag = (f->vlan == IXL_VLAN_ANY ? 0 : f->vlan); - e->flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; - /* delete entry from vsi list */ - SLIST_REMOVE(&vsi->ftl, f, ixl_mac_filter, next); - free(f, M_DEVBUF); - j++; - } - if (j == cnt) - break; - } - if (j > 0) { - err = i40e_aq_remove_macvlan(hw, vsi->seid, d, j, NULL); - /* NOTE: returns ENOENT every time but seems to work fine, - so we'll ignore that specific error. */ - // TODO: Does this still occur on current firmwares? - if (err && hw->aq.asq_last_status != I40E_AQ_RC_ENOENT) { - int sc = 0; - for (int i = 0; i < j; i++) - sc += (!d[i].error_code); - vsi->hw_filters_del += sc; - device_printf(dev, - "Failed to remove %d/%d filters, aq error %d\n", - j - sc, j, hw->aq.asq_last_status); - } else - vsi->hw_filters_del += j; - } - free(d, M_DEVBUF); - - DEBUGOUT("ixl_del_hw_filters: end\n"); - return; -} - -static int -ixl_enable_rings(struct ixl_vsi *vsi) -{ - struct ixl_pf *pf = vsi->back; - struct i40e_hw *hw = &pf->hw; - int index, error; - u32 reg; - - error = 0; - for (int i = 0; i < vsi->num_queues; i++) { - index = vsi->first_queue + i; - i40e_pre_tx_queue_cfg(hw, index, TRUE); - - reg = rd32(hw, I40E_QTX_ENA(index)); - reg |= I40E_QTX_ENA_QENA_REQ_MASK | - I40E_QTX_ENA_QENA_STAT_MASK; - wr32(hw, I40E_QTX_ENA(index), reg); - /* Verify the enable took */ - for (int j = 0; j < 10; j++) { - reg = rd32(hw, I40E_QTX_ENA(index)); - if (reg & I40E_QTX_ENA_QENA_STAT_MASK) - break; - i40e_msec_delay(10); - } - if ((reg & I40E_QTX_ENA_QENA_STAT_MASK) == 0) { - device_printf(pf->dev, "TX queue %d disabled!\n", - index); - error = ETIMEDOUT; - } - - reg = rd32(hw, I40E_QRX_ENA(index)); - reg |= I40E_QRX_ENA_QENA_REQ_MASK | - I40E_QRX_ENA_QENA_STAT_MASK; - wr32(hw, I40E_QRX_ENA(index), reg); - /* Verify the enable took */ - for (int j = 0; j < 10; j++) { - reg = rd32(hw, I40E_QRX_ENA(index)); - if (reg & I40E_QRX_ENA_QENA_STAT_MASK) - break; - i40e_msec_delay(10); - } - if ((reg & I40E_QRX_ENA_QENA_STAT_MASK) == 0) { - device_printf(pf->dev, "RX queue %d disabled!\n", - index); - error = ETIMEDOUT; - } - } - - return (error); -} - -static int -ixl_disable_rings(struct ixl_vsi *vsi) -{ - struct ixl_pf *pf = vsi->back; - struct i40e_hw *hw = &pf->hw; - int index, error; - u32 reg; - - error = 0; - for (int i = 0; i < vsi->num_queues; i++) { - index = vsi->first_queue + i; - - i40e_pre_tx_queue_cfg(hw, index, FALSE); - i40e_usec_delay(500); - - reg = rd32(hw, I40E_QTX_ENA(index)); - reg &= ~I40E_QTX_ENA_QENA_REQ_MASK; - wr32(hw, I40E_QTX_ENA(index), reg); - /* Verify the disable took */ - for (int j = 0; j < 10; j++) { - reg = rd32(hw, I40E_QTX_ENA(index)); - if (!(reg & I40E_QTX_ENA_QENA_STAT_MASK)) - break; - i40e_msec_delay(10); - } - if (reg & I40E_QTX_ENA_QENA_STAT_MASK) { - device_printf(pf->dev, "TX queue %d still enabled!\n", - index); - error = ETIMEDOUT; - } - - reg = rd32(hw, I40E_QRX_ENA(index)); - reg &= ~I40E_QRX_ENA_QENA_REQ_MASK; - wr32(hw, I40E_QRX_ENA(index), reg); - /* Verify the disable took */ - for (int j = 0; j < 10; j++) { - reg = rd32(hw, I40E_QRX_ENA(index)); - if (!(reg & I40E_QRX_ENA_QENA_STAT_MASK)) - break; - i40e_msec_delay(10); - } - if (reg & I40E_QRX_ENA_QENA_STAT_MASK) { - device_printf(pf->dev, "RX queue %d still enabled!\n", - index); - error = ETIMEDOUT; - } - } - - return (error); -} - -/** - * ixl_handle_mdd_event - * - * Called from interrupt handler to identify possibly malicious vfs - * (But also detects events from the PF, as well) - **/ -static void ixl_handle_mdd_event(struct ixl_pf *pf) -{ - struct i40e_hw *hw = &pf->hw; - device_t dev = pf->dev; - bool mdd_detected = false; - bool pf_mdd_detected = false; - u32 reg; - - /* find what triggered the MDD event */ - reg = rd32(hw, I40E_GL_MDET_TX); - if (reg & I40E_GL_MDET_TX_VALID_MASK) { - u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >> - I40E_GL_MDET_TX_PF_NUM_SHIFT; - u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >> - I40E_GL_MDET_TX_EVENT_SHIFT; - u8 queue = (reg & I40E_GL_MDET_TX_QUEUE_MASK) >> - I40E_GL_MDET_TX_QUEUE_SHIFT; - device_printf(dev, - "Malicious Driver Detection event 0x%02x" - " on TX queue %d pf number 0x%02x\n", - event, queue, pf_num); - wr32(hw, I40E_GL_MDET_TX, 0xffffffff); - mdd_detected = true; - } - reg = rd32(hw, I40E_GL_MDET_RX); - if (reg & I40E_GL_MDET_RX_VALID_MASK) { - u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >> - I40E_GL_MDET_RX_FUNCTION_SHIFT; - u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >> - I40E_GL_MDET_RX_EVENT_SHIFT; - u8 queue = (reg & I40E_GL_MDET_RX_QUEUE_MASK) >> - I40E_GL_MDET_RX_QUEUE_SHIFT; - device_printf(dev, - "Malicious Driver Detection event 0x%02x" - " on RX queue %d of function 0x%02x\n", - event, queue, func); - wr32(hw, I40E_GL_MDET_RX, 0xffffffff); - mdd_detected = true; - } - - if (mdd_detected) { - reg = rd32(hw, I40E_PF_MDET_TX); - if (reg & I40E_PF_MDET_TX_VALID_MASK) { - wr32(hw, I40E_PF_MDET_TX, 0xFFFF); - device_printf(dev, - "MDD TX event is for this function 0x%08x", - reg); - pf_mdd_detected = true; - } - reg = rd32(hw, I40E_PF_MDET_RX); - if (reg & I40E_PF_MDET_RX_VALID_MASK) { - wr32(hw, I40E_PF_MDET_RX, 0xFFFF); - device_printf(dev, - "MDD RX event is for this function 0x%08x", - reg); - pf_mdd_detected = true; - } - } - - /* re-enable mdd interrupt cause */ - reg = rd32(hw, I40E_PFINT_ICR0_ENA); - reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; - wr32(hw, I40E_PFINT_ICR0_ENA, reg); - ixl_flush(hw); -} - -static void -ixl_enable_intr(struct ixl_vsi *vsi) -{ - struct i40e_hw *hw = vsi->hw; - struct ixl_queue *que = vsi->queues; - - if (ixl_enable_msix) { - ixl_enable_adminq(hw); - for (int i = 0; i < vsi->num_queues; i++, que++) - ixl_enable_queue(hw, que->me); - } else - ixl_enable_legacy(hw); -} - -static void -ixl_disable_rings_intr(struct ixl_vsi *vsi) -{ - struct i40e_hw *hw = vsi->hw; - struct ixl_queue *que = vsi->queues; - - for (int i = 0; i < vsi->num_queues; i++, que++) - ixl_disable_queue(hw, que->me); -} - -static void -ixl_disable_intr(struct ixl_vsi *vsi) -{ - struct i40e_hw *hw = vsi->hw; - - if (ixl_enable_msix) - ixl_disable_adminq(hw); - else - ixl_disable_legacy(hw); -} - -static void -ixl_enable_adminq(struct i40e_hw *hw) -{ - u32 reg; - - reg = I40E_PFINT_DYN_CTL0_INTENA_MASK | - I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | - (IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT); - wr32(hw, I40E_PFINT_DYN_CTL0, reg); - ixl_flush(hw); - return; -} - -static void -ixl_disable_adminq(struct i40e_hw *hw) -{ - u32 reg; - - reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT; - wr32(hw, I40E_PFINT_DYN_CTL0, reg); - - return; -} - -static void -ixl_enable_queue(struct i40e_hw *hw, int id) -{ - u32 reg; - - reg = I40E_PFINT_DYN_CTLN_INTENA_MASK | - I40E_PFINT_DYN_CTLN_CLEARPBA_MASK | - (IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT); - wr32(hw, I40E_PFINT_DYN_CTLN(id), reg); -} - -static void -ixl_disable_queue(struct i40e_hw *hw, int id) -{ - u32 reg; - - reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT; - wr32(hw, I40E_PFINT_DYN_CTLN(id), reg); - - return; -} - -static void -ixl_enable_legacy(struct i40e_hw *hw) -{ - u32 reg; - reg = I40E_PFINT_DYN_CTL0_INTENA_MASK | - I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | - (IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT); - wr32(hw, I40E_PFINT_DYN_CTL0, reg); -} - -static void -ixl_disable_legacy(struct i40e_hw *hw) -{ - u32 reg; - - reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT; - wr32(hw, I40E_PFINT_DYN_CTL0, reg); - - return; -} - -static void -ixl_update_stats_counters(struct ixl_pf *pf) -{ - struct i40e_hw *hw = &pf->hw; - struct ixl_vsi *vsi = &pf->vsi; - struct ixl_vf *vf; - - struct i40e_hw_port_stats *nsd = &pf->stats; - struct i40e_hw_port_stats *osd = &pf->stats_offsets; - - /* Update hw stats */ - ixl_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port), - pf->stat_offsets_loaded, - &osd->crc_errors, &nsd->crc_errors); - ixl_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port), - pf->stat_offsets_loaded, - &osd->illegal_bytes, &nsd->illegal_bytes); - ixl_stat_update48(hw, I40E_GLPRT_GORCH(hw->port), - I40E_GLPRT_GORCL(hw->port), - pf->stat_offsets_loaded, - &osd->eth.rx_bytes, &nsd->eth.rx_bytes); - ixl_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port), - I40E_GLPRT_GOTCL(hw->port), - pf->stat_offsets_loaded, - &osd->eth.tx_bytes, &nsd->eth.tx_bytes); - ixl_stat_update32(hw, I40E_GLPRT_RDPC(hw->port), - pf->stat_offsets_loaded, - &osd->eth.rx_discards, - &nsd->eth.rx_discards); - ixl_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port), - I40E_GLPRT_UPRCL(hw->port), - pf->stat_offsets_loaded, - &osd->eth.rx_unicast, - &nsd->eth.rx_unicast); - ixl_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port), - I40E_GLPRT_UPTCL(hw->port), - pf->stat_offsets_loaded, - &osd->eth.tx_unicast, - &nsd->eth.tx_unicast); - ixl_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port), - I40E_GLPRT_MPRCL(hw->port), - pf->stat_offsets_loaded, - &osd->eth.rx_multicast, - &nsd->eth.rx_multicast); - ixl_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port), - I40E_GLPRT_MPTCL(hw->port), - pf->stat_offsets_loaded, - &osd->eth.tx_multicast, - &nsd->eth.tx_multicast); - ixl_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port), - I40E_GLPRT_BPRCL(hw->port), - pf->stat_offsets_loaded, - &osd->eth.rx_broadcast, - &nsd->eth.rx_broadcast); - ixl_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port), - I40E_GLPRT_BPTCL(hw->port), - pf->stat_offsets_loaded, - &osd->eth.tx_broadcast, - &nsd->eth.tx_broadcast); - - ixl_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port), - pf->stat_offsets_loaded, - &osd->tx_dropped_link_down, - &nsd->tx_dropped_link_down); - ixl_stat_update32(hw, I40E_GLPRT_MLFC(hw->port), - pf->stat_offsets_loaded, - &osd->mac_local_faults, - &nsd->mac_local_faults); - ixl_stat_update32(hw, I40E_GLPRT_MRFC(hw->port), - pf->stat_offsets_loaded, - &osd->mac_remote_faults, - &nsd->mac_remote_faults); - ixl_stat_update32(hw, I40E_GLPRT_RLEC(hw->port), - pf->stat_offsets_loaded, - &osd->rx_length_errors, - &nsd->rx_length_errors); - - /* Flow control (LFC) stats */ - ixl_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port), - pf->stat_offsets_loaded, - &osd->link_xon_rx, &nsd->link_xon_rx); - ixl_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port), - pf->stat_offsets_loaded, - &osd->link_xon_tx, &nsd->link_xon_tx); - ixl_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port), - pf->stat_offsets_loaded, - &osd->link_xoff_rx, &nsd->link_xoff_rx); - ixl_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port), - pf->stat_offsets_loaded, - &osd->link_xoff_tx, &nsd->link_xoff_tx); - - /* Packet size stats rx */ - ixl_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port), - I40E_GLPRT_PRC64L(hw->port), - pf->stat_offsets_loaded, - &osd->rx_size_64, &nsd->rx_size_64); - ixl_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port), - I40E_GLPRT_PRC127L(hw->port), - pf->stat_offsets_loaded, - &osd->rx_size_127, &nsd->rx_size_127); - ixl_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port), - I40E_GLPRT_PRC255L(hw->port), - pf->stat_offsets_loaded, - &osd->rx_size_255, &nsd->rx_size_255); - ixl_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port), - I40E_GLPRT_PRC511L(hw->port), - pf->stat_offsets_loaded, - &osd->rx_size_511, &nsd->rx_size_511); - ixl_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port), - I40E_GLPRT_PRC1023L(hw->port), - pf->stat_offsets_loaded, - &osd->rx_size_1023, &nsd->rx_size_1023); - ixl_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port), - I40E_GLPRT_PRC1522L(hw->port), - pf->stat_offsets_loaded, - &osd->rx_size_1522, &nsd->rx_size_1522); - ixl_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port), - I40E_GLPRT_PRC9522L(hw->port), - pf->stat_offsets_loaded, - &osd->rx_size_big, &nsd->rx_size_big); - - /* Packet size stats tx */ - ixl_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port), - I40E_GLPRT_PTC64L(hw->port), - pf->stat_offsets_loaded, - &osd->tx_size_64, &nsd->tx_size_64); - ixl_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port), - I40E_GLPRT_PTC127L(hw->port), - pf->stat_offsets_loaded, - &osd->tx_size_127, &nsd->tx_size_127); - ixl_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port), - I40E_GLPRT_PTC255L(hw->port), - pf->stat_offsets_loaded, - &osd->tx_size_255, &nsd->tx_size_255); - ixl_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port), - I40E_GLPRT_PTC511L(hw->port), - pf->stat_offsets_loaded, - &osd->tx_size_511, &nsd->tx_size_511); - ixl_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port), - I40E_GLPRT_PTC1023L(hw->port), - pf->stat_offsets_loaded, - &osd->tx_size_1023, &nsd->tx_size_1023); - ixl_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port), - I40E_GLPRT_PTC1522L(hw->port), - pf->stat_offsets_loaded, - &osd->tx_size_1522, &nsd->tx_size_1522); - ixl_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port), - I40E_GLPRT_PTC9522L(hw->port), - pf->stat_offsets_loaded, - &osd->tx_size_big, &nsd->tx_size_big); - - ixl_stat_update32(hw, I40E_GLPRT_RUC(hw->port), - pf->stat_offsets_loaded, - &osd->rx_undersize, &nsd->rx_undersize); - ixl_stat_update32(hw, I40E_GLPRT_RFC(hw->port), - pf->stat_offsets_loaded, - &osd->rx_fragments, &nsd->rx_fragments); - ixl_stat_update32(hw, I40E_GLPRT_ROC(hw->port), - pf->stat_offsets_loaded, - &osd->rx_oversize, &nsd->rx_oversize); - ixl_stat_update32(hw, I40E_GLPRT_RJC(hw->port), - pf->stat_offsets_loaded, - &osd->rx_jabber, &nsd->rx_jabber); - pf->stat_offsets_loaded = true; - /* End hw stats */ - - /* Update vsi stats */ - ixl_update_vsi_stats(vsi); - - for (int i = 0; i < pf->num_vfs; i++) { - vf = &pf->vfs[i]; - if (vf->vf_flags & VF_FLAG_ENABLED) - ixl_update_eth_stats(&pf->vfs[i].vsi); - } -} - -/* -** Tasklet handler for MSIX Adminq interrupts -** - do outside interrupt since it might sleep -*/ -static void -ixl_do_adminq(void *context, int pending) -{ - struct ixl_pf *pf = context; - struct i40e_hw *hw = &pf->hw; - struct ixl_vsi *vsi = &pf->vsi; - struct i40e_arq_event_info event; - i40e_status ret; - u32 reg, loop = 0; - u16 opcode, result; - - event.buf_len = IXL_AQ_BUF_SZ; - event.msg_buf = malloc(event.buf_len, - M_DEVBUF, M_NOWAIT | M_ZERO); - if (!event.msg_buf) { - printf("Unable to allocate adminq memory\n"); - return; - } - - IXL_PF_LOCK(pf); - /* clean and process any events */ - do { - ret = i40e_clean_arq_element(hw, &event, &result); - if (ret) - break; - opcode = LE16_TO_CPU(event.desc.opcode); - switch (opcode) { - case i40e_aqc_opc_get_link_status: - ixl_link_event(pf, &event); - ixl_update_link_status(pf); - break; - case i40e_aqc_opc_send_msg_to_pf: -#ifdef PCI_IOV - ixl_handle_vf_msg(pf, &event); -#endif - break; - case i40e_aqc_opc_event_lan_overflow: - break; - default: -#ifdef IXL_DEBUG - printf("AdminQ unknown event %x\n", opcode); -#endif - break; - } - - } while (result && (loop++ < IXL_ADM_LIMIT)); - - reg = rd32(hw, I40E_PFINT_ICR0_ENA); - reg |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK; - wr32(hw, I40E_PFINT_ICR0_ENA, reg); - free(event.msg_buf, M_DEVBUF); - - /* - * If there are still messages to process, reschedule ourselves. - * Otherwise, re-enable our interrupt and go to sleep. - */ - if (result > 0) - taskqueue_enqueue(pf->tq, &pf->adminq); - else - ixl_enable_intr(vsi); - - IXL_PF_UNLOCK(pf); -} - -#ifdef IXL_DEBUG_SYSCTL -static int -ixl_debug_info(SYSCTL_HANDLER_ARGS) -{ - struct ixl_pf *pf; - int error, input = 0; - - error = sysctl_handle_int(oidp, &input, 0, req); - - if (error || !req->newptr) - return (error); - - if (input == 1) { - pf = (struct ixl_pf *)arg1; - ixl_print_debug_info(pf); - } - - return (error); -} - -static void -ixl_print_debug_info(struct ixl_pf *pf) -{ - struct i40e_hw *hw = &pf->hw; - struct ixl_vsi *vsi = &pf->vsi; - struct ixl_queue *que = vsi->queues; - struct rx_ring *rxr = &que->rxr; - struct tx_ring *txr = &que->txr; - u32 reg; - - - printf("Queue irqs = %jx\n", (uintmax_t)que->irqs); - printf("AdminQ irqs = %jx\n", (uintmax_t)pf->admin_irq); - printf("RX next check = %x\n", rxr->next_check); - printf("RX not ready = %jx\n", (uintmax_t)rxr->not_done); - printf("RX packets = %jx\n", (uintmax_t)rxr->rx_packets); - printf("TX desc avail = %x\n", txr->avail); - - reg = rd32(hw, I40E_GLV_GORCL(0xc)); - printf("RX Bytes = %x\n", reg); - reg = rd32(hw, I40E_GLPRT_GORCL(hw->port)); - printf("Port RX Bytes = %x\n", reg); - reg = rd32(hw, I40E_GLV_RDPC(0xc)); - printf("RX discard = %x\n", reg); - reg = rd32(hw, I40E_GLPRT_RDPC(hw->port)); - printf("Port RX discard = %x\n", reg); - - reg = rd32(hw, I40E_GLV_TEPC(0xc)); - printf("TX errors = %x\n", reg); - reg = rd32(hw, I40E_GLV_GOTCL(0xc)); - printf("TX Bytes = %x\n", reg); - - reg = rd32(hw, I40E_GLPRT_RUC(hw->port)); - printf("RX undersize = %x\n", reg); - reg = rd32(hw, I40E_GLPRT_RFC(hw->port)); - printf("RX fragments = %x\n", reg); - reg = rd32(hw, I40E_GLPRT_ROC(hw->port)); - printf("RX oversize = %x\n", reg); - reg = rd32(hw, I40E_GLPRT_RLEC(hw->port)); - printf("RX length error = %x\n", reg); - reg = rd32(hw, I40E_GLPRT_MRFC(hw->port)); - printf("mac remote fault = %x\n", reg); - reg = rd32(hw, I40E_GLPRT_MLFC(hw->port)); - printf("mac local fault = %x\n", reg); -} -#endif - -/** - * Update VSI-specific ethernet statistics counters. - **/ -void ixl_update_eth_stats(struct ixl_vsi *vsi) -{ - struct ixl_pf *pf = (struct ixl_pf *)vsi->back; - struct i40e_hw *hw = &pf->hw; - struct i40e_eth_stats *es; - struct i40e_eth_stats *oes; - struct i40e_hw_port_stats *nsd; - u16 stat_idx = vsi->info.stat_counter_idx; - - es = &vsi->eth_stats; - oes = &vsi->eth_stats_offsets; - nsd = &pf->stats; - - /* Gather up the stats that the hw collects */ - ixl_stat_update32(hw, I40E_GLV_TEPC(stat_idx), - vsi->stat_offsets_loaded, - &oes->tx_errors, &es->tx_errors); - ixl_stat_update32(hw, I40E_GLV_RDPC(stat_idx), - vsi->stat_offsets_loaded, - &oes->rx_discards, &es->rx_discards); - - ixl_stat_update48(hw, I40E_GLV_GORCH(stat_idx), - I40E_GLV_GORCL(stat_idx), - vsi->stat_offsets_loaded, - &oes->rx_bytes, &es->rx_bytes); - ixl_stat_update48(hw, I40E_GLV_UPRCH(stat_idx), - I40E_GLV_UPRCL(stat_idx), - vsi->stat_offsets_loaded, - &oes->rx_unicast, &es->rx_unicast); - ixl_stat_update48(hw, I40E_GLV_MPRCH(stat_idx), - I40E_GLV_MPRCL(stat_idx), - vsi->stat_offsets_loaded, - &oes->rx_multicast, &es->rx_multicast); - ixl_stat_update48(hw, I40E_GLV_BPRCH(stat_idx), - I40E_GLV_BPRCL(stat_idx), - vsi->stat_offsets_loaded, - &oes->rx_broadcast, &es->rx_broadcast); - - ixl_stat_update48(hw, I40E_GLV_GOTCH(stat_idx), - I40E_GLV_GOTCL(stat_idx), - vsi->stat_offsets_loaded, - &oes->tx_bytes, &es->tx_bytes); - ixl_stat_update48(hw, I40E_GLV_UPTCH(stat_idx), - I40E_GLV_UPTCL(stat_idx), - vsi->stat_offsets_loaded, - &oes->tx_unicast, &es->tx_unicast); - ixl_stat_update48(hw, I40E_GLV_MPTCH(stat_idx), - I40E_GLV_MPTCL(stat_idx), - vsi->stat_offsets_loaded, - &oes->tx_multicast, &es->tx_multicast); - ixl_stat_update48(hw, I40E_GLV_BPTCH(stat_idx), - I40E_GLV_BPTCL(stat_idx), - vsi->stat_offsets_loaded, - &oes->tx_broadcast, &es->tx_broadcast); - vsi->stat_offsets_loaded = true; -} - -static void -ixl_update_vsi_stats(struct ixl_vsi *vsi) -{ - struct ixl_pf *pf; - struct ifnet *ifp; - struct i40e_eth_stats *es; - u64 tx_discards; - - struct i40e_hw_port_stats *nsd; - - pf = vsi->back; - ifp = vsi->ifp; - es = &vsi->eth_stats; - nsd = &pf->stats; - - ixl_update_eth_stats(vsi); - - tx_discards = es->tx_discards + nsd->tx_dropped_link_down; - for (int i = 0; i < vsi->num_queues; i++) - tx_discards += vsi->queues[i].txr.br->br_drops; - - /* Update ifnet stats */ - IXL_SET_IPACKETS(vsi, es->rx_unicast + - es->rx_multicast + - es->rx_broadcast); - IXL_SET_OPACKETS(vsi, es->tx_unicast + - es->tx_multicast + - es->tx_broadcast); - IXL_SET_IBYTES(vsi, es->rx_bytes); - IXL_SET_OBYTES(vsi, es->tx_bytes); - IXL_SET_IMCASTS(vsi, es->rx_multicast); - IXL_SET_OMCASTS(vsi, es->tx_multicast); - - IXL_SET_IERRORS(vsi, nsd->crc_errors + nsd->illegal_bytes + - nsd->rx_undersize + nsd->rx_oversize + nsd->rx_fragments + - nsd->rx_jabber); - IXL_SET_OERRORS(vsi, es->tx_errors); - IXL_SET_IQDROPS(vsi, es->rx_discards + nsd->eth.rx_discards); - IXL_SET_OQDROPS(vsi, tx_discards); - IXL_SET_NOPROTO(vsi, es->rx_unknown_protocol); - IXL_SET_COLLISIONS(vsi, 0); -} - -/** - * Reset all of the stats for the given pf - **/ -void ixl_pf_reset_stats(struct ixl_pf *pf) -{ - bzero(&pf->stats, sizeof(struct i40e_hw_port_stats)); - bzero(&pf->stats_offsets, sizeof(struct i40e_hw_port_stats)); - pf->stat_offsets_loaded = false; -} - -/** - * Resets all stats of the given vsi - **/ -void ixl_vsi_reset_stats(struct ixl_vsi *vsi) -{ - bzero(&vsi->eth_stats, sizeof(struct i40e_eth_stats)); - bzero(&vsi->eth_stats_offsets, sizeof(struct i40e_eth_stats)); - vsi->stat_offsets_loaded = false; -} - -/** - * Read and update a 48 bit stat from the hw - * - * Since the device stats are not reset at PFReset, they likely will not - * be zeroed when the driver starts. We'll save the first values read - * and use them as offsets to be subtracted from the raw values in order - * to report stats that count from zero. - **/ -static void -ixl_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg, - bool offset_loaded, u64 *offset, u64 *stat) -{ - u64 new_data; - -#if defined(__FreeBSD__) && (__FreeBSD_version >= 1000000) && defined(__amd64__) - new_data = rd64(hw, loreg); -#else - /* - * Use two rd32's instead of one rd64; FreeBSD versions before - * 10 don't support 8 byte bus reads/writes. - */ - new_data = rd32(hw, loreg); - new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32; -#endif - - if (!offset_loaded) - *offset = new_data; - if (new_data >= *offset) - *stat = new_data - *offset; - else - *stat = (new_data + ((u64)1 << 48)) - *offset; - *stat &= 0xFFFFFFFFFFFFULL; -} - -/** - * Read and update a 32 bit stat from the hw - **/ -static void -ixl_stat_update32(struct i40e_hw *hw, u32 reg, - bool offset_loaded, u64 *offset, u64 *stat) -{ - u32 new_data; - - new_data = rd32(hw, reg); - if (!offset_loaded) - *offset = new_data; - if (new_data >= *offset) - *stat = (u32)(new_data - *offset); - else - *stat = (u32)((new_data + ((u64)1 << 32)) - *offset); -} - -/* -** Set flow control using sysctl: -** 0 - off -** 1 - rx pause -** 2 - tx pause -** 3 - full -*/ -static int -ixl_set_flowcntl(SYSCTL_HANDLER_ARGS) -{ - /* - * TODO: ensure flow control is disabled if - * priority flow control is enabled - * - * TODO: ensure tx CRC by hardware should be enabled - * if tx flow control is enabled. - */ - struct ixl_pf *pf = (struct ixl_pf *)arg1; - struct i40e_hw *hw = &pf->hw; - device_t dev = pf->dev; - int error = 0; - enum i40e_status_code aq_error = 0; - u8 fc_aq_err = 0; - - /* Get request */ - error = sysctl_handle_int(oidp, &pf->fc, 0, req); - if ((error) || (req->newptr == NULL)) - return (error); - if (pf->fc < 0 || pf->fc > 3) { - device_printf(dev, - "Invalid fc mode; valid modes are 0 through 3\n"); - return (EINVAL); - } - - /* - ** Changing flow control mode currently does not work on - ** 40GBASE-CR4 PHYs - */ - if (hw->phy.link_info.phy_type == I40E_PHY_TYPE_40GBASE_CR4 - || hw->phy.link_info.phy_type == I40E_PHY_TYPE_40GBASE_CR4_CU) { - device_printf(dev, "Changing flow control mode unsupported" - " on 40GBase-CR4 media.\n"); - return (ENODEV); - } - - /* Set fc ability for port */ - hw->fc.requested_mode = pf->fc; - aq_error = i40e_set_fc(hw, &fc_aq_err, TRUE); - if (aq_error) { - device_printf(dev, - "%s: Error setting new fc mode %d; fc_err %#x\n", - __func__, aq_error, fc_aq_err); - return (EAGAIN); - } - - return (0); -} - -static int -ixl_current_speed(SYSCTL_HANDLER_ARGS) -{ - struct ixl_pf *pf = (struct ixl_pf *)arg1; - struct i40e_hw *hw = &pf->hw; - int error = 0, index = 0; - - char *speeds[] = { - "Unknown", - "100M", - "1G", - "10G", - "40G", - "20G" - }; - - ixl_update_link_status(pf); - - switch (hw->phy.link_info.link_speed) { - case I40E_LINK_SPEED_100MB: - index = 1; - break; - case I40E_LINK_SPEED_1GB: - index = 2; - break; - case I40E_LINK_SPEED_10GB: - index = 3; - break; - case I40E_LINK_SPEED_40GB: - index = 4; - break; - case I40E_LINK_SPEED_20GB: - index = 5; - break; - case I40E_LINK_SPEED_UNKNOWN: - default: - index = 0; - break; - } - - error = sysctl_handle_string(oidp, speeds[index], - strlen(speeds[index]), req); - return (error); -} - -static int -ixl_set_advertised_speeds(struct ixl_pf *pf, int speeds) -{ - struct i40e_hw *hw = &pf->hw; - device_t dev = pf->dev; - struct i40e_aq_get_phy_abilities_resp abilities; - struct i40e_aq_set_phy_config config; - enum i40e_status_code aq_error = 0; - - /* Get current capability information */ - aq_error = i40e_aq_get_phy_capabilities(hw, - FALSE, FALSE, &abilities, NULL); - if (aq_error) { - device_printf(dev, - "%s: Error getting phy capabilities %d," - " aq error: %d\n", __func__, aq_error, - hw->aq.asq_last_status); - return (EAGAIN); - } - - /* Prepare new config */ - bzero(&config, sizeof(config)); - config.phy_type = abilities.phy_type; - config.abilities = abilities.abilities - | I40E_AQ_PHY_ENABLE_ATOMIC_LINK; - config.eee_capability = abilities.eee_capability; - config.eeer = abilities.eeer_val; - config.low_power_ctrl = abilities.d3_lpan; - /* Translate into aq cmd link_speed */ - if (speeds & 0x8) - config.link_speed |= I40E_LINK_SPEED_20GB; - if (speeds & 0x4) - config.link_speed |= I40E_LINK_SPEED_10GB; - if (speeds & 0x2) - config.link_speed |= I40E_LINK_SPEED_1GB; - if (speeds & 0x1) - config.link_speed |= I40E_LINK_SPEED_100MB; - - /* Do aq command & restart link */ - aq_error = i40e_aq_set_phy_config(hw, &config, NULL); - if (aq_error) { - device_printf(dev, - "%s: Error setting new phy config %d," - " aq error: %d\n", __func__, aq_error, - hw->aq.asq_last_status); - return (EAGAIN); - } - - /* - ** This seems a bit heavy handed, but we - ** need to get a reinit on some devices - */ - IXL_PF_LOCK(pf); - ixl_stop(pf); - ixl_init_locked(pf); - IXL_PF_UNLOCK(pf); - - return (0); -} - -/* -** Control link advertise speed: -** Flags: -** 0x1 - advertise 100 Mb -** 0x2 - advertise 1G -** 0x4 - advertise 10G -** 0x8 - advertise 20G -** -** Does not work on 40G devices. -*/ -static int -ixl_set_advertise(SYSCTL_HANDLER_ARGS) -{ - struct ixl_pf *pf = (struct ixl_pf *)arg1; - struct i40e_hw *hw = &pf->hw; - device_t dev = pf->dev; - int requested_ls = 0; - int error = 0; - - /* - ** FW doesn't support changing advertised speed - ** for 40G devices; speed is always 40G. - */ - if (i40e_is_40G_device(hw->device_id)) - return (ENODEV); - - /* Read in new mode */ - requested_ls = pf->advertised_speed; - error = sysctl_handle_int(oidp, &requested_ls, 0, req); - if ((error) || (req->newptr == NULL)) - return (error); - /* Check for sane value */ - if (requested_ls < 0x1 || requested_ls > 0xE) { - device_printf(dev, "Invalid advertised speed; " - "valid modes are 0x1 through 0xE\n"); - return (EINVAL); - } - /* Then check for validity based on adapter type */ - switch (hw->device_id) { - case I40E_DEV_ID_10G_BASE_T: - if (requested_ls & 0x8) { - device_printf(dev, - "20Gbs speed not supported on this device.\n"); - return (EINVAL); - } - break; - case I40E_DEV_ID_20G_KR2: - if (requested_ls & 0x1) { - device_printf(dev, - "100Mbs speed not supported on this device.\n"); - return (EINVAL); - } - break; - default: - if (requested_ls & ~0x6) { - device_printf(dev, - "Only 1/10Gbs speeds are supported on this device.\n"); - return (EINVAL); - } - break; - } - - /* Exit if no change */ - if (pf->advertised_speed == requested_ls) - return (0); - - error = ixl_set_advertised_speeds(pf, requested_ls); - if (error) - return (error); - - pf->advertised_speed = requested_ls; - ixl_update_link_status(pf); - return (0); -} - -/* -** Get the width and transaction speed of -** the bus this adapter is plugged into. -*/ -static u16 -ixl_get_bus_info(struct i40e_hw *hw, device_t dev) -{ - u16 link; - u32 offset; - - - /* Get the PCI Express Capabilities offset */ - pci_find_cap(dev, PCIY_EXPRESS, &offset); - - /* ...and read the Link Status Register */ - link = pci_read_config(dev, offset + PCIER_LINK_STA, 2); - - switch (link & I40E_PCI_LINK_WIDTH) { - case I40E_PCI_LINK_WIDTH_1: - hw->bus.width = i40e_bus_width_pcie_x1; - break; - case I40E_PCI_LINK_WIDTH_2: - hw->bus.width = i40e_bus_width_pcie_x2; - break; - case I40E_PCI_LINK_WIDTH_4: - hw->bus.width = i40e_bus_width_pcie_x4; - break; - case I40E_PCI_LINK_WIDTH_8: - hw->bus.width = i40e_bus_width_pcie_x8; - break; - default: - hw->bus.width = i40e_bus_width_unknown; - break; - } - - switch (link & I40E_PCI_LINK_SPEED) { - case I40E_PCI_LINK_SPEED_2500: - hw->bus.speed = i40e_bus_speed_2500; - break; - case I40E_PCI_LINK_SPEED_5000: - hw->bus.speed = i40e_bus_speed_5000; - break; - case I40E_PCI_LINK_SPEED_8000: - hw->bus.speed = i40e_bus_speed_8000; - break; - default: - hw->bus.speed = i40e_bus_speed_unknown; - break; - } - - - device_printf(dev,"PCI Express Bus: Speed %s %s\n", - ((hw->bus.speed == i40e_bus_speed_8000) ? "8.0GT/s": - (hw->bus.speed == i40e_bus_speed_5000) ? "5.0GT/s": - (hw->bus.speed == i40e_bus_speed_2500) ? "2.5GT/s":"Unknown"), - (hw->bus.width == i40e_bus_width_pcie_x8) ? "Width x8" : - (hw->bus.width == i40e_bus_width_pcie_x4) ? "Width x4" : - (hw->bus.width == i40e_bus_width_pcie_x1) ? "Width x1" : - ("Unknown")); - - if ((hw->bus.width <= i40e_bus_width_pcie_x8) && - (hw->bus.speed < i40e_bus_speed_8000)) { - device_printf(dev, "PCI-Express bandwidth available" - " for this device\n may be insufficient for" - " optimal performance.\n"); - device_printf(dev, "For expected performance a x8 " - "PCIE Gen3 slot is required.\n"); - } - - return (link); -} - -static int -ixl_sysctl_show_fw(SYSCTL_HANDLER_ARGS) -{ - struct ixl_pf *pf = (struct ixl_pf *)arg1; - struct i40e_hw *hw = &pf->hw; - char buf[32]; - - snprintf(buf, sizeof(buf), - "f%d.%d a%d.%d n%02x.%02x e%08x", - hw->aq.fw_maj_ver, hw->aq.fw_min_ver, - hw->aq.api_maj_ver, hw->aq.api_min_ver, - (hw->nvm.version & IXL_NVM_VERSION_HI_MASK) >> - IXL_NVM_VERSION_HI_SHIFT, - (hw->nvm.version & IXL_NVM_VERSION_LO_MASK) >> - IXL_NVM_VERSION_LO_SHIFT, - hw->nvm.eetrack); - return (sysctl_handle_string(oidp, buf, strlen(buf), req)); -} - - -#ifdef IXL_DEBUG_SYSCTL -static int -ixl_sysctl_link_status(SYSCTL_HANDLER_ARGS) -{ - struct ixl_pf *pf = (struct ixl_pf *)arg1; - struct i40e_hw *hw = &pf->hw; - struct i40e_link_status link_status; - char buf[512]; - - enum i40e_status_code aq_error = 0; - - aq_error = i40e_aq_get_link_info(hw, TRUE, &link_status, NULL); - if (aq_error) { - printf("i40e_aq_get_link_info() error %d\n", aq_error); - return (EPERM); - } - - sprintf(buf, "\n" - "PHY Type : %#04x\n" - "Speed : %#04x\n" - "Link info: %#04x\n" - "AN info : %#04x\n" - "Ext info : %#04x", - link_status.phy_type, link_status.link_speed, - link_status.link_info, link_status.an_info, - link_status.ext_info); - - return (sysctl_handle_string(oidp, buf, strlen(buf), req)); -} - -static int -ixl_sysctl_phy_abilities(SYSCTL_HANDLER_ARGS) -{ - struct ixl_pf *pf = (struct ixl_pf *)arg1; - struct i40e_hw *hw = &pf->hw; - char buf[512]; - enum i40e_status_code aq_error = 0; - - struct i40e_aq_get_phy_abilities_resp abilities; - - aq_error = i40e_aq_get_phy_capabilities(hw, - TRUE, FALSE, &abilities, NULL); - if (aq_error) { - printf("i40e_aq_get_phy_capabilities() error %d\n", aq_error); - return (EPERM); - } - - sprintf(buf, "\n" - "PHY Type : %#010x\n" - "Speed : %#04x\n" - "Abilities: %#04x\n" - "EEE cap : %#06x\n" - "EEER reg : %#010x\n" - "D3 Lpan : %#04x", - abilities.phy_type, abilities.link_speed, - abilities.abilities, abilities.eee_capability, - abilities.eeer_val, abilities.d3_lpan); - - return (sysctl_handle_string(oidp, buf, strlen(buf), req)); -} - -static int -ixl_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS) -{ - struct ixl_pf *pf = (struct ixl_pf *)arg1; - struct ixl_vsi *vsi = &pf->vsi; - struct ixl_mac_filter *f; - char *buf, *buf_i; - - int error = 0; - int ftl_len = 0; - int ftl_counter = 0; - int buf_len = 0; - int entry_len = 42; - - SLIST_FOREACH(f, &vsi->ftl, next) { - ftl_len++; - } - - if (ftl_len < 1) { - sysctl_handle_string(oidp, "(none)", 6, req); - return (0); - } - - buf_len = sizeof(char) * (entry_len + 1) * ftl_len + 2; - buf = buf_i = malloc(buf_len, M_DEVBUF, M_NOWAIT); - - sprintf(buf_i++, "\n"); - SLIST_FOREACH(f, &vsi->ftl, next) { - sprintf(buf_i, - MAC_FORMAT ", vlan %4d, flags %#06x", - MAC_FORMAT_ARGS(f->macaddr), f->vlan, f->flags); - buf_i += entry_len; - /* don't print '\n' for last entry */ - if (++ftl_counter != ftl_len) { - sprintf(buf_i, "\n"); - buf_i++; - } - } - - error = sysctl_handle_string(oidp, buf, strlen(buf), req); - if (error) - printf("sysctl error: %d\n", error); - free(buf, M_DEVBUF); - return error; -} - -#define IXL_SW_RES_SIZE 0x14 -static int -ixl_res_alloc_cmp(const void *a, const void *b) -{ - const struct i40e_aqc_switch_resource_alloc_element_resp *one, *two; - one = (const struct i40e_aqc_switch_resource_alloc_element_resp *)a; - two = (const struct i40e_aqc_switch_resource_alloc_element_resp *)b; - - return ((int)one->resource_type - (int)two->resource_type); -} - -static int -ixl_sysctl_hw_res_alloc(SYSCTL_HANDLER_ARGS) -{ - struct ixl_pf *pf = (struct ixl_pf *)arg1; - struct i40e_hw *hw = &pf->hw; - device_t dev = pf->dev; - struct sbuf *buf; - int error = 0; - - u8 num_entries; - struct i40e_aqc_switch_resource_alloc_element_resp resp[IXL_SW_RES_SIZE]; - - buf = sbuf_new_for_sysctl(NULL, NULL, 0, req); - if (!buf) { - device_printf(dev, "Could not allocate sbuf for output.\n"); - return (ENOMEM); - } - - bzero(resp, sizeof(resp)); - error = i40e_aq_get_switch_resource_alloc(hw, &num_entries, - resp, - IXL_SW_RES_SIZE, - NULL); - if (error) { - device_printf(dev, - "%s: get_switch_resource_alloc() error %d, aq error %d\n", - __func__, error, hw->aq.asq_last_status); - sbuf_delete(buf); - return error; - } - - /* Sort entries by type for display */ - qsort(resp, num_entries, - sizeof(struct i40e_aqc_switch_resource_alloc_element_resp), - &ixl_res_alloc_cmp); - - sbuf_cat(buf, "\n"); - sbuf_printf(buf, "# of entries: %d\n", num_entries); - sbuf_printf(buf, - "Type | Guaranteed | Total | Used | Un-allocated\n" - " | (this) | (all) | (this) | (all) \n"); - for (int i = 0; i < num_entries; i++) { - sbuf_printf(buf, - "%#4x | %10d %5d %6d %12d", - resp[i].resource_type, - resp[i].guaranteed, - resp[i].total, - resp[i].used, - resp[i].total_unalloced); - if (i < num_entries - 1) - sbuf_cat(buf, "\n"); - } - - error = sbuf_finish(buf); - sbuf_delete(buf); - - return (error); -} - -/* -** Caller must init and delete sbuf; this function will clear and -** finish it for caller. -*/ -static char * -ixl_switch_element_string(struct sbuf *s, u16 seid, bool uplink) -{ - sbuf_clear(s); - - if (seid == 0 && uplink) - sbuf_cat(s, "Network"); - else if (seid == 0) - sbuf_cat(s, "Host"); - else if (seid == 1) - sbuf_cat(s, "EMP"); - else if (seid <= 5) - sbuf_printf(s, "MAC %d", seid - 2); - else if (seid <= 15) - sbuf_cat(s, "Reserved"); - else if (seid <= 31) - sbuf_printf(s, "PF %d", seid - 16); - else if (seid <= 159) - sbuf_printf(s, "VF %d", seid - 32); - else if (seid <= 287) - sbuf_cat(s, "Reserved"); - else if (seid <= 511) - sbuf_cat(s, "Other"); // for other structures - else if (seid <= 895) - sbuf_printf(s, "VSI %d", seid - 512); - else if (seid <= 1023) - sbuf_printf(s, "Reserved"); - else - sbuf_cat(s, "Invalid"); - - sbuf_finish(s); - return sbuf_data(s); -} - -static int -ixl_sysctl_switch_config(SYSCTL_HANDLER_ARGS) -{ - struct ixl_pf *pf = (struct ixl_pf *)arg1; - struct i40e_hw *hw = &pf->hw; - device_t dev = pf->dev; - struct sbuf *buf; - struct sbuf *nmbuf; - int error = 0; - u8 aq_buf[I40E_AQ_LARGE_BUF]; - - u16 next = 0; - struct i40e_aqc_get_switch_config_resp *sw_config; - sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf; - - buf = sbuf_new_for_sysctl(NULL, NULL, 0, req); - if (!buf) { - device_printf(dev, "Could not allocate sbuf for sysctl output.\n"); - return (ENOMEM); - } - - error = i40e_aq_get_switch_config(hw, sw_config, - sizeof(aq_buf), &next, NULL); - if (error) { - device_printf(dev, - "%s: aq_get_switch_config() error %d, aq error %d\n", - __func__, error, hw->aq.asq_last_status); - sbuf_delete(buf); - return error; - } - - nmbuf = sbuf_new_auto(); - if (!nmbuf) { - device_printf(dev, "Could not allocate sbuf for name output.\n"); - return (ENOMEM); - } - - sbuf_cat(buf, "\n"); - // Assuming <= 255 elements in switch - sbuf_printf(buf, "# of elements: %d\n", sw_config->header.num_reported); - /* Exclude: - ** Revision -- all elements are revision 1 for now - */ - sbuf_printf(buf, - "SEID ( Name ) | Uplink | Downlink | Conn Type\n" - " | | | (uplink)\n"); - for (int i = 0; i < sw_config->header.num_reported; i++) { - // "%4d (%8s) | %8s %8s %#8x", - sbuf_printf(buf, "%4d", sw_config->element[i].seid); - sbuf_cat(buf, " "); - sbuf_printf(buf, "(%8s)", ixl_switch_element_string(nmbuf, - sw_config->element[i].seid, false)); - sbuf_cat(buf, " | "); - sbuf_printf(buf, "%8s", ixl_switch_element_string(nmbuf, - sw_config->element[i].uplink_seid, true)); - sbuf_cat(buf, " "); - sbuf_printf(buf, "%8s", ixl_switch_element_string(nmbuf, - sw_config->element[i].downlink_seid, false)); - sbuf_cat(buf, " "); - sbuf_printf(buf, "%#8x", sw_config->element[i].connection_type); - if (i < sw_config->header.num_reported - 1) - sbuf_cat(buf, "\n"); - } - sbuf_delete(nmbuf); - - error = sbuf_finish(buf); - sbuf_delete(buf); - - return (error); -} -#endif /* IXL_DEBUG_SYSCTL */ - - -#ifdef PCI_IOV -static int -ixl_vf_alloc_vsi(struct ixl_pf *pf, struct ixl_vf *vf) -{ - struct i40e_hw *hw; - struct ixl_vsi *vsi; - struct i40e_vsi_context vsi_ctx; - int i; - uint16_t first_queue; - enum i40e_status_code code; - - hw = &pf->hw; - vsi = &pf->vsi; - - vsi_ctx.pf_num = hw->pf_id; - vsi_ctx.uplink_seid = pf->veb_seid; - vsi_ctx.connection_type = IXL_VSI_DATA_PORT; - vsi_ctx.vf_num = hw->func_caps.vf_base_id + vf->vf_num; - vsi_ctx.flags = I40E_AQ_VSI_TYPE_VF; - - bzero(&vsi_ctx.info, sizeof(vsi_ctx.info)); - - vsi_ctx.info.valid_sections = htole16(I40E_AQ_VSI_PROP_SWITCH_VALID); - vsi_ctx.info.switch_id = htole16(0); - - vsi_ctx.info.valid_sections |= htole16(I40E_AQ_VSI_PROP_SECURITY_VALID); - vsi_ctx.info.sec_flags = 0; - if (vf->vf_flags & VF_FLAG_MAC_ANTI_SPOOF) - vsi_ctx.info.sec_flags |= I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK; - - vsi_ctx.info.valid_sections |= htole16(I40E_AQ_VSI_PROP_VLAN_VALID); - vsi_ctx.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | - I40E_AQ_VSI_PVLAN_EMOD_NOTHING; - - vsi_ctx.info.valid_sections |= - htole16(I40E_AQ_VSI_PROP_QUEUE_MAP_VALID); - vsi_ctx.info.mapping_flags = htole16(I40E_AQ_VSI_QUE_MAP_NONCONTIG); - first_queue = vsi->num_queues + vf->vf_num * IXLV_MAX_QUEUES; - for (i = 0; i < IXLV_MAX_QUEUES; i++) - vsi_ctx.info.queue_mapping[i] = htole16(first_queue + i); - for (; i < nitems(vsi_ctx.info.queue_mapping); i++) - vsi_ctx.info.queue_mapping[i] = htole16(I40E_AQ_VSI_QUEUE_MASK); - - vsi_ctx.info.tc_mapping[0] = htole16( - (0 << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | - (1 << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT)); - - code = i40e_aq_add_vsi(hw, &vsi_ctx, NULL); - if (code != I40E_SUCCESS) - return (ixl_adminq_err_to_errno(hw->aq.asq_last_status)); - vf->vsi.seid = vsi_ctx.seid; - vf->vsi.vsi_num = vsi_ctx.vsi_number; - vf->vsi.first_queue = first_queue; - vf->vsi.num_queues = IXLV_MAX_QUEUES; - - code = i40e_aq_get_vsi_params(hw, &vsi_ctx, NULL); - if (code != I40E_SUCCESS) - return (ixl_adminq_err_to_errno(hw->aq.asq_last_status)); - - code = i40e_aq_config_vsi_bw_limit(hw, vf->vsi.seid, 0, 0, NULL); - if (code != I40E_SUCCESS) { - device_printf(pf->dev, "Failed to disable BW limit: %d\n", - ixl_adminq_err_to_errno(hw->aq.asq_last_status)); - return (ixl_adminq_err_to_errno(hw->aq.asq_last_status)); - } - - memcpy(&vf->vsi.info, &vsi_ctx.info, sizeof(vf->vsi.info)); - return (0); -} - -static int -ixl_vf_setup_vsi(struct ixl_pf *pf, struct ixl_vf *vf) -{ - struct i40e_hw *hw; - int error; - - hw = &pf->hw; - - error = ixl_vf_alloc_vsi(pf, vf); - if (error != 0) - return (error); - - vf->vsi.hw_filters_add = 0; - vf->vsi.hw_filters_del = 0; - ixl_add_filter(&vf->vsi, ixl_bcast_addr, IXL_VLAN_ANY); - ixl_reconfigure_filters(&vf->vsi); - - return (0); -} - -static void -ixl_vf_map_vsi_queue(struct i40e_hw *hw, struct ixl_vf *vf, int qnum, - uint32_t val) -{ - uint32_t qtable; - int index, shift; - - /* - * Two queues are mapped in a single register, so we have to do some - * gymnastics to convert the queue number into a register index and - * shift. - */ - index = qnum / 2; - shift = (qnum % 2) * I40E_VSILAN_QTABLE_QINDEX_1_SHIFT; - - qtable = rd32(hw, I40E_VSILAN_QTABLE(index, vf->vsi.vsi_num)); - qtable &= ~(I40E_VSILAN_QTABLE_QINDEX_0_MASK << shift); - qtable |= val << shift; - wr32(hw, I40E_VSILAN_QTABLE(index, vf->vsi.vsi_num), qtable); -} - -static void -ixl_vf_map_queues(struct ixl_pf *pf, struct ixl_vf *vf) -{ - struct i40e_hw *hw; - uint32_t qtable; - int i; - - hw = &pf->hw; - - /* - * Contiguous mappings aren't actually supported by the hardware, - * so we have to use non-contiguous mappings. - */ - wr32(hw, I40E_VSILAN_QBASE(vf->vsi.vsi_num), - I40E_VSILAN_QBASE_VSIQTABLE_ENA_MASK); - - wr32(hw, I40E_VPLAN_MAPENA(vf->vf_num), - I40E_VPLAN_MAPENA_TXRX_ENA_MASK); - - for (i = 0; i < vf->vsi.num_queues; i++) { - qtable = (vf->vsi.first_queue + i) << - I40E_VPLAN_QTABLE_QINDEX_SHIFT; - - wr32(hw, I40E_VPLAN_QTABLE(i, vf->vf_num), qtable); - } - - /* Map queues allocated to VF to its VSI. */ - for (i = 0; i < vf->vsi.num_queues; i++) - ixl_vf_map_vsi_queue(hw, vf, i, vf->vsi.first_queue + i); - - /* Set rest of VSI queues as unused. */ - for (; i < IXL_MAX_VSI_QUEUES; i++) - ixl_vf_map_vsi_queue(hw, vf, i, - I40E_VSILAN_QTABLE_QINDEX_0_MASK); - - ixl_flush(hw); -} - -static void -ixl_vf_vsi_release(struct ixl_pf *pf, struct ixl_vsi *vsi) -{ - struct i40e_hw *hw; - - hw = &pf->hw; - - if (vsi->seid == 0) - return; - - i40e_aq_delete_element(hw, vsi->seid, NULL); -} - -static void -ixl_vf_disable_queue_intr(struct i40e_hw *hw, uint32_t vfint_reg) -{ - - wr32(hw, vfint_reg, I40E_VFINT_DYN_CTLN_CLEARPBA_MASK); - ixl_flush(hw); -} - -static void -ixl_vf_unregister_intr(struct i40e_hw *hw, uint32_t vpint_reg) -{ - - wr32(hw, vpint_reg, I40E_VPINT_LNKLSTN_FIRSTQ_TYPE_MASK | - I40E_VPINT_LNKLSTN_FIRSTQ_INDX_MASK); - ixl_flush(hw); -} - -static void -ixl_vf_release_resources(struct ixl_pf *pf, struct ixl_vf *vf) -{ - struct i40e_hw *hw; - uint32_t vfint_reg, vpint_reg; - int i; - - hw = &pf->hw; - - ixl_vf_vsi_release(pf, &vf->vsi); - - /* Index 0 has a special register. */ - ixl_vf_disable_queue_intr(hw, I40E_VFINT_DYN_CTL0(vf->vf_num)); - - for (i = 1; i < hw->func_caps.num_msix_vectors_vf; i++) { - vfint_reg = IXL_VFINT_DYN_CTLN_REG(hw, i , vf->vf_num); - ixl_vf_disable_queue_intr(hw, vfint_reg); - } - - /* Index 0 has a special register. */ - ixl_vf_unregister_intr(hw, I40E_VPINT_LNKLST0(vf->vf_num)); - - for (i = 1; i < hw->func_caps.num_msix_vectors_vf; i++) { - vpint_reg = IXL_VPINT_LNKLSTN_REG(hw, i, vf->vf_num); - ixl_vf_unregister_intr(hw, vpint_reg); - } - - vf->vsi.num_queues = 0; -} - -static int -ixl_flush_pcie(struct ixl_pf *pf, struct ixl_vf *vf) -{ - struct i40e_hw *hw; - int i; - uint16_t global_vf_num; - uint32_t ciad; - - hw = &pf->hw; - global_vf_num = hw->func_caps.vf_base_id + vf->vf_num; - - wr32(hw, I40E_PF_PCI_CIAA, IXL_PF_PCI_CIAA_VF_DEVICE_STATUS | - (global_vf_num << I40E_PF_PCI_CIAA_VF_NUM_SHIFT)); - for (i = 0; i < IXL_VF_RESET_TIMEOUT; i++) { - ciad = rd32(hw, I40E_PF_PCI_CIAD); - if ((ciad & IXL_PF_PCI_CIAD_VF_TRANS_PENDING_MASK) == 0) - return (0); - DELAY(1); - } - - return (ETIMEDOUT); -} - -static void -ixl_reset_vf(struct ixl_pf *pf, struct ixl_vf *vf) -{ - struct i40e_hw *hw; - uint32_t vfrtrig; - - hw = &pf->hw; - - vfrtrig = rd32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num)); - vfrtrig |= I40E_VPGEN_VFRTRIG_VFSWR_MASK; - wr32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num), vfrtrig); - ixl_flush(hw); - - ixl_reinit_vf(pf, vf); -} - -static void -ixl_reinit_vf(struct ixl_pf *pf, struct ixl_vf *vf) -{ - struct i40e_hw *hw; - uint32_t vfrstat, vfrtrig; - int i, error; - - hw = &pf->hw; - - error = ixl_flush_pcie(pf, vf); - if (error != 0) - device_printf(pf->dev, - "Timed out waiting for PCIe activity to stop on VF-%d\n", - vf->vf_num); - - for (i = 0; i < IXL_VF_RESET_TIMEOUT; i++) { - DELAY(10); - - vfrstat = rd32(hw, I40E_VPGEN_VFRSTAT(vf->vf_num)); - if (vfrstat & I40E_VPGEN_VFRSTAT_VFRD_MASK) - break; - } - - if (i == IXL_VF_RESET_TIMEOUT) - device_printf(pf->dev, "VF %d failed to reset\n", vf->vf_num); - - wr32(hw, I40E_VFGEN_RSTAT1(vf->vf_num), I40E_VFR_COMPLETED); - - vfrtrig = rd32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num)); - vfrtrig &= ~I40E_VPGEN_VFRTRIG_VFSWR_MASK; - wr32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num), vfrtrig); - - if (vf->vsi.seid != 0) - ixl_disable_rings(&vf->vsi); - - ixl_vf_release_resources(pf, vf); - ixl_vf_setup_vsi(pf, vf); - ixl_vf_map_queues(pf, vf); - - wr32(hw, I40E_VFGEN_RSTAT1(vf->vf_num), I40E_VFR_VFACTIVE); - ixl_flush(hw); -} - -static const char * -ixl_vc_opcode_str(uint16_t op) -{ - - switch (op) { - case I40E_VIRTCHNL_OP_VERSION: - return ("VERSION"); - case I40E_VIRTCHNL_OP_RESET_VF: - return ("RESET_VF"); - case I40E_VIRTCHNL_OP_GET_VF_RESOURCES: - return ("GET_VF_RESOURCES"); - case I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE: - return ("CONFIG_TX_QUEUE"); - case I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE: - return ("CONFIG_RX_QUEUE"); - case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES: - return ("CONFIG_VSI_QUEUES"); - case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP: - return ("CONFIG_IRQ_MAP"); - case I40E_VIRTCHNL_OP_ENABLE_QUEUES: - return ("ENABLE_QUEUES"); - case I40E_VIRTCHNL_OP_DISABLE_QUEUES: - return ("DISABLE_QUEUES"); - case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS: - return ("ADD_ETHER_ADDRESS"); - case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS: - return ("DEL_ETHER_ADDRESS"); - case I40E_VIRTCHNL_OP_ADD_VLAN: - return ("ADD_VLAN"); - case I40E_VIRTCHNL_OP_DEL_VLAN: - return ("DEL_VLAN"); - case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE: - return ("CONFIG_PROMISCUOUS_MODE"); - case I40E_VIRTCHNL_OP_GET_STATS: - return ("GET_STATS"); - case I40E_VIRTCHNL_OP_FCOE: - return ("FCOE"); - case I40E_VIRTCHNL_OP_EVENT: - return ("EVENT"); - default: - return ("UNKNOWN"); - } -} - -static int -ixl_vc_opcode_level(uint16_t opcode) -{ - - switch (opcode) { - case I40E_VIRTCHNL_OP_GET_STATS: - return (10); - default: - return (5); - } -} - -static void -ixl_send_vf_msg(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op, - enum i40e_status_code status, void *msg, uint16_t len) -{ - struct i40e_hw *hw; - int global_vf_id; - - hw = &pf->hw; - global_vf_id = hw->func_caps.vf_base_id + vf->vf_num; - - I40E_VC_DEBUG(pf, ixl_vc_opcode_level(op), - "Sending msg (op=%s[%d], status=%d) to VF-%d\n", - ixl_vc_opcode_str(op), op, status, vf->vf_num); - - i40e_aq_send_msg_to_vf(hw, global_vf_id, op, status, msg, len, NULL); -} - -static void -ixl_send_vf_ack(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op) -{ - - ixl_send_vf_msg(pf, vf, op, I40E_SUCCESS, NULL, 0); -} - -static void -ixl_send_vf_nack_msg(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op, - enum i40e_status_code status, const char *file, int line) -{ - - I40E_VC_DEBUG(pf, 1, - "Sending NACK (op=%s[%d], err=%d) to VF-%d from %s:%d\n", - ixl_vc_opcode_str(op), op, status, vf->vf_num, file, line); - ixl_send_vf_msg(pf, vf, op, status, NULL, 0); -} - -static void -ixl_vf_version_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, - uint16_t msg_size) -{ - struct i40e_virtchnl_version_info reply; - - if (msg_size != sizeof(struct i40e_virtchnl_version_info)) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_VERSION, - I40E_ERR_PARAM); - return; - } - - reply.major = I40E_VIRTCHNL_VERSION_MAJOR; - reply.minor = I40E_VIRTCHNL_VERSION_MINOR; - ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_VERSION, I40E_SUCCESS, &reply, - sizeof(reply)); -} - -static void -ixl_vf_reset_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, - uint16_t msg_size) -{ - - if (msg_size != 0) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_RESET_VF, - I40E_ERR_PARAM); - return; - } - - ixl_reset_vf(pf, vf); - - /* No response to a reset message. */ -} - -static void -ixl_vf_get_resources_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, - uint16_t msg_size) -{ - struct i40e_virtchnl_vf_resource reply; - - if (msg_size != 0) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_VF_RESOURCES, - I40E_ERR_PARAM); - return; - } - - bzero(&reply, sizeof(reply)); - - reply.vf_offload_flags = I40E_VIRTCHNL_VF_OFFLOAD_L2; - - reply.num_vsis = 1; - reply.num_queue_pairs = vf->vsi.num_queues; - reply.max_vectors = pf->hw.func_caps.num_msix_vectors_vf; - reply.vsi_res[0].vsi_id = vf->vsi.vsi_num; - reply.vsi_res[0].vsi_type = I40E_VSI_SRIOV; - reply.vsi_res[0].num_queue_pairs = vf->vsi.num_queues; - memcpy(reply.vsi_res[0].default_mac_addr, vf->mac, ETHER_ADDR_LEN); - - ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_GET_VF_RESOURCES, - I40E_SUCCESS, &reply, sizeof(reply)); -} - -static int -ixl_vf_config_tx_queue(struct ixl_pf *pf, struct ixl_vf *vf, - struct i40e_virtchnl_txq_info *info) -{ - struct i40e_hw *hw; - struct i40e_hmc_obj_txq txq; - uint16_t global_queue_num, global_vf_num; - enum i40e_status_code status; - uint32_t qtx_ctl; - - hw = &pf->hw; - global_queue_num = vf->vsi.first_queue + info->queue_id; - global_vf_num = hw->func_caps.vf_base_id + vf->vf_num; - bzero(&txq, sizeof(txq)); - - status = i40e_clear_lan_tx_queue_context(hw, global_queue_num); - if (status != I40E_SUCCESS) - return (EINVAL); - - txq.base = info->dma_ring_addr / IXL_TX_CTX_BASE_UNITS; - - txq.head_wb_ena = info->headwb_enabled; - txq.head_wb_addr = info->dma_headwb_addr; - txq.qlen = info->ring_len; - txq.rdylist = le16_to_cpu(vf->vsi.info.qs_handle[0]); - txq.rdylist_act = 0; - - status = i40e_set_lan_tx_queue_context(hw, global_queue_num, &txq); - if (status != I40E_SUCCESS) - return (EINVAL); - - qtx_ctl = I40E_QTX_CTL_VF_QUEUE | - (hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) | - (global_vf_num << I40E_QTX_CTL_VFVM_INDX_SHIFT); - wr32(hw, I40E_QTX_CTL(global_queue_num), qtx_ctl); - ixl_flush(hw); - - return (0); -} - -static int -ixl_vf_config_rx_queue(struct ixl_pf *pf, struct ixl_vf *vf, - struct i40e_virtchnl_rxq_info *info) -{ - struct i40e_hw *hw; - struct i40e_hmc_obj_rxq rxq; - uint16_t global_queue_num; - enum i40e_status_code status; - - hw = &pf->hw; - global_queue_num = vf->vsi.first_queue + info->queue_id; - bzero(&rxq, sizeof(rxq)); - - if (info->databuffer_size > IXL_VF_MAX_BUFFER) - return (EINVAL); - - if (info->max_pkt_size > IXL_VF_MAX_FRAME || - info->max_pkt_size < ETHER_MIN_LEN) - return (EINVAL); - - if (info->splithdr_enabled) { - if (info->hdr_size > IXL_VF_MAX_HDR_BUFFER) - return (EINVAL); - - rxq.hsplit_0 = info->rx_split_pos & - (I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_L2 | - I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_IP | - I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_TCP_UDP | - I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_SCTP); - rxq.hbuff = info->hdr_size >> I40E_RXQ_CTX_HBUFF_SHIFT; - - rxq.dtype = 2; - } - - status = i40e_clear_lan_rx_queue_context(hw, global_queue_num); - if (status != I40E_SUCCESS) - return (EINVAL); - - rxq.base = info->dma_ring_addr / IXL_RX_CTX_BASE_UNITS; - rxq.qlen = info->ring_len; - - rxq.dbuff = info->databuffer_size >> I40E_RXQ_CTX_DBUFF_SHIFT; - - rxq.dsize = 1; - rxq.crcstrip = 1; - rxq.l2tsel = 1; - - rxq.rxmax = info->max_pkt_size; - rxq.tphrdesc_ena = 1; - rxq.tphwdesc_ena = 1; - rxq.tphdata_ena = 1; - rxq.tphhead_ena = 1; - rxq.lrxqthresh = 2; - rxq.prefena = 1; - - status = i40e_set_lan_rx_queue_context(hw, global_queue_num, &rxq); - if (status != I40E_SUCCESS) - return (EINVAL); - - return (0); -} - -static void -ixl_vf_config_vsi_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, - uint16_t msg_size) -{ - struct i40e_virtchnl_vsi_queue_config_info *info; - struct i40e_virtchnl_queue_pair_info *pair; - int i; - - if (msg_size < sizeof(*info)) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, - I40E_ERR_PARAM); - return; - } - - info = msg; - if (info->num_queue_pairs == 0) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, - I40E_ERR_PARAM); - return; - } - - if (msg_size != sizeof(*info) + info->num_queue_pairs * sizeof(*pair)) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, - I40E_ERR_PARAM); - return; - } - - if (info->vsi_id != vf->vsi.vsi_num) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, - I40E_ERR_PARAM); - return; - } - - for (i = 0; i < info->num_queue_pairs; i++) { - pair = &info->qpair[i]; - - if (pair->txq.vsi_id != vf->vsi.vsi_num || - pair->rxq.vsi_id != vf->vsi.vsi_num || - pair->txq.queue_id != pair->rxq.queue_id || - pair->txq.queue_id >= vf->vsi.num_queues) { - - i40e_send_vf_nack(pf, vf, - I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM); - return; - } - - if (ixl_vf_config_tx_queue(pf, vf, &pair->txq) != 0) { - i40e_send_vf_nack(pf, vf, - I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM); - return; - } - - if (ixl_vf_config_rx_queue(pf, vf, &pair->rxq) != 0) { - i40e_send_vf_nack(pf, vf, - I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM); - return; - } - } - - ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES); -} - -static void -ixl_vf_set_qctl(struct ixl_pf *pf, - const struct i40e_virtchnl_vector_map *vector, - enum i40e_queue_type cur_type, uint16_t cur_queue, - enum i40e_queue_type *last_type, uint16_t *last_queue) -{ - uint32_t offset, qctl; - uint16_t itr_indx; - - if (cur_type == I40E_QUEUE_TYPE_RX) { - offset = I40E_QINT_RQCTL(cur_queue); - itr_indx = vector->rxitr_idx; - } else { - offset = I40E_QINT_TQCTL(cur_queue); - itr_indx = vector->txitr_idx; - } - - qctl = htole32((vector->vector_id << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | - (*last_type << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) | - (*last_queue << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | - I40E_QINT_RQCTL_CAUSE_ENA_MASK | - (itr_indx << I40E_QINT_RQCTL_ITR_INDX_SHIFT)); - - wr32(&pf->hw, offset, qctl); - - *last_type = cur_type; - *last_queue = cur_queue; -} - -static void -ixl_vf_config_vector(struct ixl_pf *pf, struct ixl_vf *vf, - const struct i40e_virtchnl_vector_map *vector) -{ - struct i40e_hw *hw; - u_int qindex; - enum i40e_queue_type type, last_type; - uint32_t lnklst_reg; - uint16_t rxq_map, txq_map, cur_queue, last_queue; - - hw = &pf->hw; - - rxq_map = vector->rxq_map; - txq_map = vector->txq_map; - - last_queue = IXL_END_OF_INTR_LNKLST; - last_type = I40E_QUEUE_TYPE_RX; - - /* - * The datasheet says to optimize performance, RX queues and TX queues - * should be interleaved in the interrupt linked list, so we process - * both at once here. - */ - while ((rxq_map != 0) || (txq_map != 0)) { - if (txq_map != 0) { - qindex = ffs(txq_map) - 1; - type = I40E_QUEUE_TYPE_TX; - cur_queue = vf->vsi.first_queue + qindex; - ixl_vf_set_qctl(pf, vector, type, cur_queue, - &last_type, &last_queue); - txq_map &= ~(1 << qindex); - } - - if (rxq_map != 0) { - qindex = ffs(rxq_map) - 1; - type = I40E_QUEUE_TYPE_RX; - cur_queue = vf->vsi.first_queue + qindex; - ixl_vf_set_qctl(pf, vector, type, cur_queue, - &last_type, &last_queue); - rxq_map &= ~(1 << qindex); - } - } - - if (vector->vector_id == 0) - lnklst_reg = I40E_VPINT_LNKLST0(vf->vf_num); - else - lnklst_reg = IXL_VPINT_LNKLSTN_REG(hw, vector->vector_id, - vf->vf_num); - wr32(hw, lnklst_reg, - (last_queue << I40E_VPINT_LNKLST0_FIRSTQ_INDX_SHIFT) | - (last_type << I40E_VPINT_LNKLST0_FIRSTQ_TYPE_SHIFT)); - - ixl_flush(hw); -} - -static void -ixl_vf_config_irq_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, - uint16_t msg_size) -{ - struct i40e_virtchnl_irq_map_info *map; - struct i40e_virtchnl_vector_map *vector; - struct i40e_hw *hw; - int i, largest_txq, largest_rxq; - - hw = &pf->hw; - - if (msg_size < sizeof(*map)) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, - I40E_ERR_PARAM); - return; - } - - map = msg; - if (map->num_vectors == 0) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, - I40E_ERR_PARAM); - return; - } - - if (msg_size != sizeof(*map) + map->num_vectors * sizeof(*vector)) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, - I40E_ERR_PARAM); - return; - } - - for (i = 0; i < map->num_vectors; i++) { - vector = &map->vecmap[i]; - - if ((vector->vector_id >= hw->func_caps.num_msix_vectors_vf) || - vector->vsi_id != vf->vsi.vsi_num) { - i40e_send_vf_nack(pf, vf, - I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, I40E_ERR_PARAM); - return; - } - - if (vector->rxq_map != 0) { - largest_rxq = fls(vector->rxq_map) - 1; - if (largest_rxq >= vf->vsi.num_queues) { - i40e_send_vf_nack(pf, vf, - I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, - I40E_ERR_PARAM); - return; - } - } - - if (vector->txq_map != 0) { - largest_txq = fls(vector->txq_map) - 1; - if (largest_txq >= vf->vsi.num_queues) { - i40e_send_vf_nack(pf, vf, - I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, - I40E_ERR_PARAM); - return; - } - } - - if (vector->rxitr_idx > IXL_MAX_ITR_IDX || - vector->txitr_idx > IXL_MAX_ITR_IDX) { - i40e_send_vf_nack(pf, vf, - I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, - I40E_ERR_PARAM); - return; - } - - ixl_vf_config_vector(pf, vf, vector); - } - - ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP); -} - -static void -ixl_vf_enable_queues_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, - uint16_t msg_size) -{ - struct i40e_virtchnl_queue_select *select; - int error; - - if (msg_size != sizeof(*select)) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES, - I40E_ERR_PARAM); - return; - } - - select = msg; - if (select->vsi_id != vf->vsi.vsi_num || - select->rx_queues == 0 || select->tx_queues == 0) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES, - I40E_ERR_PARAM); - return; - } - - error = ixl_enable_rings(&vf->vsi); - if (error) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES, - I40E_ERR_TIMEOUT); - return; - } - - ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES); -} - -static void -ixl_vf_disable_queues_msg(struct ixl_pf *pf, struct ixl_vf *vf, - void *msg, uint16_t msg_size) -{ - struct i40e_virtchnl_queue_select *select; - int error; - - if (msg_size != sizeof(*select)) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES, - I40E_ERR_PARAM); - return; - } - - select = msg; - if (select->vsi_id != vf->vsi.vsi_num || - select->rx_queues == 0 || select->tx_queues == 0) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES, - I40E_ERR_PARAM); - return; - } - - error = ixl_disable_rings(&vf->vsi); - if (error) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES, - I40E_ERR_TIMEOUT); - return; - } - - ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES); -} - -static boolean_t -ixl_zero_mac(const uint8_t *addr) -{ - uint8_t zero[ETHER_ADDR_LEN] = {0, 0, 0, 0, 0, 0}; - - return (cmp_etheraddr(addr, zero)); -} - -static boolean_t -ixl_bcast_mac(const uint8_t *addr) -{ - - return (cmp_etheraddr(addr, ixl_bcast_addr)); -} - -static int -ixl_vf_mac_valid(struct ixl_vf *vf, const uint8_t *addr) -{ - - if (ixl_zero_mac(addr) || ixl_bcast_mac(addr)) - return (EINVAL); - - /* - * If the VF is not allowed to change its MAC address, don't let it - * set a MAC filter for an address that is not a multicast address and - * is not its assigned MAC. - */ - if (!(vf->vf_flags & VF_FLAG_SET_MAC_CAP) && - !(ETHER_IS_MULTICAST(addr) || cmp_etheraddr(addr, vf->mac))) - return (EPERM); - - return (0); -} - -static void -ixl_vf_add_mac_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, - uint16_t msg_size) -{ - struct i40e_virtchnl_ether_addr_list *addr_list; - struct i40e_virtchnl_ether_addr *addr; - struct ixl_vsi *vsi; - int i; - size_t expected_size; - - vsi = &vf->vsi; - - if (msg_size < sizeof(*addr_list)) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, - I40E_ERR_PARAM); - return; - } - - addr_list = msg; - expected_size = sizeof(*addr_list) + - addr_list->num_elements * sizeof(*addr); - - if (addr_list->num_elements == 0 || - addr_list->vsi_id != vsi->vsi_num || - msg_size != expected_size) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, - I40E_ERR_PARAM); - return; - } - - for (i = 0; i < addr_list->num_elements; i++) { - if (ixl_vf_mac_valid(vf, addr_list->list[i].addr) != 0) { - i40e_send_vf_nack(pf, vf, - I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, I40E_ERR_PARAM); - return; - } - } - - for (i = 0; i < addr_list->num_elements; i++) { - addr = &addr_list->list[i]; - ixl_add_filter(vsi, addr->addr, IXL_VLAN_ANY); - } - - ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS); -} - -static void -ixl_vf_del_mac_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, - uint16_t msg_size) -{ - struct i40e_virtchnl_ether_addr_list *addr_list; - struct i40e_virtchnl_ether_addr *addr; - size_t expected_size; - int i; - - if (msg_size < sizeof(*addr_list)) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, - I40E_ERR_PARAM); - return; - } - - addr_list = msg; - expected_size = sizeof(*addr_list) + - addr_list->num_elements * sizeof(*addr); - - if (addr_list->num_elements == 0 || - addr_list->vsi_id != vf->vsi.vsi_num || - msg_size != expected_size) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, - I40E_ERR_PARAM); - return; - } - - for (i = 0; i < addr_list->num_elements; i++) { - addr = &addr_list->list[i]; - if (ixl_zero_mac(addr->addr) || ixl_bcast_mac(addr->addr)) { - i40e_send_vf_nack(pf, vf, - I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, I40E_ERR_PARAM); - return; - } - } - - for (i = 0; i < addr_list->num_elements; i++) { - addr = &addr_list->list[i]; - ixl_del_filter(&vf->vsi, addr->addr, IXL_VLAN_ANY); - } - - ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS); -} - -static enum i40e_status_code -ixl_vf_enable_vlan_strip(struct ixl_pf *pf, struct ixl_vf *vf) -{ - struct i40e_vsi_context vsi_ctx; - - vsi_ctx.seid = vf->vsi.seid; - - bzero(&vsi_ctx.info, sizeof(vsi_ctx.info)); - vsi_ctx.info.valid_sections = htole16(I40E_AQ_VSI_PROP_VLAN_VALID); - vsi_ctx.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | - I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH; - return (i40e_aq_update_vsi_params(&pf->hw, &vsi_ctx, NULL)); -} - -static void -ixl_vf_add_vlan_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, - uint16_t msg_size) -{ - struct i40e_virtchnl_vlan_filter_list *filter_list; - enum i40e_status_code code; - size_t expected_size; - int i; - - if (msg_size < sizeof(*filter_list)) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, - I40E_ERR_PARAM); - return; - } - - filter_list = msg; - expected_size = sizeof(*filter_list) + - filter_list->num_elements * sizeof(uint16_t); - if (filter_list->num_elements == 0 || - filter_list->vsi_id != vf->vsi.vsi_num || - msg_size != expected_size) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, - I40E_ERR_PARAM); - return; - } - - if (!(vf->vf_flags & VF_FLAG_VLAN_CAP)) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, - I40E_ERR_PARAM); - return; - } - - for (i = 0; i < filter_list->num_elements; i++) { - if (filter_list->vlan_id[i] > EVL_VLID_MASK) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, - I40E_ERR_PARAM); - return; - } - } - - code = ixl_vf_enable_vlan_strip(pf, vf); - if (code != I40E_SUCCESS) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, - I40E_ERR_PARAM); - } - - for (i = 0; i < filter_list->num_elements; i++) - ixl_add_filter(&vf->vsi, vf->mac, filter_list->vlan_id[i]); - - ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN); -} - -static void -ixl_vf_del_vlan_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, - uint16_t msg_size) -{ - struct i40e_virtchnl_vlan_filter_list *filter_list; - int i; - size_t expected_size; - - if (msg_size < sizeof(*filter_list)) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN, - I40E_ERR_PARAM); - return; - } - - filter_list = msg; - expected_size = sizeof(*filter_list) + - filter_list->num_elements * sizeof(uint16_t); - if (filter_list->num_elements == 0 || - filter_list->vsi_id != vf->vsi.vsi_num || - msg_size != expected_size) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN, - I40E_ERR_PARAM); - return; - } - - for (i = 0; i < filter_list->num_elements; i++) { - if (filter_list->vlan_id[i] > EVL_VLID_MASK) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, - I40E_ERR_PARAM); - return; - } - } - - if (!(vf->vf_flags & VF_FLAG_VLAN_CAP)) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, - I40E_ERR_PARAM); - return; - } - - for (i = 0; i < filter_list->num_elements; i++) - ixl_del_filter(&vf->vsi, vf->mac, filter_list->vlan_id[i]); - - ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN); -} - -static void -ixl_vf_config_promisc_msg(struct ixl_pf *pf, struct ixl_vf *vf, - void *msg, uint16_t msg_size) -{ - struct i40e_virtchnl_promisc_info *info; - enum i40e_status_code code; - - if (msg_size != sizeof(*info)) { - i40e_send_vf_nack(pf, vf, - I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM); - return; - } - - if (!(vf->vf_flags & VF_FLAG_PROMISC_CAP)) { - i40e_send_vf_nack(pf, vf, - I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM); - return; - } - - info = msg; - if (info->vsi_id != vf->vsi.vsi_num) { - i40e_send_vf_nack(pf, vf, - I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM); - return; - } - - code = i40e_aq_set_vsi_unicast_promiscuous(&pf->hw, info->vsi_id, - info->flags & I40E_FLAG_VF_UNICAST_PROMISC, NULL); - if (code != I40E_SUCCESS) { - i40e_send_vf_nack(pf, vf, - I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, code); - return; - } - - code = i40e_aq_set_vsi_multicast_promiscuous(&pf->hw, info->vsi_id, - info->flags & I40E_FLAG_VF_MULTICAST_PROMISC, NULL); - if (code != I40E_SUCCESS) { - i40e_send_vf_nack(pf, vf, - I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, code); - return; - } - - ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE); -} - -static void -ixl_vf_get_stats_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, - uint16_t msg_size) -{ - struct i40e_virtchnl_queue_select *queue; - - if (msg_size != sizeof(*queue)) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_STATS, - I40E_ERR_PARAM); - return; - } - - queue = msg; - if (queue->vsi_id != vf->vsi.vsi_num) { - i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_STATS, - I40E_ERR_PARAM); - return; - } - - ixl_update_eth_stats(&vf->vsi); - - ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_GET_STATS, - I40E_SUCCESS, &vf->vsi.eth_stats, sizeof(vf->vsi.eth_stats)); -} - -static void -ixl_handle_vf_msg(struct ixl_pf *pf, struct i40e_arq_event_info *event) -{ - struct ixl_vf *vf; - void *msg; - uint16_t vf_num, msg_size; - uint32_t opcode; - - vf_num = le16toh(event->desc.retval) - pf->hw.func_caps.vf_base_id; - opcode = le32toh(event->desc.cookie_high); - - if (vf_num >= pf->num_vfs) { - device_printf(pf->dev, "Got msg from illegal VF: %d\n", vf_num); - return; - } - - vf = &pf->vfs[vf_num]; - msg = event->msg_buf; - msg_size = event->msg_len; - - I40E_VC_DEBUG(pf, ixl_vc_opcode_level(opcode), - "Got msg %s(%d) from VF-%d of size %d\n", - ixl_vc_opcode_str(opcode), opcode, vf_num, msg_size); - - switch (opcode) { - case I40E_VIRTCHNL_OP_VERSION: - ixl_vf_version_msg(pf, vf, msg, msg_size); - break; - case I40E_VIRTCHNL_OP_RESET_VF: - ixl_vf_reset_msg(pf, vf, msg, msg_size); - break; - case I40E_VIRTCHNL_OP_GET_VF_RESOURCES: - ixl_vf_get_resources_msg(pf, vf, msg, msg_size); - break; - case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES: - ixl_vf_config_vsi_msg(pf, vf, msg, msg_size); - break; - case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP: - ixl_vf_config_irq_msg(pf, vf, msg, msg_size); - break; - case I40E_VIRTCHNL_OP_ENABLE_QUEUES: - ixl_vf_enable_queues_msg(pf, vf, msg, msg_size); - break; - case I40E_VIRTCHNL_OP_DISABLE_QUEUES: - ixl_vf_disable_queues_msg(pf, vf, msg, msg_size); - break; - case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS: - ixl_vf_add_mac_msg(pf, vf, msg, msg_size); - break; - case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS: - ixl_vf_del_mac_msg(pf, vf, msg, msg_size); - break; - case I40E_VIRTCHNL_OP_ADD_VLAN: - ixl_vf_add_vlan_msg(pf, vf, msg, msg_size); - break; - case I40E_VIRTCHNL_OP_DEL_VLAN: - ixl_vf_del_vlan_msg(pf, vf, msg, msg_size); - break; - case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE: - ixl_vf_config_promisc_msg(pf, vf, msg, msg_size); - break; - case I40E_VIRTCHNL_OP_GET_STATS: - ixl_vf_get_stats_msg(pf, vf, msg, msg_size); - break; - - /* These two opcodes have been superseded by CONFIG_VSI_QUEUES. */ - case I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE: - case I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE: - default: - i40e_send_vf_nack(pf, vf, opcode, I40E_ERR_NOT_IMPLEMENTED); - break; - } -} - -/* Handle any VFs that have reset themselves via a Function Level Reset(FLR). */ -static void -ixl_handle_vflr(void *arg, int pending) -{ - struct ixl_pf *pf; - struct i40e_hw *hw; - uint16_t global_vf_num; - uint32_t vflrstat_index, vflrstat_mask, vflrstat, icr0; - int i; - - pf = arg; - hw = &pf->hw; - - IXL_PF_LOCK(pf); - for (i = 0; i < pf->num_vfs; i++) { - global_vf_num = hw->func_caps.vf_base_id + i; - - vflrstat_index = IXL_GLGEN_VFLRSTAT_INDEX(global_vf_num); - vflrstat_mask = IXL_GLGEN_VFLRSTAT_MASK(global_vf_num); - vflrstat = rd32(hw, I40E_GLGEN_VFLRSTAT(vflrstat_index)); - if (vflrstat & vflrstat_mask) { - wr32(hw, I40E_GLGEN_VFLRSTAT(vflrstat_index), - vflrstat_mask); - - ixl_reinit_vf(pf, &pf->vfs[i]); - } - } - - icr0 = rd32(hw, I40E_PFINT_ICR0_ENA); - icr0 |= I40E_PFINT_ICR0_ENA_VFLR_MASK; - wr32(hw, I40E_PFINT_ICR0_ENA, icr0); - ixl_flush(hw); - - IXL_PF_UNLOCK(pf); -} - -static int -ixl_adminq_err_to_errno(enum i40e_admin_queue_err err) -{ - - switch (err) { - case I40E_AQ_RC_EPERM: - return (EPERM); - case I40E_AQ_RC_ENOENT: - return (ENOENT); - case I40E_AQ_RC_ESRCH: - return (ESRCH); - case I40E_AQ_RC_EINTR: - return (EINTR); - case I40E_AQ_RC_EIO: - return (EIO); - case I40E_AQ_RC_ENXIO: - return (ENXIO); - case I40E_AQ_RC_E2BIG: - return (E2BIG); - case I40E_AQ_RC_EAGAIN: - return (EAGAIN); - case I40E_AQ_RC_ENOMEM: - return (ENOMEM); - case I40E_AQ_RC_EACCES: - return (EACCES); - case I40E_AQ_RC_EFAULT: - return (EFAULT); - case I40E_AQ_RC_EBUSY: - return (EBUSY); - case I40E_AQ_RC_EEXIST: - return (EEXIST); - case I40E_AQ_RC_EINVAL: - return (EINVAL); - case I40E_AQ_RC_ENOTTY: - return (ENOTTY); - case I40E_AQ_RC_ENOSPC: - return (ENOSPC); - case I40E_AQ_RC_ENOSYS: - return (ENOSYS); - case I40E_AQ_RC_ERANGE: - return (ERANGE); - case I40E_AQ_RC_EFLUSHED: - return (EINVAL); /* No exact equivalent in errno.h */ - case I40E_AQ_RC_BAD_ADDR: - return (EFAULT); - case I40E_AQ_RC_EMODE: - return (EPERM); - case I40E_AQ_RC_EFBIG: - return (EFBIG); - default: - return (EINVAL); - } -} - -static int -ixl_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *params) -{ - struct ixl_pf *pf; - struct i40e_hw *hw; - struct ixl_vsi *pf_vsi; - enum i40e_status_code ret; - int i, error; - - pf = device_get_softc(dev); - hw = &pf->hw; - pf_vsi = &pf->vsi; - - IXL_PF_LOCK(pf); - pf->vfs = malloc(sizeof(struct ixl_vf) * num_vfs, M_IXL, M_NOWAIT | - M_ZERO); - - if (pf->vfs == NULL) { - error = ENOMEM; - goto fail; - } - - for (i = 0; i < num_vfs; i++) - sysctl_ctx_init(&pf->vfs[i].ctx); - - ret = i40e_aq_add_veb(hw, pf_vsi->uplink_seid, pf_vsi->seid, - 1, FALSE, FALSE, &pf->veb_seid, NULL); - if (ret != I40E_SUCCESS) { - error = ixl_adminq_err_to_errno(hw->aq.asq_last_status); - device_printf(dev, "add_veb failed; code=%d error=%d", ret, - error); - goto fail; - } - - ixl_configure_msix(pf); - ixl_enable_adminq(hw); - - pf->num_vfs = num_vfs; - IXL_PF_UNLOCK(pf); - return (0); - -fail: - free(pf->vfs, M_IXL); - pf->vfs = NULL; - IXL_PF_UNLOCK(pf); - return (error); -} - -static void -ixl_iov_uninit(device_t dev) -{ - struct ixl_pf *pf; - struct i40e_hw *hw; - struct ixl_vsi *vsi; - struct ifnet *ifp; - struct ixl_vf *vfs; - int i, num_vfs; - - pf = device_get_softc(dev); - hw = &pf->hw; - vsi = &pf->vsi; - ifp = vsi->ifp; - - IXL_PF_LOCK(pf); - for (i = 0; i < pf->num_vfs; i++) { - if (pf->vfs[i].vsi.seid != 0) - i40e_aq_delete_element(hw, pf->vfs[i].vsi.seid, NULL); - } - - if (pf->veb_seid != 0) { - i40e_aq_delete_element(hw, pf->veb_seid, NULL); - pf->veb_seid = 0; - } - -#if __FreeBSD_version > 1100022 - if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) +#ifdef IFLIB +#include "iflib_if_ixl.c" #else - if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) +#include "legacy_if_ixl.c" #endif - ixl_disable_intr(vsi); - - vfs = pf->vfs; - num_vfs = pf->num_vfs; - - pf->vfs = NULL; - pf->num_vfs = 0; - IXL_PF_UNLOCK(pf); - - /* Do this after the unlock as sysctl_ctx_free might sleep. */ - for (i = 0; i < num_vfs; i++) - sysctl_ctx_free(&vfs[i].ctx); - free(vfs, M_IXL); -} - -static int -ixl_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *params) -{ - char sysctl_name[QUEUE_NAME_LEN]; - struct ixl_pf *pf; - struct ixl_vf *vf; - const void *mac; - size_t size; - int error; - - pf = device_get_softc(dev); - vf = &pf->vfs[vfnum]; - - IXL_PF_LOCK(pf); - vf->vf_num = vfnum; - - vf->vsi.back = pf; - vf->vf_flags = VF_FLAG_ENABLED; - SLIST_INIT(&vf->vsi.ftl); - - error = ixl_vf_setup_vsi(pf, vf); - if (error != 0) - goto out; - - if (nvlist_exists_binary(params, "mac-addr")) { - mac = nvlist_get_binary(params, "mac-addr", &size); - bcopy(mac, vf->mac, ETHER_ADDR_LEN); - - if (nvlist_get_bool(params, "allow-set-mac")) - vf->vf_flags |= VF_FLAG_SET_MAC_CAP; - } else - /* - * If the administrator has not specified a MAC address then - * we must allow the VF to choose one. - */ - vf->vf_flags |= VF_FLAG_SET_MAC_CAP; - - if (nvlist_get_bool(params, "mac-anti-spoof")) - vf->vf_flags |= VF_FLAG_MAC_ANTI_SPOOF; - - if (nvlist_get_bool(params, "allow-promisc")) - vf->vf_flags |= VF_FLAG_PROMISC_CAP; - - vf->vf_flags |= VF_FLAG_VLAN_CAP; - - ixl_reset_vf(pf, vf); -out: - IXL_PF_UNLOCK(pf); - if (error == 0) { - snprintf(sysctl_name, sizeof(sysctl_name), "vf%d", vfnum); - ixl_add_vsi_sysctls(pf, &vf->vsi, &vf->ctx, sysctl_name); - } - - return (error); -} -#endif /* PCI_IOV */ Index: sys/dev/ixl/if_ixl_common.c =================================================================== --- /dev/null +++ sys/dev/ixl/if_ixl_common.c @@ -0,0 +1,129 @@ +#include "opt_iflib.h" + +#ifdef IFLIB +#ifndef IXL_STANDALONE_BUILD +#include "opt_inet.h" +#include "opt_inet6.h" +#include "opt_rss.h" +#endif + +#include "ixl.h" +#include "ixl_pf.h" + +#ifdef RSS +#include +#endif + +#include "ifdi_if.h" + + +/********************************************************************* + * + * Media Ioctl callback + * + * This routine is called when the user changes speed/duplex using + * media/mediopt option with ifconfig. + * + **********************************************************************/ +int +ixl_if_media_change(if_ctx_t ctx) +{ + struct ifmedia *ifm = iflib_get_media(ctx); + + INIT_DEBUGOUT("ixl_media_change: begin"); + + if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) + return (EINVAL); + + if_printf(iflib_get_ifp(ctx), "Media change is currently not supported.\n"); + return (ENODEV); +} + +int +ixl_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nqs, int nqsets) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + struct ixl_tx_queue *que; + if_shared_ctx_t sctx; + int i; + + MPASS(vsi->num_tx_queues > 0); + MPASS(nqs == 1); + MPASS(vsi->num_tx_queues == nqsets); + /* Allocate queue structure memory */ + sctx = iflib_get_sctx(ctx); + if (!(vsi->tx_queues = + (struct ixl_tx_queue *) malloc(sizeof(struct ixl_tx_queue) *nqsets, M_IXL, M_NOWAIT | M_ZERO))) { + device_printf(iflib_get_dev(ctx), "Unable to allocate TX ring memory\n"); + return (ENOMEM); + } + + for (i = 0, que = vsi->tx_queues; i < nqsets; i++, que++) { + struct tx_ring *txr = &que->txr; + + txr->me = i; + que->vsi = vsi; + + /* get the virtual and physical address of the hardware queues */ + txr->tail = I40E_QTX_TAIL(txr->me); + txr->tx_base = (struct i40e_tx_desc *)vaddrs[i]; + txr->tx_paddr = paddrs[i]; + txr->que = que; + } + + device_printf(iflib_get_dev(ctx), "allocated for %d queues\n", vsi->num_tx_queues); + return (0); +} + +int +ixl_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nqs, int nqsets) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + struct ixl_rx_queue *que; + if_shared_ctx_t sctx; + int i; + + MPASS(vsi->num_rx_queues > 0); + MPASS(nqs == 1); + MPASS(vsi->num_rx_queues == nqsets); + /* Allocate queue structure memory */ + sctx = iflib_get_sctx(ctx); + if (!(vsi->rx_queues = + (struct ixl_rx_queue *) malloc(sizeof(struct ixl_rx_queue) * + nqsets, M_IXL, M_NOWAIT | M_ZERO))) { + device_printf(iflib_get_dev(ctx), "Unable to allocate TX ring memory\n"); + return (ENOMEM); + } + + for (i = 0, que = vsi->rx_queues; i < nqsets; i++, que++) { + struct rx_ring *rxr = &que->rxr; + + rxr->me = i; + que->vsi = vsi; + + /* get the virtual and physical address of the hardware queues */ + rxr->tail = I40E_QRX_TAIL(rxr->me); + rxr->rx_base = (union i40e_rx_desc *)vaddrs[i]; + rxr->rx_paddr = paddrs[i]; + rxr->que = que; + } + + device_printf(iflib_get_dev(ctx), "allocated for %d queues\n", vsi->num_rx_queues); + return (0); +} + +void +ixl_if_queues_free(if_ctx_t ctx) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + + if (vsi->tx_queues != NULL) { + free(vsi->tx_queues, M_IXL); + vsi->tx_queues = NULL; + } + if (vsi->rx_queues != NULL) { + free(vsi->rx_queues, M_IXL); + vsi->rx_queues = NULL; + } +} +#endif /* IFLIB */ Index: sys/dev/ixl/if_ixlv.c =================================================================== --- sys/dev/ixl/if_ixlv.c +++ sys/dev/ixl/if_ixlv.c @@ -1,2972 +1,6 @@ -/****************************************************************************** - - Copyright (c) 2013-2015, Intel Corporation - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - 3. Neither the name of the Intel Corporation nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - -******************************************************************************/ -/*$FreeBSD$*/ - -#ifndef IXL_STANDALONE_BUILD -#include "opt_inet.h" -#include "opt_inet6.h" -#include "opt_rss.h" -#endif - -#include "ixl.h" -#include "ixlv.h" - -#ifdef RSS -#include -#endif - -/********************************************************************* - * Driver version - *********************************************************************/ -char ixlv_driver_version[] = "1.2.6"; - -/********************************************************************* - * PCI Device ID Table - * - * Used by probe to select devices to load on - * Last field stores an index into ixlv_strings - * Last entry must be all 0s - * - * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } - *********************************************************************/ - -static ixl_vendor_info_t ixlv_vendor_info_array[] = -{ - {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_VF, 0, 0, 0}, - {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_VF_HV, 0, 0, 0}, - /* required last entry */ - {0, 0, 0, 0, 0} -}; - -/********************************************************************* - * Table of branding strings - *********************************************************************/ - -static char *ixlv_strings[] = { - "Intel(R) Ethernet Connection XL710 VF Driver" -}; - - -/********************************************************************* - * Function prototypes - *********************************************************************/ -static int ixlv_probe(device_t); -static int ixlv_attach(device_t); -static int ixlv_detach(device_t); -static int ixlv_shutdown(device_t); -static void ixlv_init_locked(struct ixlv_sc *); -static int ixlv_allocate_pci_resources(struct ixlv_sc *); -static void ixlv_free_pci_resources(struct ixlv_sc *); -static int ixlv_assign_msix(struct ixlv_sc *); -static int ixlv_init_msix(struct ixlv_sc *); -static int ixlv_init_taskqueue(struct ixlv_sc *); -static int ixlv_setup_queues(struct ixlv_sc *); -static void ixlv_config_rss(struct ixlv_sc *); -static void ixlv_stop(struct ixlv_sc *); -static void ixlv_add_multi(struct ixl_vsi *); -static void ixlv_del_multi(struct ixl_vsi *); -static void ixlv_free_queues(struct ixl_vsi *); -static int ixlv_setup_interface(device_t, struct ixlv_sc *); - -static int ixlv_media_change(struct ifnet *); -static void ixlv_media_status(struct ifnet *, struct ifmediareq *); - -static void ixlv_local_timer(void *); - -static int ixlv_add_mac_filter(struct ixlv_sc *, u8 *, u16); -static int ixlv_del_mac_filter(struct ixlv_sc *sc, u8 *macaddr); -static void ixlv_init_filters(struct ixlv_sc *); -static void ixlv_free_filters(struct ixlv_sc *); - -static void ixlv_msix_que(void *); -static void ixlv_msix_adminq(void *); -static void ixlv_do_adminq(void *, int); -static void ixlv_do_adminq_locked(struct ixlv_sc *sc); -static void ixlv_handle_que(void *, int); -static int ixlv_reset(struct ixlv_sc *); -static int ixlv_reset_complete(struct i40e_hw *); -static void ixlv_set_queue_rx_itr(struct ixl_queue *); -static void ixlv_set_queue_tx_itr(struct ixl_queue *); -static void ixl_init_cmd_complete(struct ixl_vc_cmd *, void *, - enum i40e_status_code); - -static void ixlv_enable_adminq_irq(struct i40e_hw *); -static void ixlv_disable_adminq_irq(struct i40e_hw *); -static void ixlv_enable_queue_irq(struct i40e_hw *, int); -static void ixlv_disable_queue_irq(struct i40e_hw *, int); - -static void ixlv_setup_vlan_filters(struct ixlv_sc *); -static void ixlv_register_vlan(void *, struct ifnet *, u16); -static void ixlv_unregister_vlan(void *, struct ifnet *, u16); - -static void ixlv_init_hw(struct ixlv_sc *); -static int ixlv_setup_vc(struct ixlv_sc *); -static int ixlv_vf_config(struct ixlv_sc *); - -static void ixlv_cap_txcsum_tso(struct ixl_vsi *, - struct ifnet *, int); - -static void ixlv_add_sysctls(struct ixlv_sc *); -static int ixlv_sysctl_qtx_tail_handler(SYSCTL_HANDLER_ARGS); -static int ixlv_sysctl_qrx_tail_handler(SYSCTL_HANDLER_ARGS); - -/********************************************************************* - * FreeBSD Device Interface Entry Points - *********************************************************************/ - -static device_method_t ixlv_methods[] = { - /* Device interface */ - DEVMETHOD(device_probe, ixlv_probe), - DEVMETHOD(device_attach, ixlv_attach), - DEVMETHOD(device_detach, ixlv_detach), - DEVMETHOD(device_shutdown, ixlv_shutdown), - {0, 0} -}; - -static driver_t ixlv_driver = { - "ixlv", ixlv_methods, sizeof(struct ixlv_sc), -}; - -devclass_t ixlv_devclass; -DRIVER_MODULE(ixlv, pci, ixlv_driver, ixlv_devclass, 0, 0); - -MODULE_DEPEND(ixlv, pci, 1, 1, 1); -MODULE_DEPEND(ixlv, ether, 1, 1, 1); - -/* -** TUNEABLE PARAMETERS: -*/ - -static SYSCTL_NODE(_hw, OID_AUTO, ixlv, CTLFLAG_RD, 0, - "IXLV driver parameters"); - -/* -** Number of descriptors per ring: -** - TX and RX are the same size -*/ -static int ixlv_ringsz = DEFAULT_RING; -TUNABLE_INT("hw.ixlv.ringsz", &ixlv_ringsz); -SYSCTL_INT(_hw_ixlv, OID_AUTO, ring_size, CTLFLAG_RDTUN, - &ixlv_ringsz, 0, "Descriptor Ring Size"); - -/* Set to zero to auto calculate */ -int ixlv_max_queues = 0; -TUNABLE_INT("hw.ixlv.max_queues", &ixlv_max_queues); -SYSCTL_INT(_hw_ixlv, OID_AUTO, max_queues, CTLFLAG_RDTUN, - &ixlv_max_queues, 0, "Number of Queues"); - -/* -** Number of entries in Tx queue buf_ring. -** Increasing this will reduce the number of -** errors when transmitting fragmented UDP -** packets. -*/ -static int ixlv_txbrsz = DEFAULT_TXBRSZ; -TUNABLE_INT("hw.ixlv.txbrsz", &ixlv_txbrsz); -SYSCTL_INT(_hw_ixlv, OID_AUTO, txbr_size, CTLFLAG_RDTUN, - &ixlv_txbrsz, 0, "TX Buf Ring Size"); - -/* -** Controls for Interrupt Throttling -** - true/false for dynamic adjustment -** - default values for static ITR -*/ -int ixlv_dynamic_rx_itr = 0; -TUNABLE_INT("hw.ixlv.dynamic_rx_itr", &ixlv_dynamic_rx_itr); -SYSCTL_INT(_hw_ixlv, OID_AUTO, dynamic_rx_itr, CTLFLAG_RDTUN, - &ixlv_dynamic_rx_itr, 0, "Dynamic RX Interrupt Rate"); - -int ixlv_dynamic_tx_itr = 0; -TUNABLE_INT("hw.ixlv.dynamic_tx_itr", &ixlv_dynamic_tx_itr); -SYSCTL_INT(_hw_ixlv, OID_AUTO, dynamic_tx_itr, CTLFLAG_RDTUN, - &ixlv_dynamic_tx_itr, 0, "Dynamic TX Interrupt Rate"); - -int ixlv_rx_itr = IXL_ITR_8K; -TUNABLE_INT("hw.ixlv.rx_itr", &ixlv_rx_itr); -SYSCTL_INT(_hw_ixlv, OID_AUTO, rx_itr, CTLFLAG_RDTUN, - &ixlv_rx_itr, 0, "RX Interrupt Rate"); - -int ixlv_tx_itr = IXL_ITR_4K; -TUNABLE_INT("hw.ixlv.tx_itr", &ixlv_tx_itr); -SYSCTL_INT(_hw_ixlv, OID_AUTO, tx_itr, CTLFLAG_RDTUN, - &ixlv_tx_itr, 0, "TX Interrupt Rate"); - - -/********************************************************************* - * Device identification routine - * - * ixlv_probe determines if the driver should be loaded on - * the hardware based on PCI vendor/device id of the device. - * - * return BUS_PROBE_DEFAULT on success, positive on failure - *********************************************************************/ - -static int -ixlv_probe(device_t dev) -{ - ixl_vendor_info_t *ent; - - u16 pci_vendor_id, pci_device_id; - u16 pci_subvendor_id, pci_subdevice_id; - char device_name[256]; - - INIT_DEBUGOUT("ixlv_probe: begin"); - - pci_vendor_id = pci_get_vendor(dev); - if (pci_vendor_id != I40E_INTEL_VENDOR_ID) - return (ENXIO); - - pci_device_id = pci_get_device(dev); - pci_subvendor_id = pci_get_subvendor(dev); - pci_subdevice_id = pci_get_subdevice(dev); - - ent = ixlv_vendor_info_array; - while (ent->vendor_id != 0) { - if ((pci_vendor_id == ent->vendor_id) && - (pci_device_id == ent->device_id) && - - ((pci_subvendor_id == ent->subvendor_id) || - (ent->subvendor_id == 0)) && - - ((pci_subdevice_id == ent->subdevice_id) || - (ent->subdevice_id == 0))) { - sprintf(device_name, "%s, Version - %s", - ixlv_strings[ent->index], - ixlv_driver_version); - device_set_desc_copy(dev, device_name); - return (BUS_PROBE_DEFAULT); - } - ent++; - } - return (ENXIO); -} - -/********************************************************************* - * Device initialization routine - * - * The attach entry point is called when the driver is being loaded. - * This routine identifies the type of hardware, allocates all resources - * and initializes the hardware. - * - * return 0 on success, positive on failure - *********************************************************************/ - -static int -ixlv_attach(device_t dev) -{ - struct ixlv_sc *sc; - struct i40e_hw *hw; - struct ixl_vsi *vsi; - int error = 0; - - INIT_DBG_DEV(dev, "begin"); - - /* Allocate, clear, and link in our primary soft structure */ - sc = device_get_softc(dev); - sc->dev = sc->osdep.dev = dev; - hw = &sc->hw; - vsi = &sc->vsi; - vsi->dev = dev; - - /* Initialize hw struct */ - ixlv_init_hw(sc); - - /* Allocate filter lists */ - ixlv_init_filters(sc); - - /* Core Lock Init*/ - mtx_init(&sc->mtx, device_get_nameunit(dev), - "IXL SC Lock", MTX_DEF); - - /* Set up the timer callout */ - callout_init_mtx(&sc->timer, &sc->mtx, 0); - - /* Do PCI setup - map BAR0, etc */ - if (ixlv_allocate_pci_resources(sc)) { - device_printf(dev, "%s: Allocation of PCI resources failed\n", - __func__); - error = ENXIO; - goto err_early; - } - - INIT_DBG_DEV(dev, "Allocated PCI resources and MSIX vectors"); - - error = i40e_set_mac_type(hw); - if (error) { - device_printf(dev, "%s: set_mac_type failed: %d\n", - __func__, error); - goto err_pci_res; - } - - error = ixlv_reset_complete(hw); - if (error) { - device_printf(dev, "%s: Device is still being reset\n", - __func__); - goto err_pci_res; - } - - INIT_DBG_DEV(dev, "VF Device is ready for configuration"); - - error = ixlv_setup_vc(sc); - if (error) { - device_printf(dev, "%s: Error setting up PF comms, %d\n", - __func__, error); - goto err_pci_res; - } - - INIT_DBG_DEV(dev, "PF API version verified"); - - /* TODO: Figure out why MDD events occur when this reset is removed. */ - /* Need API version before sending reset message */ - error = ixlv_reset(sc); - if (error) { - device_printf(dev, "VF reset failed; reload the driver\n"); - goto err_aq; - } - - INIT_DBG_DEV(dev, "VF reset complete"); - - /* Ask for VF config from PF */ - error = ixlv_vf_config(sc); - if (error) { - device_printf(dev, "Error getting configuration from PF: %d\n", - error); - goto err_aq; - } - - INIT_DBG_DEV(dev, "VF config from PF:"); - INIT_DBG_DEV(dev, "VSIs %d, Queues %d, Max Vectors %d, Max MTU %d", - sc->vf_res->num_vsis, - sc->vf_res->num_queue_pairs, - sc->vf_res->max_vectors, - sc->vf_res->max_mtu); - INIT_DBG_DEV(dev, "Offload flags: %#010x", - sc->vf_res->vf_offload_flags); - - // TODO: Move this into ixlv_vf_config? - /* got VF config message back from PF, now we can parse it */ - for (int i = 0; i < sc->vf_res->num_vsis; i++) { - if (sc->vf_res->vsi_res[i].vsi_type == I40E_VSI_SRIOV) - sc->vsi_res = &sc->vf_res->vsi_res[i]; - } - if (!sc->vsi_res) { - device_printf(dev, "%s: no LAN VSI found\n", __func__); - error = EIO; - goto err_res_buf; - } - - INIT_DBG_DEV(dev, "Resource Acquisition complete"); - - /* If no mac address was assigned just make a random one */ - if (!ixlv_check_ether_addr(hw->mac.addr)) { - u8 addr[ETHER_ADDR_LEN]; - arc4rand(&addr, sizeof(addr), 0); - addr[0] &= 0xFE; - addr[0] |= 0x02; - bcopy(addr, hw->mac.addr, sizeof(addr)); - } - - vsi->id = sc->vsi_res->vsi_id; - vsi->back = (void *)sc; - sc->link_up = TRUE; - - /* This allocates the memory and early settings */ - if (ixlv_setup_queues(sc) != 0) { - device_printf(dev, "%s: setup queues failed!\n", - __func__); - error = EIO; - goto out; - } - - /* Setup the stack interface */ - if (ixlv_setup_interface(dev, sc) != 0) { - device_printf(dev, "%s: setup interface failed!\n", - __func__); - error = EIO; - goto out; - } - - INIT_DBG_DEV(dev, "Queue memory and interface setup"); - - /* Do queue interrupt setup */ - ixlv_assign_msix(sc); - - /* Start AdminQ taskqueue */ - ixlv_init_taskqueue(sc); - - /* Initialize stats */ - bzero(&sc->vsi.eth_stats, sizeof(struct i40e_eth_stats)); - ixlv_add_sysctls(sc); - - /* Register for VLAN events */ - vsi->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, - ixlv_register_vlan, vsi, EVENTHANDLER_PRI_FIRST); - vsi->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, - ixlv_unregister_vlan, vsi, EVENTHANDLER_PRI_FIRST); - - /* We want AQ enabled early */ - ixlv_enable_adminq_irq(hw); - - /* Set things up to run init */ - sc->init_state = IXLV_INIT_READY; - - ixl_vc_init_mgr(sc, &sc->vc_mgr); - - INIT_DBG_DEV(dev, "end"); - return (error); - -out: - ixlv_free_queues(vsi); -err_res_buf: - free(sc->vf_res, M_DEVBUF); -err_aq: - i40e_shutdown_adminq(hw); -err_pci_res: - ixlv_free_pci_resources(sc); -err_early: - mtx_destroy(&sc->mtx); - ixlv_free_filters(sc); - INIT_DBG_DEV(dev, "end: error %d", error); - return (error); -} - -/********************************************************************* - * Device removal routine - * - * The detach entry point is called when the driver is being removed. - * This routine stops the adapter and deallocates all the resources - * that were allocated for driver operation. - * - * return 0 on success, positive on failure - *********************************************************************/ - -static int -ixlv_detach(device_t dev) -{ - struct ixlv_sc *sc = device_get_softc(dev); - struct ixl_vsi *vsi = &sc->vsi; - - INIT_DBG_DEV(dev, "begin"); - - /* Make sure VLANS are not using driver */ - if (vsi->ifp->if_vlantrunk != NULL) { - if_printf(vsi->ifp, "Vlan in use, detach first\n"); - INIT_DBG_DEV(dev, "end"); - return (EBUSY); - } - - /* Stop driver */ - ether_ifdetach(vsi->ifp); - if (vsi->ifp->if_drv_flags & IFF_DRV_RUNNING) { - mtx_lock(&sc->mtx); - ixlv_stop(sc); - mtx_unlock(&sc->mtx); - } - - /* Unregister VLAN events */ - if (vsi->vlan_attach != NULL) - EVENTHANDLER_DEREGISTER(vlan_config, vsi->vlan_attach); - if (vsi->vlan_detach != NULL) - EVENTHANDLER_DEREGISTER(vlan_unconfig, vsi->vlan_detach); - - /* Drain VC mgr */ - callout_drain(&sc->vc_mgr.callout); - - i40e_shutdown_adminq(&sc->hw); - taskqueue_free(sc->tq); - if_free(vsi->ifp); - free(sc->vf_res, M_DEVBUF); - ixlv_free_pci_resources(sc); - ixlv_free_queues(vsi); - mtx_destroy(&sc->mtx); - ixlv_free_filters(sc); - - bus_generic_detach(dev); - INIT_DBG_DEV(dev, "end"); - return (0); -} - -/********************************************************************* - * - * Shutdown entry point - * - **********************************************************************/ - -static int -ixlv_shutdown(device_t dev) -{ - struct ixlv_sc *sc = device_get_softc(dev); - - INIT_DBG_DEV(dev, "begin"); - - mtx_lock(&sc->mtx); - ixlv_stop(sc); - mtx_unlock(&sc->mtx); - - INIT_DBG_DEV(dev, "end"); - return (0); -} - -/* - * Configure TXCSUM(IPV6) and TSO(4/6) - * - the hardware handles these together so we - * need to tweak them - */ -static void -ixlv_cap_txcsum_tso(struct ixl_vsi *vsi, struct ifnet *ifp, int mask) -{ - /* Enable/disable TXCSUM/TSO4 */ - if (!(ifp->if_capenable & IFCAP_TXCSUM) - && !(ifp->if_capenable & IFCAP_TSO4)) { - if (mask & IFCAP_TXCSUM) { - ifp->if_capenable |= IFCAP_TXCSUM; - /* enable TXCSUM, restore TSO if previously enabled */ - if (vsi->flags & IXL_FLAGS_KEEP_TSO4) { - vsi->flags &= ~IXL_FLAGS_KEEP_TSO4; - ifp->if_capenable |= IFCAP_TSO4; - } - } - else if (mask & IFCAP_TSO4) { - ifp->if_capenable |= (IFCAP_TXCSUM | IFCAP_TSO4); - vsi->flags &= ~IXL_FLAGS_KEEP_TSO4; - if_printf(ifp, - "TSO4 requires txcsum, enabling both...\n"); - } - } else if((ifp->if_capenable & IFCAP_TXCSUM) - && !(ifp->if_capenable & IFCAP_TSO4)) { - if (mask & IFCAP_TXCSUM) - ifp->if_capenable &= ~IFCAP_TXCSUM; - else if (mask & IFCAP_TSO4) - ifp->if_capenable |= IFCAP_TSO4; - } else if((ifp->if_capenable & IFCAP_TXCSUM) - && (ifp->if_capenable & IFCAP_TSO4)) { - if (mask & IFCAP_TXCSUM) { - vsi->flags |= IXL_FLAGS_KEEP_TSO4; - ifp->if_capenable &= ~(IFCAP_TXCSUM | IFCAP_TSO4); - if_printf(ifp, - "TSO4 requires txcsum, disabling both...\n"); - } else if (mask & IFCAP_TSO4) - ifp->if_capenable &= ~IFCAP_TSO4; - } - - /* Enable/disable TXCSUM_IPV6/TSO6 */ - if (!(ifp->if_capenable & IFCAP_TXCSUM_IPV6) - && !(ifp->if_capenable & IFCAP_TSO6)) { - if (mask & IFCAP_TXCSUM_IPV6) { - ifp->if_capenable |= IFCAP_TXCSUM_IPV6; - if (vsi->flags & IXL_FLAGS_KEEP_TSO6) { - vsi->flags &= ~IXL_FLAGS_KEEP_TSO6; - ifp->if_capenable |= IFCAP_TSO6; - } - } else if (mask & IFCAP_TSO6) { - ifp->if_capenable |= (IFCAP_TXCSUM_IPV6 | IFCAP_TSO6); - vsi->flags &= ~IXL_FLAGS_KEEP_TSO6; - if_printf(ifp, - "TSO6 requires txcsum6, enabling both...\n"); - } - } else if((ifp->if_capenable & IFCAP_TXCSUM_IPV6) - && !(ifp->if_capenable & IFCAP_TSO6)) { - if (mask & IFCAP_TXCSUM_IPV6) - ifp->if_capenable &= ~IFCAP_TXCSUM_IPV6; - else if (mask & IFCAP_TSO6) - ifp->if_capenable |= IFCAP_TSO6; - } else if ((ifp->if_capenable & IFCAP_TXCSUM_IPV6) - && (ifp->if_capenable & IFCAP_TSO6)) { - if (mask & IFCAP_TXCSUM_IPV6) { - vsi->flags |= IXL_FLAGS_KEEP_TSO6; - ifp->if_capenable &= ~(IFCAP_TXCSUM_IPV6 | IFCAP_TSO6); - if_printf(ifp, - "TSO6 requires txcsum6, disabling both...\n"); - } else if (mask & IFCAP_TSO6) - ifp->if_capenable &= ~IFCAP_TSO6; - } -} - -/********************************************************************* - * Ioctl entry point - * - * ixlv_ioctl is called when the user wants to configure the - * interface. - * - * return 0 on success, positive on failure - **********************************************************************/ - -static int -ixlv_ioctl(struct ifnet *ifp, u_long command, caddr_t data) -{ - struct ixl_vsi *vsi = ifp->if_softc; - struct ixlv_sc *sc = vsi->back; - struct ifreq *ifr = (struct ifreq *)data; -#if defined(INET) || defined(INET6) - struct ifaddr *ifa = (struct ifaddr *)data; - bool avoid_reset = FALSE; -#endif - int error = 0; - - - switch (command) { - - case SIOCSIFADDR: -#ifdef INET - if (ifa->ifa_addr->sa_family == AF_INET) - avoid_reset = TRUE; -#endif -#ifdef INET6 - if (ifa->ifa_addr->sa_family == AF_INET6) - avoid_reset = TRUE; -#endif -#if defined(INET) || defined(INET6) - /* - ** Calling init results in link renegotiation, - ** so we avoid doing it when possible. - */ - if (avoid_reset) { - ifp->if_flags |= IFF_UP; - if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) - ixlv_init(vsi); -#ifdef INET - if (!(ifp->if_flags & IFF_NOARP)) - arp_ifinit(ifp, ifa); -#endif - } else - error = ether_ioctl(ifp, command, data); - break; -#endif - case SIOCSIFMTU: - IOCTL_DBG_IF2(ifp, "SIOCSIFMTU (Set Interface MTU)"); - mtx_lock(&sc->mtx); - if (ifr->ifr_mtu > IXL_MAX_FRAME - - ETHER_HDR_LEN - ETHER_CRC_LEN - ETHER_VLAN_ENCAP_LEN) { - error = EINVAL; - IOCTL_DBG_IF(ifp, "mtu too large"); - } else { - IOCTL_DBG_IF2(ifp, "mtu: %lu -> %d", ifp->if_mtu, ifr->ifr_mtu); - // ERJ: Interestingly enough, these types don't match - ifp->if_mtu = (u_long)ifr->ifr_mtu; - vsi->max_frame_size = - ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN - + ETHER_VLAN_ENCAP_LEN; - ixlv_init_locked(sc); - } - mtx_unlock(&sc->mtx); - break; - case SIOCSIFFLAGS: - IOCTL_DBG_IF2(ifp, "SIOCSIFFLAGS (Set Interface Flags)"); - mtx_lock(&sc->mtx); - if (ifp->if_flags & IFF_UP) { - if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) - ixlv_init_locked(sc); - } else - if (ifp->if_drv_flags & IFF_DRV_RUNNING) - ixlv_stop(sc); - sc->if_flags = ifp->if_flags; - mtx_unlock(&sc->mtx); - break; - case SIOCADDMULTI: - IOCTL_DBG_IF2(ifp, "SIOCADDMULTI"); - if (ifp->if_drv_flags & IFF_DRV_RUNNING) { - mtx_lock(&sc->mtx); - ixlv_disable_intr(vsi); - ixlv_add_multi(vsi); - ixlv_enable_intr(vsi); - mtx_unlock(&sc->mtx); - } - break; - case SIOCDELMULTI: - IOCTL_DBG_IF2(ifp, "SIOCDELMULTI"); - if (sc->init_state == IXLV_RUNNING) { - mtx_lock(&sc->mtx); - ixlv_disable_intr(vsi); - ixlv_del_multi(vsi); - ixlv_enable_intr(vsi); - mtx_unlock(&sc->mtx); - } - break; - case SIOCSIFMEDIA: - case SIOCGIFMEDIA: - IOCTL_DBG_IF2(ifp, "SIOCxIFMEDIA (Get/Set Interface Media)"); - error = ifmedia_ioctl(ifp, ifr, &sc->media, command); - break; - case SIOCSIFCAP: - { - int mask = ifr->ifr_reqcap ^ ifp->if_capenable; - IOCTL_DBG_IF2(ifp, "SIOCSIFCAP (Set Capabilities)"); - - ixlv_cap_txcsum_tso(vsi, ifp, mask); - - if (mask & IFCAP_RXCSUM) - ifp->if_capenable ^= IFCAP_RXCSUM; - if (mask & IFCAP_RXCSUM_IPV6) - ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; - if (mask & IFCAP_LRO) - ifp->if_capenable ^= IFCAP_LRO; - if (mask & IFCAP_VLAN_HWTAGGING) - ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; - if (mask & IFCAP_VLAN_HWFILTER) - ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; - if (mask & IFCAP_VLAN_HWTSO) - ifp->if_capenable ^= IFCAP_VLAN_HWTSO; - if (ifp->if_drv_flags & IFF_DRV_RUNNING) { - ixlv_init(vsi); - } - VLAN_CAPABILITIES(ifp); - - break; - } - - default: - IOCTL_DBG_IF2(ifp, "UNKNOWN (0x%X)", (int)command); - error = ether_ioctl(ifp, command, data); - break; - } - - return (error); -} - -/* -** To do a reinit on the VF is unfortunately more complicated -** than a physical device, we must have the PF more or less -** completely recreate our memory, so many things that were -** done only once at attach in traditional drivers now must be -** redone at each reinitialization. This function does that -** 'prelude' so we can then call the normal locked init code. -*/ -int -ixlv_reinit_locked(struct ixlv_sc *sc) -{ - struct i40e_hw *hw = &sc->hw; - struct ixl_vsi *vsi = &sc->vsi; - struct ifnet *ifp = vsi->ifp; - struct ixlv_mac_filter *mf, *mf_temp; - struct ixlv_vlan_filter *vf; - int error = 0; - - INIT_DBG_IF(ifp, "begin"); - - if (ifp->if_drv_flags & IFF_DRV_RUNNING) - ixlv_stop(sc); - - error = ixlv_reset(sc); - - INIT_DBG_IF(ifp, "VF was reset"); - - /* set the state in case we went thru RESET */ - sc->init_state = IXLV_RUNNING; - - /* - ** Resetting the VF drops all filters from hardware; - ** we need to mark them to be re-added in init. - */ - SLIST_FOREACH_SAFE(mf, sc->mac_filters, next, mf_temp) { - if (mf->flags & IXL_FILTER_DEL) { - SLIST_REMOVE(sc->mac_filters, mf, - ixlv_mac_filter, next); - free(mf, M_DEVBUF); - } else - mf->flags |= IXL_FILTER_ADD; - } - if (vsi->num_vlans != 0) - SLIST_FOREACH(vf, sc->vlan_filters, next) - vf->flags = IXL_FILTER_ADD; - else { /* clean any stale filters */ - while (!SLIST_EMPTY(sc->vlan_filters)) { - vf = SLIST_FIRST(sc->vlan_filters); - SLIST_REMOVE_HEAD(sc->vlan_filters, next); - free(vf, M_DEVBUF); - } - } - - ixlv_enable_adminq_irq(hw); - ixl_vc_flush(&sc->vc_mgr); - - INIT_DBG_IF(ifp, "end"); - return (error); -} - -static void -ixl_init_cmd_complete(struct ixl_vc_cmd *cmd, void *arg, - enum i40e_status_code code) -{ - struct ixlv_sc *sc; - - sc = arg; - - /* - * Ignore "Adapter Stopped" message as that happens if an ifconfig down - * happens while a command is in progress, so we don't print an error - * in that case. - */ - if (code != I40E_SUCCESS && code != I40E_ERR_ADAPTER_STOPPED) { - if_printf(sc->vsi.ifp, - "Error %d waiting for PF to complete operation %d\n", - code, cmd->request); - } -} - -static void -ixlv_init_locked(struct ixlv_sc *sc) -{ - struct i40e_hw *hw = &sc->hw; - struct ixl_vsi *vsi = &sc->vsi; - struct ixl_queue *que = vsi->queues; - struct ifnet *ifp = vsi->ifp; - int error = 0; - - INIT_DBG_IF(ifp, "begin"); - - IXLV_CORE_LOCK_ASSERT(sc); - - /* Do a reinit first if an init has already been done */ - if ((sc->init_state == IXLV_RUNNING) || - (sc->init_state == IXLV_RESET_REQUIRED) || - (sc->init_state == IXLV_RESET_PENDING)) - error = ixlv_reinit_locked(sc); - /* Don't bother with init if we failed reinit */ - if (error) - goto init_done; - - /* Remove existing MAC filter if new MAC addr is set */ - if (bcmp(IF_LLADDR(ifp), hw->mac.addr, ETHER_ADDR_LEN) != 0) { - error = ixlv_del_mac_filter(sc, hw->mac.addr); - if (error == 0) - ixl_vc_enqueue(&sc->vc_mgr, &sc->del_mac_cmd, - IXLV_FLAG_AQ_DEL_MAC_FILTER, ixl_init_cmd_complete, - sc); - } - - /* Check for an LAA mac address... */ - bcopy(IF_LLADDR(ifp), hw->mac.addr, ETHER_ADDR_LEN); - - ifp->if_hwassist = 0; - if (ifp->if_capenable & IFCAP_TSO) - ifp->if_hwassist |= CSUM_TSO; - if (ifp->if_capenable & IFCAP_TXCSUM) - ifp->if_hwassist |= (CSUM_OFFLOAD_IPV4 & ~CSUM_IP); - if (ifp->if_capenable & IFCAP_TXCSUM_IPV6) - ifp->if_hwassist |= CSUM_OFFLOAD_IPV6; - - /* Add mac filter for this VF to PF */ - if (i40e_validate_mac_addr(hw->mac.addr) == I40E_SUCCESS) { - error = ixlv_add_mac_filter(sc, hw->mac.addr, 0); - if (!error || error == EEXIST) - ixl_vc_enqueue(&sc->vc_mgr, &sc->add_mac_cmd, - IXLV_FLAG_AQ_ADD_MAC_FILTER, ixl_init_cmd_complete, - sc); - } - - /* Setup vlan's if needed */ - ixlv_setup_vlan_filters(sc); - - /* Prepare the queues for operation */ - for (int i = 0; i < vsi->num_queues; i++, que++) { - struct rx_ring *rxr = &que->rxr; - - ixl_init_tx_ring(que); - - if (vsi->max_frame_size <= MCLBYTES) - rxr->mbuf_sz = MCLBYTES; - else - rxr->mbuf_sz = MJUMPAGESIZE; - ixl_init_rx_ring(que); - } - - /* Configure queues */ - ixl_vc_enqueue(&sc->vc_mgr, &sc->config_queues_cmd, - IXLV_FLAG_AQ_CONFIGURE_QUEUES, ixl_init_cmd_complete, sc); - - /* Set up RSS */ - ixlv_config_rss(sc); - - /* Map vectors */ - ixl_vc_enqueue(&sc->vc_mgr, &sc->map_vectors_cmd, - IXLV_FLAG_AQ_MAP_VECTORS, ixl_init_cmd_complete, sc); - - /* Enable queues */ - ixl_vc_enqueue(&sc->vc_mgr, &sc->enable_queues_cmd, - IXLV_FLAG_AQ_ENABLE_QUEUES, ixl_init_cmd_complete, sc); - - /* Start the local timer */ - callout_reset(&sc->timer, hz, ixlv_local_timer, sc); - - sc->init_state = IXLV_RUNNING; - -init_done: - INIT_DBG_IF(ifp, "end"); - return; -} - -/* -** Init entry point for the stack -*/ -void -ixlv_init(void *arg) -{ - struct ixl_vsi *vsi = (struct ixl_vsi *)arg; - struct ixlv_sc *sc = vsi->back; - int retries = 0; - - mtx_lock(&sc->mtx); - ixlv_init_locked(sc); - mtx_unlock(&sc->mtx); - - /* Wait for init_locked to finish */ - while (!(vsi->ifp->if_drv_flags & IFF_DRV_RUNNING) - && ++retries < 100) { - i40e_msec_delay(10); - } - if (retries >= IXLV_AQ_MAX_ERR) - if_printf(vsi->ifp, - "Init failed to complete in alloted time!\n"); -} - -/* - * ixlv_attach() helper function; gathers information about - * the (virtual) hardware for use elsewhere in the driver. - */ -static void -ixlv_init_hw(struct ixlv_sc *sc) -{ - struct i40e_hw *hw = &sc->hw; - device_t dev = sc->dev; - - /* Save off the information about this board */ - hw->vendor_id = pci_get_vendor(dev); - hw->device_id = pci_get_device(dev); - hw->revision_id = pci_read_config(dev, PCIR_REVID, 1); - hw->subsystem_vendor_id = - pci_read_config(dev, PCIR_SUBVEND_0, 2); - hw->subsystem_device_id = - pci_read_config(dev, PCIR_SUBDEV_0, 2); - - hw->bus.device = pci_get_slot(dev); - hw->bus.func = pci_get_function(dev); -} - -/* - * ixlv_attach() helper function; initalizes the admin queue - * and attempts to establish contact with the PF by - * retrying the initial "API version" message several times - * or until the PF responds. - */ -static int -ixlv_setup_vc(struct ixlv_sc *sc) -{ - struct i40e_hw *hw = &sc->hw; - device_t dev = sc->dev; - int error = 0, ret_error = 0, asq_retries = 0; - bool send_api_ver_retried = 0; - - /* Need to set these AQ paramters before initializing AQ */ - hw->aq.num_arq_entries = IXL_AQ_LEN; - hw->aq.num_asq_entries = IXL_AQ_LEN; - hw->aq.arq_buf_size = IXL_AQ_BUFSZ; - hw->aq.asq_buf_size = IXL_AQ_BUFSZ; - - for (int i = 0; i < IXLV_AQ_MAX_ERR; i++) { - /* Initialize admin queue */ - error = i40e_init_adminq(hw); - if (error) { - device_printf(dev, "%s: init_adminq failed: %d\n", - __func__, error); - ret_error = 1; - continue; - } - - INIT_DBG_DEV(dev, "Initialized Admin Queue, attempt %d", i+1); - -retry_send: - /* Send VF's API version */ - error = ixlv_send_api_ver(sc); - if (error) { - i40e_shutdown_adminq(hw); - ret_error = 2; - device_printf(dev, "%s: unable to send api" - " version to PF on attempt %d, error %d\n", - __func__, i+1, error); - } - - asq_retries = 0; - while (!i40e_asq_done(hw)) { - if (++asq_retries > IXLV_AQ_MAX_ERR) { - i40e_shutdown_adminq(hw); - DDPRINTF(dev, "Admin Queue timeout " - "(waiting for send_api_ver), %d more retries...", - IXLV_AQ_MAX_ERR - (i + 1)); - ret_error = 3; - break; - } - i40e_msec_delay(10); - } - if (asq_retries > IXLV_AQ_MAX_ERR) - continue; - - INIT_DBG_DEV(dev, "Sent API version message to PF"); - - /* Verify that the VF accepts the PF's API version */ - error = ixlv_verify_api_ver(sc); - if (error == ETIMEDOUT) { - if (!send_api_ver_retried) { - /* Resend message, one more time */ - send_api_ver_retried++; - device_printf(dev, - "%s: Timeout while verifying API version on first" - " try!\n", __func__); - goto retry_send; - } else { - device_printf(dev, - "%s: Timeout while verifying API version on second" - " try!\n", __func__); - ret_error = 4; - break; - } - } - if (error) { - device_printf(dev, - "%s: Unable to verify API version," - " error %d\n", __func__, error); - ret_error = 5; - } - break; - } - - if (ret_error >= 4) - i40e_shutdown_adminq(hw); - return (ret_error); -} - -/* - * ixlv_attach() helper function; asks the PF for this VF's - * configuration, and saves the information if it receives it. - */ -static int -ixlv_vf_config(struct ixlv_sc *sc) -{ - struct i40e_hw *hw = &sc->hw; - device_t dev = sc->dev; - int bufsz, error = 0, ret_error = 0; - int asq_retries, retried = 0; - -retry_config: - error = ixlv_send_vf_config_msg(sc); - if (error) { - device_printf(dev, - "%s: Unable to send VF config request, attempt %d," - " error %d\n", __func__, retried + 1, error); - ret_error = 2; - } - - asq_retries = 0; - while (!i40e_asq_done(hw)) { - if (++asq_retries > IXLV_AQ_MAX_ERR) { - device_printf(dev, "%s: Admin Queue timeout " - "(waiting for send_vf_config_msg), attempt %d\n", - __func__, retried + 1); - ret_error = 3; - goto fail; - } - i40e_msec_delay(10); - } - - INIT_DBG_DEV(dev, "Sent VF config message to PF, attempt %d", - retried + 1); - - if (!sc->vf_res) { - bufsz = sizeof(struct i40e_virtchnl_vf_resource) + - (I40E_MAX_VF_VSI * sizeof(struct i40e_virtchnl_vsi_resource)); - sc->vf_res = malloc(bufsz, M_DEVBUF, M_NOWAIT); - if (!sc->vf_res) { - device_printf(dev, - "%s: Unable to allocate memory for VF configuration" - " message from PF on attempt %d\n", __func__, retried + 1); - ret_error = 1; - goto fail; - } - } - - /* Check for VF config response */ - error = ixlv_get_vf_config(sc); - if (error == ETIMEDOUT) { - /* The 1st time we timeout, send the configuration message again */ - if (!retried) { - retried++; - goto retry_config; - } - } - if (error) { - device_printf(dev, - "%s: Unable to get VF configuration from PF after %d tries!\n", - __func__, retried + 1); - ret_error = 4; - } - goto done; - -fail: - free(sc->vf_res, M_DEVBUF); -done: - return (ret_error); -} - -/* - * Allocate MSI/X vectors, setup the AQ vector early - */ -static int -ixlv_init_msix(struct ixlv_sc *sc) -{ - device_t dev = sc->dev; - int rid, want, vectors, queues, available; - - rid = PCIR_BAR(IXL_BAR); - sc->msix_mem = bus_alloc_resource_any(dev, - SYS_RES_MEMORY, &rid, RF_ACTIVE); - if (!sc->msix_mem) { - /* May not be enabled */ - device_printf(sc->dev, - "Unable to map MSIX table \n"); - goto fail; - } - - available = pci_msix_count(dev); - if (available == 0) { /* system has msix disabled */ - bus_release_resource(dev, SYS_RES_MEMORY, - rid, sc->msix_mem); - sc->msix_mem = NULL; - goto fail; - } - - /* Figure out a reasonable auto config value */ - queues = (mp_ncpus > (available - 1)) ? (available - 1) : mp_ncpus; - - /* Override with hardcoded value if sane */ - if ((ixlv_max_queues != 0) && (ixlv_max_queues <= queues)) - queues = ixlv_max_queues; -#ifdef RSS - /* If we're doing RSS, clamp at the number of RSS buckets */ - if (queues > rss_getnumbuckets()) - queues = rss_getnumbuckets(); -#endif - /* Enforce the VF max value */ - if (queues > IXLV_MAX_QUEUES) - queues = IXLV_MAX_QUEUES; - - /* - ** Want one vector (RX/TX pair) per queue - ** plus an additional for the admin queue. - */ - want = queues + 1; - if (want <= available) /* Have enough */ - vectors = want; - else { - device_printf(sc->dev, - "MSIX Configuration Problem, " - "%d vectors available but %d wanted!\n", - available, want); - goto fail; - } - -#ifdef RSS - /* - * If we're doing RSS, the number of queues needs to - * match the number of RSS buckets that are configured. - * - * + If there's more queues than RSS buckets, we'll end - * up with queues that get no traffic. - * - * + If there's more RSS buckets than queues, we'll end - * up having multiple RSS buckets map to the same queue, - * so there'll be some contention. - */ - if (queues != rss_getnumbuckets()) { - device_printf(dev, - "%s: queues (%d) != RSS buckets (%d)" - "; performance will be impacted.\n", - __func__, queues, rss_getnumbuckets()); - } -#endif - - if (pci_alloc_msix(dev, &vectors) == 0) { - device_printf(sc->dev, - "Using MSIX interrupts with %d vectors\n", vectors); - sc->msix = vectors; - sc->vsi.num_queues = queues; - } - - /* - ** Explicitly set the guest PCI BUSMASTER capability - ** and we must rewrite the ENABLE in the MSIX control - ** register again at this point to cause the host to - ** successfully initialize us. - */ - { - u16 pci_cmd_word; - int msix_ctrl; - pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); - pci_cmd_word |= PCIM_CMD_BUSMASTEREN; - pci_write_config(dev, PCIR_COMMAND, pci_cmd_word, 2); - pci_find_cap(dev, PCIY_MSIX, &rid); - rid += PCIR_MSIX_CTRL; - msix_ctrl = pci_read_config(dev, rid, 2); - msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE; - pci_write_config(dev, rid, msix_ctrl, 2); - } - - /* Next we need to setup the vector for the Admin Queue */ - rid = 1; // zero vector + 1 - sc->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, - &rid, RF_SHAREABLE | RF_ACTIVE); - if (sc->res == NULL) { - device_printf(dev,"Unable to allocate" - " bus resource: AQ interrupt \n"); - goto fail; - } - if (bus_setup_intr(dev, sc->res, - INTR_TYPE_NET | INTR_MPSAFE, NULL, - ixlv_msix_adminq, sc, &sc->tag)) { - sc->res = NULL; - device_printf(dev, "Failed to register AQ handler"); - goto fail; - } - bus_describe_intr(dev, sc->res, sc->tag, "adminq"); - - return (vectors); - -fail: - /* The VF driver MUST use MSIX */ - return (0); -} - -static int -ixlv_allocate_pci_resources(struct ixlv_sc *sc) -{ - int rid; - device_t dev = sc->dev; - - rid = PCIR_BAR(0); - sc->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, - &rid, RF_ACTIVE); - - if (!(sc->pci_mem)) { - device_printf(dev,"Unable to allocate bus resource: memory\n"); - return (ENXIO); - } - - sc->osdep.mem_bus_space_tag = - rman_get_bustag(sc->pci_mem); - sc->osdep.mem_bus_space_handle = - rman_get_bushandle(sc->pci_mem); - sc->osdep.mem_bus_space_size = rman_get_size(sc->pci_mem); - sc->osdep.flush_reg = I40E_VFGEN_RSTAT; - sc->hw.hw_addr = (u8 *) &sc->osdep.mem_bus_space_handle; - - sc->hw.back = &sc->osdep; - - /* Disable adminq interrupts */ - ixlv_disable_adminq_irq(&sc->hw); - - /* - ** Now setup MSI/X, it will return - ** us the number of supported vectors - */ - sc->msix = ixlv_init_msix(sc); - - /* We fail without MSIX support */ - if (sc->msix == 0) - return (ENXIO); - - return (0); -} - -static void -ixlv_free_pci_resources(struct ixlv_sc *sc) -{ - struct ixl_vsi *vsi = &sc->vsi; - struct ixl_queue *que = vsi->queues; - device_t dev = sc->dev; - - /* We may get here before stations are setup */ - if (que == NULL) - goto early; - - /* - ** Release all msix queue resources: - */ - for (int i = 0; i < vsi->num_queues; i++, que++) { - int rid = que->msix + 1; - if (que->tag != NULL) { - bus_teardown_intr(dev, que->res, que->tag); - que->tag = NULL; - } - if (que->res != NULL) - bus_release_resource(dev, SYS_RES_IRQ, rid, que->res); - } - -early: - /* Clean the AdminQ interrupt */ - if (sc->tag != NULL) { - bus_teardown_intr(dev, sc->res, sc->tag); - sc->tag = NULL; - } - if (sc->res != NULL) - bus_release_resource(dev, SYS_RES_IRQ, 1, sc->res); - - pci_release_msi(dev); - - if (sc->msix_mem != NULL) - bus_release_resource(dev, SYS_RES_MEMORY, - PCIR_BAR(IXL_BAR), sc->msix_mem); - - if (sc->pci_mem != NULL) - bus_release_resource(dev, SYS_RES_MEMORY, - PCIR_BAR(0), sc->pci_mem); - - return; -} - -/* - * Create taskqueue and tasklet for Admin Queue interrupts. - */ -static int -ixlv_init_taskqueue(struct ixlv_sc *sc) -{ - int error = 0; - - TASK_INIT(&sc->aq_irq, 0, ixlv_do_adminq, sc); - - sc->tq = taskqueue_create_fast("ixl_adm", M_NOWAIT, - taskqueue_thread_enqueue, &sc->tq); - taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s sc->tq", - device_get_nameunit(sc->dev)); - - return (error); -} - -/********************************************************************* - * - * Setup MSIX Interrupt resources and handlers for the VSI queues - * - **********************************************************************/ -static int -ixlv_assign_msix(struct ixlv_sc *sc) -{ - device_t dev = sc->dev; - struct ixl_vsi *vsi = &sc->vsi; - struct ixl_queue *que = vsi->queues; - struct tx_ring *txr; - int error, rid, vector = 1; -#ifdef RSS - cpuset_t cpu_mask; -#endif - - for (int i = 0; i < vsi->num_queues; i++, vector++, que++) { - int cpu_id = i; - rid = vector + 1; - txr = &que->txr; - que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, - RF_SHAREABLE | RF_ACTIVE); - if (que->res == NULL) { - device_printf(dev,"Unable to allocate" - " bus resource: que interrupt [%d]\n", vector); - return (ENXIO); - } - /* Set the handler function */ - error = bus_setup_intr(dev, que->res, - INTR_TYPE_NET | INTR_MPSAFE, NULL, - ixlv_msix_que, que, &que->tag); - if (error) { - que->res = NULL; - device_printf(dev, "Failed to register que handler"); - return (error); - } - bus_describe_intr(dev, que->res, que->tag, "que %d", i); - /* Bind the vector to a CPU */ -#ifdef RSS - cpu_id = rss_getcpu(i % rss_getnumbuckets()); -#endif - bus_bind_intr(dev, que->res, cpu_id); - que->msix = vector; - vsi->que_mask |= (u64)(1 << que->msix); - TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que); - TASK_INIT(&que->task, 0, ixlv_handle_que, que); - que->tq = taskqueue_create_fast("ixlv_que", M_NOWAIT, - taskqueue_thread_enqueue, &que->tq); -#ifdef RSS - CPU_SETOF(cpu_id, &cpu_mask); - taskqueue_start_threads_cpuset(&que->tq, 1, PI_NET, - &cpu_mask, "%s (bucket %d)", - device_get_nameunit(dev), cpu_id); -#else - taskqueue_start_threads(&que->tq, 1, PI_NET, - "%s que", device_get_nameunit(dev)); -#endif - - } - - return (0); -} - -/* -** Requests a VF reset from the PF. -** -** Requires the VF's Admin Queue to be initialized. -*/ -static int -ixlv_reset(struct ixlv_sc *sc) -{ - struct i40e_hw *hw = &sc->hw; - device_t dev = sc->dev; - int error = 0; - - /* Ask the PF to reset us if we are initiating */ - if (sc->init_state != IXLV_RESET_PENDING) - ixlv_request_reset(sc); - - i40e_msec_delay(100); - error = ixlv_reset_complete(hw); - if (error) { - device_printf(dev, "%s: VF reset failed\n", - __func__); - return (error); - } - - error = i40e_shutdown_adminq(hw); - if (error) { - device_printf(dev, "%s: shutdown_adminq failed: %d\n", - __func__, error); - return (error); - } - - error = i40e_init_adminq(hw); - if (error) { - device_printf(dev, "%s: init_adminq failed: %d\n", - __func__, error); - return(error); - } - - return (0); -} - -static int -ixlv_reset_complete(struct i40e_hw *hw) -{ - u32 reg; - - for (int i = 0; i < 100; i++) { - reg = rd32(hw, I40E_VFGEN_RSTAT) & - I40E_VFGEN_RSTAT_VFR_STATE_MASK; - - if ((reg == I40E_VFR_VFACTIVE) || - (reg == I40E_VFR_COMPLETED)) - return (0); - i40e_msec_delay(100); - } - - return (EBUSY); -} - - -/********************************************************************* - * - * Setup networking device structure and register an interface. - * - **********************************************************************/ -static int -ixlv_setup_interface(device_t dev, struct ixlv_sc *sc) -{ - struct ifnet *ifp; - struct ixl_vsi *vsi = &sc->vsi; - struct ixl_queue *que = vsi->queues; - - INIT_DBG_DEV(dev, "begin"); - - ifp = vsi->ifp = if_alloc(IFT_ETHER); - if (ifp == NULL) { - device_printf(dev, "%s: could not allocate ifnet" - " structure!\n", __func__); - return (-1); - } - - if_initname(ifp, device_get_name(dev), device_get_unit(dev)); - - ifp->if_mtu = ETHERMTU; - ifp->if_baudrate = 4000000000; // ?? - ifp->if_init = ixlv_init; - ifp->if_softc = vsi; - ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; - ifp->if_ioctl = ixlv_ioctl; - -#if __FreeBSD_version >= 1100000 - if_setgetcounterfn(ifp, ixl_get_counter); -#endif - - ifp->if_transmit = ixl_mq_start; - - ifp->if_qflush = ixl_qflush; - ifp->if_snd.ifq_maxlen = que->num_desc - 2; - - ether_ifattach(ifp, sc->hw.mac.addr); - - vsi->max_frame_size = - ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN - + ETHER_VLAN_ENCAP_LEN; - - /* - * Tell the upper layer(s) we support long frames. - */ - ifp->if_hdrlen = sizeof(struct ether_vlan_header); - - ifp->if_capabilities |= IFCAP_HWCSUM; - ifp->if_capabilities |= IFCAP_HWCSUM_IPV6; - ifp->if_capabilities |= IFCAP_TSO; - ifp->if_capabilities |= IFCAP_JUMBO_MTU; - - ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING - | IFCAP_VLAN_HWTSO - | IFCAP_VLAN_MTU - | IFCAP_VLAN_HWCSUM - | IFCAP_LRO; - ifp->if_capenable = ifp->if_capabilities; - - /* - ** Don't turn this on by default, if vlans are - ** created on another pseudo device (eg. lagg) - ** then vlan events are not passed thru, breaking - ** operation, but with HW FILTER off it works. If - ** using vlans directly on the ixl driver you can - ** enable this and get full hardware tag filtering. - */ - ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; - - /* - * Specify the media types supported by this adapter and register - * callbacks to update media and link information - */ - ifmedia_init(&sc->media, IFM_IMASK, ixlv_media_change, - ixlv_media_status); - - // JFV Add media types later? - - ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); - ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); - - INIT_DBG_DEV(dev, "end"); - return (0); -} - -/* -** Allocate and setup the interface queues -*/ -static int -ixlv_setup_queues(struct ixlv_sc *sc) -{ - device_t dev = sc->dev; - struct ixl_vsi *vsi; - struct ixl_queue *que; - struct tx_ring *txr; - struct rx_ring *rxr; - int rsize, tsize; - int error = I40E_SUCCESS; - - vsi = &sc->vsi; - vsi->back = (void *)sc; - vsi->hw = &sc->hw; - vsi->num_vlans = 0; - - /* Get memory for the station queues */ - if (!(vsi->queues = - (struct ixl_queue *) malloc(sizeof(struct ixl_queue) * - vsi->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { - device_printf(dev, "Unable to allocate queue memory\n"); - error = ENOMEM; - goto early; - } - - for (int i = 0; i < vsi->num_queues; i++) { - que = &vsi->queues[i]; - que->num_desc = ixlv_ringsz; - que->me = i; - que->vsi = vsi; - /* mark the queue as active */ - vsi->active_queues |= (u64)1 << que->me; - - txr = &que->txr; - txr->que = que; - txr->tail = I40E_QTX_TAIL1(que->me); - /* Initialize the TX lock */ - snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", - device_get_nameunit(dev), que->me); - mtx_init(&txr->mtx, txr->mtx_name, NULL, MTX_DEF); - /* - ** Create the TX descriptor ring, the extra int is - ** added as the location for HEAD WB. - */ - tsize = roundup2((que->num_desc * - sizeof(struct i40e_tx_desc)) + - sizeof(u32), DBA_ALIGN); - if (i40e_allocate_dma_mem(&sc->hw, - &txr->dma, i40e_mem_reserved, tsize, DBA_ALIGN)) { - device_printf(dev, - "Unable to allocate TX Descriptor memory\n"); - error = ENOMEM; - goto fail; - } - txr->base = (struct i40e_tx_desc *)txr->dma.va; - bzero((void *)txr->base, tsize); - /* Now allocate transmit soft structs for the ring */ - if (ixl_allocate_tx_data(que)) { - device_printf(dev, - "Critical Failure setting up TX structures\n"); - error = ENOMEM; - goto fail; - } - /* Allocate a buf ring */ - txr->br = buf_ring_alloc(ixlv_txbrsz, M_DEVBUF, - M_WAITOK, &txr->mtx); - if (txr->br == NULL) { - device_printf(dev, - "Critical Failure setting up TX buf ring\n"); - error = ENOMEM; - goto fail; - } - - /* - * Next the RX queues... - */ - rsize = roundup2(que->num_desc * - sizeof(union i40e_rx_desc), DBA_ALIGN); - rxr = &que->rxr; - rxr->que = que; - rxr->tail = I40E_QRX_TAIL1(que->me); - - /* Initialize the RX side lock */ - snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", - device_get_nameunit(dev), que->me); - mtx_init(&rxr->mtx, rxr->mtx_name, NULL, MTX_DEF); - - if (i40e_allocate_dma_mem(&sc->hw, - &rxr->dma, i40e_mem_reserved, rsize, 4096)) { //JFV - should this be DBA? - device_printf(dev, - "Unable to allocate RX Descriptor memory\n"); - error = ENOMEM; - goto fail; - } - rxr->base = (union i40e_rx_desc *)rxr->dma.va; - bzero((void *)rxr->base, rsize); - - /* Allocate receive soft structs for the ring*/ - if (ixl_allocate_rx_data(que)) { - device_printf(dev, - "Critical Failure setting up receive structs\n"); - error = ENOMEM; - goto fail; - } - } - - return (0); - -fail: - for (int i = 0; i < vsi->num_queues; i++) { - que = &vsi->queues[i]; - rxr = &que->rxr; - txr = &que->txr; - if (rxr->base) - i40e_free_dma_mem(&sc->hw, &rxr->dma); - if (txr->base) - i40e_free_dma_mem(&sc->hw, &txr->dma); - } - free(vsi->queues, M_DEVBUF); - -early: - return (error); -} - -/* -** This routine is run via an vlan config EVENT, -** it enables us to use the HW Filter table since -** we can get the vlan id. This just creates the -** entry in the soft version of the VFTA, init will -** repopulate the real table. -*/ -static void -ixlv_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) -{ - struct ixl_vsi *vsi = arg; - struct ixlv_sc *sc = vsi->back; - struct ixlv_vlan_filter *v; - - - if (ifp->if_softc != arg) /* Not our event */ - return; - - if ((vtag == 0) || (vtag > 4095)) /* Invalid */ - return; - - /* Sanity check - make sure it doesn't already exist */ - SLIST_FOREACH(v, sc->vlan_filters, next) { - if (v->vlan == vtag) - return; - } - - mtx_lock(&sc->mtx); - ++vsi->num_vlans; - v = malloc(sizeof(struct ixlv_vlan_filter), M_DEVBUF, M_NOWAIT | M_ZERO); - SLIST_INSERT_HEAD(sc->vlan_filters, v, next); - v->vlan = vtag; - v->flags = IXL_FILTER_ADD; - ixl_vc_enqueue(&sc->vc_mgr, &sc->add_vlan_cmd, - IXLV_FLAG_AQ_ADD_VLAN_FILTER, ixl_init_cmd_complete, sc); - mtx_unlock(&sc->mtx); - return; -} - -/* -** This routine is run via an vlan -** unconfig EVENT, remove our entry -** in the soft vfta. -*/ -static void -ixlv_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) -{ - struct ixl_vsi *vsi = arg; - struct ixlv_sc *sc = vsi->back; - struct ixlv_vlan_filter *v; - int i = 0; - - if (ifp->if_softc != arg) - return; - - if ((vtag == 0) || (vtag > 4095)) /* Invalid */ - return; - - mtx_lock(&sc->mtx); - SLIST_FOREACH(v, sc->vlan_filters, next) { - if (v->vlan == vtag) { - v->flags = IXL_FILTER_DEL; - ++i; - --vsi->num_vlans; - } - } - if (i) - ixl_vc_enqueue(&sc->vc_mgr, &sc->del_vlan_cmd, - IXLV_FLAG_AQ_DEL_VLAN_FILTER, ixl_init_cmd_complete, sc); - mtx_unlock(&sc->mtx); - return; -} - -/* -** Get a new filter and add it to the mac filter list. -*/ -static struct ixlv_mac_filter * -ixlv_get_mac_filter(struct ixlv_sc *sc) -{ - struct ixlv_mac_filter *f; - - f = malloc(sizeof(struct ixlv_mac_filter), - M_DEVBUF, M_NOWAIT | M_ZERO); - if (f) - SLIST_INSERT_HEAD(sc->mac_filters, f, next); - - return (f); -} - -/* -** Find the filter with matching MAC address -*/ -static struct ixlv_mac_filter * -ixlv_find_mac_filter(struct ixlv_sc *sc, u8 *macaddr) -{ - struct ixlv_mac_filter *f; - bool match = FALSE; - - SLIST_FOREACH(f, sc->mac_filters, next) { - if (cmp_etheraddr(f->macaddr, macaddr)) { - match = TRUE; - break; - } - } - - if (!match) - f = NULL; - return (f); -} - -/* -** Admin Queue interrupt handler -*/ -static void -ixlv_msix_adminq(void *arg) -{ - struct ixlv_sc *sc = arg; - struct i40e_hw *hw = &sc->hw; - u32 reg, mask; - - reg = rd32(hw, I40E_VFINT_ICR01); - mask = rd32(hw, I40E_VFINT_ICR0_ENA1); - - reg = rd32(hw, I40E_VFINT_DYN_CTL01); - reg |= I40E_VFINT_DYN_CTL01_CLEARPBA_MASK; - wr32(hw, I40E_VFINT_DYN_CTL01, reg); - - /* schedule task */ - taskqueue_enqueue(sc->tq, &sc->aq_irq); - return; -} - -void -ixlv_enable_intr(struct ixl_vsi *vsi) -{ - struct i40e_hw *hw = vsi->hw; - struct ixl_queue *que = vsi->queues; - - ixlv_enable_adminq_irq(hw); - for (int i = 0; i < vsi->num_queues; i++, que++) - ixlv_enable_queue_irq(hw, que->me); -} - -void -ixlv_disable_intr(struct ixl_vsi *vsi) -{ - struct i40e_hw *hw = vsi->hw; - struct ixl_queue *que = vsi->queues; - - ixlv_disable_adminq_irq(hw); - for (int i = 0; i < vsi->num_queues; i++, que++) - ixlv_disable_queue_irq(hw, que->me); -} - - -static void -ixlv_disable_adminq_irq(struct i40e_hw *hw) -{ - wr32(hw, I40E_VFINT_DYN_CTL01, 0); - wr32(hw, I40E_VFINT_ICR0_ENA1, 0); - /* flush */ - rd32(hw, I40E_VFGEN_RSTAT); - return; -} - -static void -ixlv_enable_adminq_irq(struct i40e_hw *hw) -{ - wr32(hw, I40E_VFINT_DYN_CTL01, - I40E_VFINT_DYN_CTL01_INTENA_MASK | - I40E_VFINT_DYN_CTL01_ITR_INDX_MASK); - wr32(hw, I40E_VFINT_ICR0_ENA1, I40E_VFINT_ICR0_ENA1_ADMINQ_MASK); - /* flush */ - rd32(hw, I40E_VFGEN_RSTAT); - return; -} - -static void -ixlv_enable_queue_irq(struct i40e_hw *hw, int id) -{ - u32 reg; - - reg = I40E_VFINT_DYN_CTLN1_INTENA_MASK | - I40E_VFINT_DYN_CTLN1_CLEARPBA_MASK; - wr32(hw, I40E_VFINT_DYN_CTLN1(id), reg); -} - -static void -ixlv_disable_queue_irq(struct i40e_hw *hw, int id) -{ - wr32(hw, I40E_VFINT_DYN_CTLN1(id), 0); - rd32(hw, I40E_VFGEN_RSTAT); - return; -} - - -/* -** Provide a update to the queue RX -** interrupt moderation value. -*/ -static void -ixlv_set_queue_rx_itr(struct ixl_queue *que) -{ - struct ixl_vsi *vsi = que->vsi; - struct i40e_hw *hw = vsi->hw; - struct rx_ring *rxr = &que->rxr; - u16 rx_itr; - u16 rx_latency = 0; - int rx_bytes; - - - /* Idle, do nothing */ - if (rxr->bytes == 0) - return; - - if (ixlv_dynamic_rx_itr) { - rx_bytes = rxr->bytes/rxr->itr; - rx_itr = rxr->itr; - - /* Adjust latency range */ - switch (rxr->latency) { - case IXL_LOW_LATENCY: - if (rx_bytes > 10) { - rx_latency = IXL_AVE_LATENCY; - rx_itr = IXL_ITR_20K; - } - break; - case IXL_AVE_LATENCY: - if (rx_bytes > 20) { - rx_latency = IXL_BULK_LATENCY; - rx_itr = IXL_ITR_8K; - } else if (rx_bytes <= 10) { - rx_latency = IXL_LOW_LATENCY; - rx_itr = IXL_ITR_100K; - } - break; - case IXL_BULK_LATENCY: - if (rx_bytes <= 20) { - rx_latency = IXL_AVE_LATENCY; - rx_itr = IXL_ITR_20K; - } - break; - } - - rxr->latency = rx_latency; - - if (rx_itr != rxr->itr) { - /* do an exponential smoothing */ - rx_itr = (10 * rx_itr * rxr->itr) / - ((9 * rx_itr) + rxr->itr); - rxr->itr = rx_itr & IXL_MAX_ITR; - wr32(hw, I40E_VFINT_ITRN1(IXL_RX_ITR, - que->me), rxr->itr); - } - } else { /* We may have have toggled to non-dynamic */ - if (vsi->rx_itr_setting & IXL_ITR_DYNAMIC) - vsi->rx_itr_setting = ixlv_rx_itr; - /* Update the hardware if needed */ - if (rxr->itr != vsi->rx_itr_setting) { - rxr->itr = vsi->rx_itr_setting; - wr32(hw, I40E_VFINT_ITRN1(IXL_RX_ITR, - que->me), rxr->itr); - } - } - rxr->bytes = 0; - rxr->packets = 0; - return; -} - - -/* -** Provide a update to the queue TX -** interrupt moderation value. -*/ -static void -ixlv_set_queue_tx_itr(struct ixl_queue *que) -{ - struct ixl_vsi *vsi = que->vsi; - struct i40e_hw *hw = vsi->hw; - struct tx_ring *txr = &que->txr; - u16 tx_itr; - u16 tx_latency = 0; - int tx_bytes; - - - /* Idle, do nothing */ - if (txr->bytes == 0) - return; - - if (ixlv_dynamic_tx_itr) { - tx_bytes = txr->bytes/txr->itr; - tx_itr = txr->itr; - - switch (txr->latency) { - case IXL_LOW_LATENCY: - if (tx_bytes > 10) { - tx_latency = IXL_AVE_LATENCY; - tx_itr = IXL_ITR_20K; - } - break; - case IXL_AVE_LATENCY: - if (tx_bytes > 20) { - tx_latency = IXL_BULK_LATENCY; - tx_itr = IXL_ITR_8K; - } else if (tx_bytes <= 10) { - tx_latency = IXL_LOW_LATENCY; - tx_itr = IXL_ITR_100K; - } - break; - case IXL_BULK_LATENCY: - if (tx_bytes <= 20) { - tx_latency = IXL_AVE_LATENCY; - tx_itr = IXL_ITR_20K; - } - break; - } - - txr->latency = tx_latency; - - if (tx_itr != txr->itr) { - /* do an exponential smoothing */ - tx_itr = (10 * tx_itr * txr->itr) / - ((9 * tx_itr) + txr->itr); - txr->itr = tx_itr & IXL_MAX_ITR; - wr32(hw, I40E_VFINT_ITRN1(IXL_TX_ITR, - que->me), txr->itr); - } - - } else { /* We may have have toggled to non-dynamic */ - if (vsi->tx_itr_setting & IXL_ITR_DYNAMIC) - vsi->tx_itr_setting = ixlv_tx_itr; - /* Update the hardware if needed */ - if (txr->itr != vsi->tx_itr_setting) { - txr->itr = vsi->tx_itr_setting; - wr32(hw, I40E_VFINT_ITRN1(IXL_TX_ITR, - que->me), txr->itr); - } - } - txr->bytes = 0; - txr->packets = 0; - return; -} - - -/* -** -** MSIX Interrupt Handlers and Tasklets -** -*/ -static void -ixlv_handle_que(void *context, int pending) -{ - struct ixl_queue *que = context; - struct ixl_vsi *vsi = que->vsi; - struct i40e_hw *hw = vsi->hw; - struct tx_ring *txr = &que->txr; - struct ifnet *ifp = vsi->ifp; - bool more; - - if (ifp->if_drv_flags & IFF_DRV_RUNNING) { - more = ixl_rxeof(que, IXL_RX_LIMIT); - mtx_lock(&txr->mtx); - ixl_txeof(que); - if (!drbr_empty(ifp, txr->br)) - ixl_mq_start_locked(ifp, txr); - mtx_unlock(&txr->mtx); - if (more) { - taskqueue_enqueue(que->tq, &que->task); - return; - } - } - - /* Reenable this interrupt - hmmm */ - ixlv_enable_queue_irq(hw, que->me); - return; -} - - -/********************************************************************* - * - * MSIX Queue Interrupt Service routine - * - **********************************************************************/ -static void -ixlv_msix_que(void *arg) -{ - struct ixl_queue *que = arg; - struct ixl_vsi *vsi = que->vsi; - struct i40e_hw *hw = vsi->hw; - struct tx_ring *txr = &que->txr; - bool more_tx, more_rx; - - /* Spurious interrupts are ignored */ - if (!(vsi->ifp->if_drv_flags & IFF_DRV_RUNNING)) - return; - - ++que->irqs; - - more_rx = ixl_rxeof(que, IXL_RX_LIMIT); - - mtx_lock(&txr->mtx); - more_tx = ixl_txeof(que); - /* - ** Make certain that if the stack - ** has anything queued the task gets - ** scheduled to handle it. - */ - if (!drbr_empty(vsi->ifp, txr->br)) - more_tx = 1; - mtx_unlock(&txr->mtx); - - ixlv_set_queue_rx_itr(que); - ixlv_set_queue_tx_itr(que); - - if (more_tx || more_rx) - taskqueue_enqueue(que->tq, &que->task); - else - ixlv_enable_queue_irq(hw, que->me); - - return; -} - - -/********************************************************************* - * - * Media Ioctl callback - * - * This routine is called whenever the user queries the status of - * the interface using ifconfig. - * - **********************************************************************/ -static void -ixlv_media_status(struct ifnet * ifp, struct ifmediareq * ifmr) -{ - struct ixl_vsi *vsi = ifp->if_softc; - struct ixlv_sc *sc = vsi->back; - - INIT_DBG_IF(ifp, "begin"); - - mtx_lock(&sc->mtx); - - ixlv_update_link_status(sc); - - ifmr->ifm_status = IFM_AVALID; - ifmr->ifm_active = IFM_ETHER; - - if (!sc->link_up) { - mtx_unlock(&sc->mtx); - INIT_DBG_IF(ifp, "end: link not up"); - return; - } - - ifmr->ifm_status |= IFM_ACTIVE; - /* Hardware is always full-duplex */ - ifmr->ifm_active |= IFM_FDX; - mtx_unlock(&sc->mtx); - INIT_DBG_IF(ifp, "end"); - return; -} - -/********************************************************************* - * - * Media Ioctl callback - * - * This routine is called when the user changes speed/duplex using - * media/mediopt option with ifconfig. - * - **********************************************************************/ -static int -ixlv_media_change(struct ifnet * ifp) -{ - struct ixl_vsi *vsi = ifp->if_softc; - struct ifmedia *ifm = &vsi->media; - - INIT_DBG_IF(ifp, "begin"); - - if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) - return (EINVAL); - - INIT_DBG_IF(ifp, "end"); - return (0); -} - - -/********************************************************************* - * Multicast Initialization - * - * This routine is called by init to reset a fresh state. - * - **********************************************************************/ - -static void -ixlv_init_multi(struct ixl_vsi *vsi) -{ - struct ixlv_mac_filter *f; - struct ixlv_sc *sc = vsi->back; - int mcnt = 0; - - IOCTL_DBG_IF(vsi->ifp, "begin"); - - /* First clear any multicast filters */ - SLIST_FOREACH(f, sc->mac_filters, next) { - if ((f->flags & IXL_FILTER_USED) - && (f->flags & IXL_FILTER_MC)) { - f->flags |= IXL_FILTER_DEL; - mcnt++; - } - } - if (mcnt > 0) - ixl_vc_enqueue(&sc->vc_mgr, &sc->del_multi_cmd, - IXLV_FLAG_AQ_DEL_MAC_FILTER, ixl_init_cmd_complete, - sc); - - IOCTL_DBG_IF(vsi->ifp, "end"); -} - -static void -ixlv_add_multi(struct ixl_vsi *vsi) -{ - struct ifmultiaddr *ifma; - struct ifnet *ifp = vsi->ifp; - struct ixlv_sc *sc = vsi->back; - int mcnt = 0; - - IOCTL_DBG_IF(ifp, "begin"); - - if_maddr_rlock(ifp); - /* - ** Get a count, to decide if we - ** simply use multicast promiscuous. - */ - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { - if (ifma->ifma_addr->sa_family != AF_LINK) - continue; - mcnt++; - } - if_maddr_runlock(ifp); - - // TODO: Remove -- cannot set promiscuous mode in a VF - if (__predict_false(mcnt >= MAX_MULTICAST_ADDR)) { - /* delete all multicast filters */ - ixlv_init_multi(vsi); - sc->promiscuous_flags |= I40E_FLAG_VF_MULTICAST_PROMISC; - ixl_vc_enqueue(&sc->vc_mgr, &sc->add_multi_cmd, - IXLV_FLAG_AQ_CONFIGURE_PROMISC, ixl_init_cmd_complete, - sc); - IOCTL_DEBUGOUT("%s: end: too many filters", __func__); - return; - } - - mcnt = 0; - if_maddr_rlock(ifp); - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { - if (ifma->ifma_addr->sa_family != AF_LINK) - continue; - if (!ixlv_add_mac_filter(sc, - (u8*)LLADDR((struct sockaddr_dl *) ifma->ifma_addr), - IXL_FILTER_MC)) - mcnt++; - } - if_maddr_runlock(ifp); - /* - ** Notify AQ task that sw filters need to be - ** added to hw list - */ - if (mcnt > 0) - ixl_vc_enqueue(&sc->vc_mgr, &sc->add_multi_cmd, - IXLV_FLAG_AQ_ADD_MAC_FILTER, ixl_init_cmd_complete, - sc); - - IOCTL_DBG_IF(ifp, "end"); -} - -static void -ixlv_del_multi(struct ixl_vsi *vsi) -{ - struct ixlv_mac_filter *f; - struct ifmultiaddr *ifma; - struct ifnet *ifp = vsi->ifp; - struct ixlv_sc *sc = vsi->back; - int mcnt = 0; - bool match = FALSE; - - IOCTL_DBG_IF(ifp, "begin"); - - /* Search for removed multicast addresses */ - if_maddr_rlock(ifp); - SLIST_FOREACH(f, sc->mac_filters, next) { - if ((f->flags & IXL_FILTER_USED) - && (f->flags & IXL_FILTER_MC)) { - /* check if mac address in filter is in sc's list */ - match = FALSE; - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { - if (ifma->ifma_addr->sa_family != AF_LINK) - continue; - u8 *mc_addr = - (u8 *)LLADDR((struct sockaddr_dl *)ifma->ifma_addr); - if (cmp_etheraddr(f->macaddr, mc_addr)) { - match = TRUE; - break; - } - } - /* if this filter is not in the sc's list, remove it */ - if (match == FALSE && !(f->flags & IXL_FILTER_DEL)) { - f->flags |= IXL_FILTER_DEL; - mcnt++; - IOCTL_DBG_IF(ifp, "marked: " MAC_FORMAT, - MAC_FORMAT_ARGS(f->macaddr)); - } - else if (match == FALSE) - IOCTL_DBG_IF(ifp, "exists: " MAC_FORMAT, - MAC_FORMAT_ARGS(f->macaddr)); - } - } - if_maddr_runlock(ifp); - - if (mcnt > 0) - ixl_vc_enqueue(&sc->vc_mgr, &sc->del_multi_cmd, - IXLV_FLAG_AQ_DEL_MAC_FILTER, ixl_init_cmd_complete, - sc); - - IOCTL_DBG_IF(ifp, "end"); -} - -/********************************************************************* - * Timer routine - * - * This routine checks for link status,updates statistics, - * and runs the watchdog check. - * - **********************************************************************/ - -static void -ixlv_local_timer(void *arg) -{ - struct ixlv_sc *sc = arg; - struct i40e_hw *hw = &sc->hw; - struct ixl_vsi *vsi = &sc->vsi; - struct ixl_queue *que = vsi->queues; - device_t dev = sc->dev; - int hung = 0; - u32 mask, val; - - IXLV_CORE_LOCK_ASSERT(sc); - - /* If Reset is in progress just bail */ - if (sc->init_state == IXLV_RESET_PENDING) - return; - - /* Check for when PF triggers a VF reset */ - val = rd32(hw, I40E_VFGEN_RSTAT) & - I40E_VFGEN_RSTAT_VFR_STATE_MASK; - - if (val != I40E_VFR_VFACTIVE - && val != I40E_VFR_COMPLETED) { - DDPRINTF(dev, "reset in progress! (%d)", val); - return; - } - - ixlv_request_stats(sc); - - /* clean and process any events */ - taskqueue_enqueue(sc->tq, &sc->aq_irq); - - /* - ** Check status on the queues for a hang - */ - mask = (I40E_VFINT_DYN_CTLN1_INTENA_MASK | - I40E_VFINT_DYN_CTLN1_SWINT_TRIG_MASK); - - for (int i = 0; i < vsi->num_queues; i++,que++) { - /* Any queues with outstanding work get a sw irq */ - if (que->busy) - wr32(hw, I40E_VFINT_DYN_CTLN1(que->me), mask); - /* - ** Each time txeof runs without cleaning, but there - ** are uncleaned descriptors it increments busy. If - ** we get to 5 we declare it hung. - */ - if (que->busy == IXL_QUEUE_HUNG) { - ++hung; - /* Mark the queue as inactive */ - vsi->active_queues &= ~((u64)1 << que->me); - continue; - } else { - /* Check if we've come back from hung */ - if ((vsi->active_queues & ((u64)1 << que->me)) == 0) - vsi->active_queues |= ((u64)1 << que->me); - } - if (que->busy >= IXL_MAX_TX_BUSY) { - device_printf(dev,"Warning queue %d " - "appears to be hung!\n", i); - que->busy = IXL_QUEUE_HUNG; - ++hung; - } - } - /* Only reset when all queues show hung */ - if (hung == vsi->num_queues) - goto hung; - callout_reset(&sc->timer, hz, ixlv_local_timer, sc); - return; - -hung: - device_printf(dev, "Local Timer: TX HANG DETECTED - Resetting!!\n"); - sc->init_state = IXLV_RESET_REQUIRED; - ixlv_init_locked(sc); -} - -/* -** Note: this routine updates the OS on the link state -** the real check of the hardware only happens with -** a link interrupt. -*/ -void -ixlv_update_link_status(struct ixlv_sc *sc) -{ - struct ixl_vsi *vsi = &sc->vsi; - struct ifnet *ifp = vsi->ifp; - - if (sc->link_up){ - if (vsi->link_active == FALSE) { - if (bootverbose) - if_printf(ifp,"Link is Up, %d Gbps\n", - (sc->link_speed == I40E_LINK_SPEED_40GB) ? 40:10); - vsi->link_active = TRUE; - if_link_state_change(ifp, LINK_STATE_UP); - } - } else { /* Link down */ - if (vsi->link_active == TRUE) { - if (bootverbose) - if_printf(ifp,"Link is Down\n"); - if_link_state_change(ifp, LINK_STATE_DOWN); - vsi->link_active = FALSE; - } - } - - return; -} - -/********************************************************************* - * - * This routine disables all traffic on the adapter by issuing a - * global reset on the MAC and deallocates TX/RX buffers. - * - **********************************************************************/ - -static void -ixlv_stop(struct ixlv_sc *sc) -{ - struct ifnet *ifp; - int start; - - ifp = sc->vsi.ifp; - INIT_DBG_IF(ifp, "begin"); - - IXLV_CORE_LOCK_ASSERT(sc); - - ixl_vc_flush(&sc->vc_mgr); - ixlv_disable_queues(sc); - - start = ticks; - while ((ifp->if_drv_flags & IFF_DRV_RUNNING) && - ((ticks - start) < hz/10)) - ixlv_do_adminq_locked(sc); - - /* Stop the local timer */ - callout_stop(&sc->timer); - - INIT_DBG_IF(ifp, "end"); -} - - -/********************************************************************* - * - * Free all station queue structs. - * - **********************************************************************/ -static void -ixlv_free_queues(struct ixl_vsi *vsi) -{ - struct ixlv_sc *sc = (struct ixlv_sc *)vsi->back; - struct ixl_queue *que = vsi->queues; - - for (int i = 0; i < vsi->num_queues; i++, que++) { - struct tx_ring *txr = &que->txr; - struct rx_ring *rxr = &que->rxr; - - if (!mtx_initialized(&txr->mtx)) /* uninitialized */ - continue; - IXL_TX_LOCK(txr); - ixl_free_que_tx(que); - if (txr->base) - i40e_free_dma_mem(&sc->hw, &txr->dma); - IXL_TX_UNLOCK(txr); - IXL_TX_LOCK_DESTROY(txr); - - if (!mtx_initialized(&rxr->mtx)) /* uninitialized */ - continue; - IXL_RX_LOCK(rxr); - ixl_free_que_rx(que); - if (rxr->base) - i40e_free_dma_mem(&sc->hw, &rxr->dma); - IXL_RX_UNLOCK(rxr); - IXL_RX_LOCK_DESTROY(rxr); - - } - free(vsi->queues, M_DEVBUF); -} - - -/* -** ixlv_config_rss - setup RSS -** -** RSS keys and table are cleared on VF reset. -*/ -static void -ixlv_config_rss(struct ixlv_sc *sc) -{ - struct i40e_hw *hw = &sc->hw; - struct ixl_vsi *vsi = &sc->vsi; - u32 lut = 0; - u64 set_hena = 0, hena; - int i, j, que_id; -#ifdef RSS - u32 rss_hash_config; - u32 rss_seed[IXL_KEYSZ]; -#else - u32 rss_seed[IXL_KEYSZ] = {0x41b01687, - 0x183cfd8c, 0xce880440, 0x580cbc3c, - 0x35897377, 0x328b25e1, 0x4fa98922, - 0xb7d90c14, 0xd5bad70d, 0xcd15a2c1}; -#endif - - /* Don't set up RSS if using a single queue */ - if (vsi->num_queues == 1) { - wr32(hw, I40E_VFQF_HENA(0), 0); - wr32(hw, I40E_VFQF_HENA(1), 0); - ixl_flush(hw); - return; - } - -#ifdef RSS - /* Fetch the configured RSS key */ - rss_getkey((uint8_t *) &rss_seed); -#endif - /* Fill out hash function seed */ - for (i = 0; i <= IXL_KEYSZ; i++) - wr32(hw, I40E_VFQF_HKEY(i), rss_seed[i]); - - /* Enable PCTYPES for RSS: */ -#ifdef RSS - rss_hash_config = rss_gethashconfig(); - if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4) - set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER); - if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4) - set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP); - if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4) - set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP); - if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6) - set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER); - if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX) - set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6); - if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6) - set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP); - if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6) - set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP); -#else - set_hena = - ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP) | - ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP) | - ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) | - ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) | - ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV4) | - ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP) | - ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP) | - ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) | - ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) | - ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6) | - ((u64)1 << I40E_FILTER_PCTYPE_L2_PAYLOAD); -#endif - hena = (u64)rd32(hw, I40E_VFQF_HENA(0)) | - ((u64)rd32(hw, I40E_VFQF_HENA(1)) << 32); - hena |= set_hena; - wr32(hw, I40E_VFQF_HENA(0), (u32)hena); - wr32(hw, I40E_VFQF_HENA(1), (u32)(hena >> 32)); - - /* Populate the LUT with max no. of queues in round robin fashion */ - for (i = 0, j = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++, j++) { - if (j == vsi->num_queues) - j = 0; -#ifdef RSS - /* - * Fetch the RSS bucket id for the given indirection entry. - * Cap it at the number of configured buckets (which is - * num_queues.) - */ - que_id = rss_get_indirection_to_bucket(i); - que_id = que_id % vsi->num_queues; +#include "opt_iflib.h" +#ifdef IFLIB +#include "iflib_if_ixlv.c" #else - que_id = j; +#include "legacy_if_ixlv.c" #endif - /* lut = 4-byte sliding window of 4 lut entries */ - lut = (lut << 8) | (que_id & 0xF); - /* On i = 3, we have 4 entries in lut; write to the register */ - if ((i & 3) == 3) { - wr32(hw, I40E_VFQF_HLUT(i), lut); - DDPRINTF(sc->dev, "HLUT(%2d): %#010x", i, lut); - } - } - ixl_flush(hw); -} - - -/* -** This routine refreshes vlan filters, called by init -** it scans the filter table and then updates the AQ -*/ -static void -ixlv_setup_vlan_filters(struct ixlv_sc *sc) -{ - struct ixl_vsi *vsi = &sc->vsi; - struct ixlv_vlan_filter *f; - int cnt = 0; - - if (vsi->num_vlans == 0) - return; - /* - ** Scan the filter table for vlan entries, - ** and if found call for the AQ update. - */ - SLIST_FOREACH(f, sc->vlan_filters, next) - if (f->flags & IXL_FILTER_ADD) - cnt++; - if (cnt > 0) - ixl_vc_enqueue(&sc->vc_mgr, &sc->add_vlan_cmd, - IXLV_FLAG_AQ_ADD_VLAN_FILTER, ixl_init_cmd_complete, sc); -} - - -/* -** This routine adds new MAC filters to the sc's list; -** these are later added in hardware by sending a virtual -** channel message. -*/ -static int -ixlv_add_mac_filter(struct ixlv_sc *sc, u8 *macaddr, u16 flags) -{ - struct ixlv_mac_filter *f; - - /* Does one already exist? */ - f = ixlv_find_mac_filter(sc, macaddr); - if (f != NULL) { - IDPRINTF(sc->vsi.ifp, "exists: " MAC_FORMAT, - MAC_FORMAT_ARGS(macaddr)); - return (EEXIST); - } - - /* If not, get a new empty filter */ - f = ixlv_get_mac_filter(sc); - if (f == NULL) { - if_printf(sc->vsi.ifp, "%s: no filters available!!\n", - __func__); - return (ENOMEM); - } - - IDPRINTF(sc->vsi.ifp, "marked: " MAC_FORMAT, - MAC_FORMAT_ARGS(macaddr)); - - bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN); - f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED); - f->flags |= flags; - return (0); -} - -/* -** Marks a MAC filter for deletion. -*/ -static int -ixlv_del_mac_filter(struct ixlv_sc *sc, u8 *macaddr) -{ - struct ixlv_mac_filter *f; - - f = ixlv_find_mac_filter(sc, macaddr); - if (f == NULL) - return (ENOENT); - - f->flags |= IXL_FILTER_DEL; - return (0); -} - -/* -** Tasklet handler for MSIX Adminq interrupts -** - done outside interrupt context since it might sleep -*/ -static void -ixlv_do_adminq(void *context, int pending) -{ - struct ixlv_sc *sc = context; - - mtx_lock(&sc->mtx); - ixlv_do_adminq_locked(sc); - mtx_unlock(&sc->mtx); - return; -} - -static void -ixlv_do_adminq_locked(struct ixlv_sc *sc) -{ - struct i40e_hw *hw = &sc->hw; - struct i40e_arq_event_info event; - struct i40e_virtchnl_msg *v_msg; - device_t dev = sc->dev; - u16 result = 0; - u32 reg, oldreg; - i40e_status ret; - - IXLV_CORE_LOCK_ASSERT(sc); - - event.buf_len = IXL_AQ_BUF_SZ; - event.msg_buf = sc->aq_buffer; - v_msg = (struct i40e_virtchnl_msg *)&event.desc; - - do { - ret = i40e_clean_arq_element(hw, &event, &result); - if (ret) - break; - ixlv_vc_completion(sc, v_msg->v_opcode, - v_msg->v_retval, event.msg_buf, event.msg_len); - if (result != 0) - bzero(event.msg_buf, IXL_AQ_BUF_SZ); - } while (result); - - /* check for Admin queue errors */ - oldreg = reg = rd32(hw, hw->aq.arq.len); - if (reg & I40E_VF_ARQLEN1_ARQVFE_MASK) { - device_printf(dev, "ARQ VF Error detected\n"); - reg &= ~I40E_VF_ARQLEN1_ARQVFE_MASK; - } - if (reg & I40E_VF_ARQLEN1_ARQOVFL_MASK) { - device_printf(dev, "ARQ Overflow Error detected\n"); - reg &= ~I40E_VF_ARQLEN1_ARQOVFL_MASK; - } - if (reg & I40E_VF_ARQLEN1_ARQCRIT_MASK) { - device_printf(dev, "ARQ Critical Error detected\n"); - reg &= ~I40E_VF_ARQLEN1_ARQCRIT_MASK; - } - if (oldreg != reg) - wr32(hw, hw->aq.arq.len, reg); - - oldreg = reg = rd32(hw, hw->aq.asq.len); - if (reg & I40E_VF_ATQLEN1_ATQVFE_MASK) { - device_printf(dev, "ASQ VF Error detected\n"); - reg &= ~I40E_VF_ATQLEN1_ATQVFE_MASK; - } - if (reg & I40E_VF_ATQLEN1_ATQOVFL_MASK) { - device_printf(dev, "ASQ Overflow Error detected\n"); - reg &= ~I40E_VF_ATQLEN1_ATQOVFL_MASK; - } - if (reg & I40E_VF_ATQLEN1_ATQCRIT_MASK) { - device_printf(dev, "ASQ Critical Error detected\n"); - reg &= ~I40E_VF_ATQLEN1_ATQCRIT_MASK; - } - if (oldreg != reg) - wr32(hw, hw->aq.asq.len, reg); - - ixlv_enable_adminq_irq(hw); -} - -static void -ixlv_add_sysctls(struct ixlv_sc *sc) -{ - device_t dev = sc->dev; - struct ixl_vsi *vsi = &sc->vsi; - struct i40e_eth_stats *es = &vsi->eth_stats; - - struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); - struct sysctl_oid *tree = device_get_sysctl_tree(dev); - struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree); - - struct sysctl_oid *vsi_node, *queue_node; - struct sysctl_oid_list *vsi_list, *queue_list; - -#define QUEUE_NAME_LEN 32 - char queue_namebuf[QUEUE_NAME_LEN]; - - struct ixl_queue *queues = vsi->queues; - struct tx_ring *txr; - struct rx_ring *rxr; - - /* Driver statistics sysctls */ - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events", - CTLFLAG_RD, &sc->watchdog_events, - "Watchdog timeouts"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "admin_irq", - CTLFLAG_RD, &sc->admin_irq, - "Admin Queue IRQ Handled"); - - /* VSI statistics sysctls */ - vsi_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "vsi", - CTLFLAG_RD, NULL, "VSI-specific statistics"); - vsi_list = SYSCTL_CHILDREN(vsi_node); - - struct ixl_sysctl_info ctls[] = - { - {&es->rx_bytes, "good_octets_rcvd", "Good Octets Received"}, - {&es->rx_unicast, "ucast_pkts_rcvd", - "Unicast Packets Received"}, - {&es->rx_multicast, "mcast_pkts_rcvd", - "Multicast Packets Received"}, - {&es->rx_broadcast, "bcast_pkts_rcvd", - "Broadcast Packets Received"}, - {&es->rx_discards, "rx_discards", "Discarded RX packets"}, - {&es->rx_unknown_protocol, "rx_unknown_proto", "RX unknown protocol packets"}, - {&es->tx_bytes, "good_octets_txd", "Good Octets Transmitted"}, - {&es->tx_unicast, "ucast_pkts_txd", "Unicast Packets Transmitted"}, - {&es->tx_multicast, "mcast_pkts_txd", - "Multicast Packets Transmitted"}, - {&es->tx_broadcast, "bcast_pkts_txd", - "Broadcast Packets Transmitted"}, - {&es->tx_errors, "tx_errors", "TX packet errors"}, - // end - {0,0,0} - }; - struct ixl_sysctl_info *entry = ctls; - while (entry->stat != NULL) - { - SYSCTL_ADD_QUAD(ctx, child, OID_AUTO, entry->name, - CTLFLAG_RD, entry->stat, - entry->description); - entry++; - } - - /* Queue sysctls */ - for (int q = 0; q < vsi->num_queues; q++) { - snprintf(queue_namebuf, QUEUE_NAME_LEN, "que%d", q); - queue_node = SYSCTL_ADD_NODE(ctx, vsi_list, OID_AUTO, queue_namebuf, - CTLFLAG_RD, NULL, "Queue Name"); - queue_list = SYSCTL_CHILDREN(queue_node); - - txr = &(queues[q].txr); - rxr = &(queues[q].rxr); - - SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "mbuf_defrag_failed", - CTLFLAG_RD, &(queues[q].mbuf_defrag_failed), - "m_defrag() failed"); - SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "dropped", - CTLFLAG_RD, &(queues[q].dropped_pkts), - "Driver dropped packets"); - SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "irqs", - CTLFLAG_RD, &(queues[q].irqs), - "irqs on this queue"); - SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "tso_tx", - CTLFLAG_RD, &(queues[q].tso), - "TSO"); - SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "tx_dma_setup", - CTLFLAG_RD, &(queues[q].tx_dma_setup), - "Driver tx dma failure in xmit"); - SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "no_desc_avail", - CTLFLAG_RD, &(txr->no_desc), - "Queue No Descriptor Available"); - SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "tx_packets", - CTLFLAG_RD, &(txr->total_packets), - "Queue Packets Transmitted"); - SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "tx_bytes", - CTLFLAG_RD, &(txr->tx_bytes), - "Queue Bytes Transmitted"); - SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "rx_packets", - CTLFLAG_RD, &(rxr->rx_packets), - "Queue Packets Received"); - SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "rx_bytes", - CTLFLAG_RD, &(rxr->rx_bytes), - "Queue Bytes Received"); - - /* Examine queue state */ - SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "qtx_head", - CTLTYPE_UINT | CTLFLAG_RD, &queues[q], - sizeof(struct ixl_queue), - ixlv_sysctl_qtx_tail_handler, "IU", - "Queue Transmit Descriptor Tail"); - SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "qrx_head", - CTLTYPE_UINT | CTLFLAG_RD, &queues[q], - sizeof(struct ixl_queue), - ixlv_sysctl_qrx_tail_handler, "IU", - "Queue Receive Descriptor Tail"); - } -} - -static void -ixlv_init_filters(struct ixlv_sc *sc) -{ - sc->mac_filters = malloc(sizeof(struct ixlv_mac_filter), - M_DEVBUF, M_NOWAIT | M_ZERO); - SLIST_INIT(sc->mac_filters); - sc->vlan_filters = malloc(sizeof(struct ixlv_vlan_filter), - M_DEVBUF, M_NOWAIT | M_ZERO); - SLIST_INIT(sc->vlan_filters); - return; -} - -static void -ixlv_free_filters(struct ixlv_sc *sc) -{ - struct ixlv_mac_filter *f; - struct ixlv_vlan_filter *v; - - while (!SLIST_EMPTY(sc->mac_filters)) { - f = SLIST_FIRST(sc->mac_filters); - SLIST_REMOVE_HEAD(sc->mac_filters, next); - free(f, M_DEVBUF); - } - while (!SLIST_EMPTY(sc->vlan_filters)) { - v = SLIST_FIRST(sc->vlan_filters); - SLIST_REMOVE_HEAD(sc->vlan_filters, next); - free(v, M_DEVBUF); - } - return; -} - -/** - * ixlv_sysctl_qtx_tail_handler - * Retrieves I40E_QTX_TAIL1 value from hardware - * for a sysctl. - */ -static int -ixlv_sysctl_qtx_tail_handler(SYSCTL_HANDLER_ARGS) -{ - struct ixl_queue *que; - int error; - u32 val; - - que = ((struct ixl_queue *)oidp->oid_arg1); - if (!que) return 0; - - val = rd32(que->vsi->hw, que->txr.tail); - error = sysctl_handle_int(oidp, &val, 0, req); - if (error || !req->newptr) - return error; - return (0); -} - -/** - * ixlv_sysctl_qrx_tail_handler - * Retrieves I40E_QRX_TAIL1 value from hardware - * for a sysctl. - */ -static int -ixlv_sysctl_qrx_tail_handler(SYSCTL_HANDLER_ARGS) -{ - struct ixl_queue *que; - int error; - u32 val; - - que = ((struct ixl_queue *)oidp->oid_arg1); - if (!que) return 0; - - val = rd32(que->vsi->hw, que->rxr.tail); - error = sysctl_handle_int(oidp, &val, 0, req); - if (error || !req->newptr) - return error; - return (0); -} - Index: sys/dev/ixl/iflib_if_ixl.c =================================================================== --- /dev/null +++ sys/dev/ixl/iflib_if_ixl.c @@ -0,0 +1,5731 @@ +/****************************************************************************** + + Copyright (c) 2013-2015, Intel Corporation + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of the Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +******************************************************************************/ +/*$FreeBSD$*/ + +#ifndef IXL_STANDALONE_BUILD +#include "opt_inet.h" +#include "opt_inet6.h" +#include "opt_rss.h" +#endif + +#ifndef IXL_DEBUG_SYSCTL +#ifdef INVARIANTS +#define IXL_DEBUG_SYSCTL +#endif +#endif + +#include "ixl.h" +#include "ixl_pf.h" + +#ifdef RSS +#include +#endif + +#include "ifdi_if.h" + +/********************************************************************* + * Driver version + *********************************************************************/ +char ixl_driver_version[] = "1.4.3"; + +/********************************************************************* + * PCI Device ID Table + * + * Used by probe to select devices to load on + * Last field stores an index into ixl_strings + * Last entry must be PVID_END + * + * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } + *********************************************************************/ + +static pci_vendor_info_t ixl_vendor_info_array[] = +{ + PVID(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_XL710, "Intel(R) Ethernet Connection XL710 Driver"), + PVID(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_B, "Intel(R) Ethernet Connection XL710 Driver"), + PVID(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_C, "Intel(R) Ethernet Connection XL710 Driver"), + PVID(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_A, "Intel(R) Ethernet Connection XL710 Driver"), + PVID(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_B, "Intel(R) Ethernet Connection XL710 Driver"), + PVID(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_C, "Intel(R) Ethernet Connection XL710 Driver"), + PVID(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T, "Intel(R) Ethernet Connection XL710 Driver"), + PVID(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T4,"Intel(R) Ethernet Connection XL710 Driver"), +#ifdef X722_SUPPORT + PVID(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_X722, "Intel(R) Ethernet Connection XL710 Driver"), + PVID(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_1G_BASE_T_X722, "Intel(R) Ethernet Connection XL710 Driver"), + PVID(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T_X722, "Intel(R) Ethernet Connection XL710 Driver"), +#endif + /* required last entry */ + PVID_END +}; + + +/********************************************************************* + * Function prototypes + *********************************************************************/ +static void *ixl_register(device_t); +static int ixl_get_hw_capabilities(struct ixl_pf *); +static void ixl_update_link_status(struct ixl_pf *); +static int ixl_allocate_pci_resources(struct ixl_pf *); +static u16 ixl_get_bus_info(struct i40e_hw *, device_t); +static int ixl_switch_config(struct ixl_pf *); +static int ixl_initialize_vsi(struct ixl_vsi *); +static void ixl_configure_msix(struct ixl_pf *); +static void ixl_configure_itr(struct ixl_pf *); +static void ixl_configure_legacy(struct ixl_pf *); +static void ixl_free_pci_resources(struct ixl_pf *); +static int ixl_setup_interface(device_t, struct ixl_vsi *); +static void ixl_link_event(struct ixl_pf *, struct i40e_arq_event_info *); +static void ixl_config_rss(struct ixl_vsi *); +static void ixl_set_queue_rx_itr(struct ixl_rx_queue *); +#ifdef notyet +static void ixl_set_queue_tx_itr(struct ixl_tx_queue *); +#endif +static int ixl_set_advertised_speeds(struct ixl_pf *, int); + +static int ixl_enable_rings(struct ixl_vsi *); +static int ixl_disable_rings(struct ixl_vsi *); +static void ixl_enable_intr(struct ixl_vsi *); + +static void ixl_enable_adminq(struct i40e_hw *); +static void ixl_disable_adminq(struct i40e_hw *); +static void ixl_enable_legacy(struct i40e_hw *); +static void ixl_disable_legacy(struct i40e_hw *); + +static void ixl_del_multi(struct ixl_vsi *); +static void ixl_setup_vlan_filters(struct ixl_vsi *); + +static void ixl_init_filters(struct ixl_vsi *); +static void ixl_reconfigure_filters(struct ixl_vsi *vsi); +static void ixl_add_filter(struct ixl_vsi *, u8 *, s16 vlan); +static void ixl_del_filter(struct ixl_vsi *, u8 *, s16 vlan); +static void ixl_add_hw_filters(struct ixl_vsi *, int, int); +static void ixl_del_hw_filters(struct ixl_vsi *, int); +static struct ixl_mac_filter * + ixl_find_filter(struct ixl_vsi *, u8 *, s16); +static void ixl_add_mc_filter(struct ixl_vsi *, u8 *); +static void ixl_free_mac_filters(struct ixl_vsi *vsi); + + +/* Sysctl debug interface */ +#ifdef IXL_DEBUG_SYSCTL +static int ixl_debug_info(SYSCTL_HANDLER_ARGS); +static void ixl_print_debug_info(struct ixl_pf *); +#endif + +/* The MSI/X Interrupt handlers */ +int ixl_intr(void *); +static int ixl_msix_que(void *); +static int ixl_msix_adminq(void *); +static void ixl_handle_mdd_event(struct ixl_pf *); + +/* Sysctl handlers */ +static int ixl_set_flowcntl(SYSCTL_HANDLER_ARGS); +static int ixl_set_advertise(SYSCTL_HANDLER_ARGS); +static int ixl_current_speed(SYSCTL_HANDLER_ARGS); +static int ixl_sysctl_show_fw(SYSCTL_HANDLER_ARGS); + +/* Statistics */ +static void ixl_add_hw_stats(struct ixl_pf *); +static void ixl_add_sysctls_mac_stats(struct sysctl_ctx_list *, + struct sysctl_oid_list *, struct i40e_hw_port_stats *); +static void ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *, + struct sysctl_oid_list *, + struct i40e_eth_stats *); +static void ixl_update_stats_counters(struct ixl_pf *); +static void ixl_update_eth_stats(struct ixl_vsi *); +static void ixl_update_vsi_stats(struct ixl_vsi *); +static void ixl_pf_reset_stats(struct ixl_pf *); +static void ixl_vsi_reset_stats(struct ixl_vsi *); +static void ixl_stat_update48(struct i40e_hw *, u32, u32, bool, + u64 *, u64 *); +static void ixl_stat_update32(struct i40e_hw *, u32, bool, + u64 *, u64 *); + +#ifdef IXL_DEBUG_SYSCTL +static int ixl_sysctl_link_status(SYSCTL_HANDLER_ARGS); +static int ixl_sysctl_phy_abilities(SYSCTL_HANDLER_ARGS); +static int ixl_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS); +static int ixl_sysctl_hw_res_alloc(SYSCTL_HANDLER_ARGS); +static int ixl_sysctl_switch_config(SYSCTL_HANDLER_ARGS); +#endif + +#ifdef PCI_IOV +static int ixl_adminq_err_to_errno(enum i40e_admin_queue_err err); + +static int ixl_if_iov_init(if_ctx_t, uint16_t num_vfs, const nvlist_t*); +static void ixl_if_iov_uninit(if_ctx_t); +static int ixl_if_vf_add(if_ctx_t, uint16_t vfnum, const nvlist_t*); + +static void ixl_handle_vf_msg(struct ixl_pf *, + struct i40e_arq_event_info *); +static void ixl_if_handle_vflr(if_ctx_t ctx); + +static void ixl_reset_vf(struct ixl_pf *pf, struct ixl_vf *vf); +static void ixl_reinit_vf(struct ixl_pf *pf, struct ixl_vf *vf); +#endif + +static int ixl_if_attach_pre(if_ctx_t); +static int ixl_if_attach_post(if_ctx_t); +static int ixl_if_msix_intr_assign(if_ctx_t, int); + +static int ixl_if_detach(if_ctx_t); + +static void ixl_if_init(if_ctx_t ctx); +static void ixl_if_stop(if_ctx_t ctx); + +static void ixl_if_intr_enable(if_ctx_t ctx); +static void ixl_if_intr_disable(if_ctx_t ctx); +static void ixl_if_queue_intr_enable(if_ctx_t ctx, uint16_t qid); +#if 0 +static void ixl_if_queue_intr_disable(if_ctx_t ctx, uint16_t qid); +#endif +static void ixl_if_multi_set(if_ctx_t); +static void ixl_if_update_admin_status(if_ctx_t); +static int ixl_if_mtu_set(if_ctx_t, uint32_t); + +static void ixl_if_media_status(if_ctx_t, struct ifmediareq *); + +static void ixl_if_vlan_register(if_ctx_t ctx, u16 vtag); +static void ixl_if_vlan_unregister(if_ctx_t ctx, u16 vtag); +static uint64_t ixl_if_get_counter(if_ctx_t ctx, ift_counter cnt); + + +static void ixl_if_timer(if_ctx_t, uint16_t); +static int ixl_if_promisc_set(if_ctx_t ctx, int flags); + + +/********************************************************************* + * FreeBSD Device Interface Entry Points + *********************************************************************/ + +static device_method_t ixl_methods[] = { + /* Device interface */ + DEVMETHOD(device_register, ixl_register), + DEVMETHOD(device_probe, iflib_device_probe), + DEVMETHOD(device_attach, iflib_device_attach), + DEVMETHOD(device_detach, iflib_device_detach), + DEVMETHOD(device_shutdown, iflib_device_suspend), +#ifdef PCI_IOV + DEVMETHOD(pci_iov_init, iflib_device_iov_init), + DEVMETHOD(pci_iov_uninit, iflib_device_iov_uninit), + DEVMETHOD(pci_iov_add_vf, iflib_device_iov_add_vf), +#endif + {0, 0} +}; + +static driver_t ixl_driver = { + "ixl", ixl_methods, sizeof(struct ixl_pf), +}; + +devclass_t ixl_devclass; +DRIVER_MODULE(ixl, pci, ixl_driver, ixl_devclass, 0, 0); + +MODULE_DEPEND(ixl, pci, 1, 1, 1); +MODULE_DEPEND(ixl, ether, 1, 1, 1); +MODULE_DEPEND(ixl, iflib, 1, 1, 1); + + +static device_method_t ixl_if_methods[] = { + DEVMETHOD(ifdi_attach_pre, ixl_if_attach_pre), + DEVMETHOD(ifdi_attach_post, ixl_if_attach_post), + DEVMETHOD(ifdi_detach, ixl_if_detach), + DEVMETHOD(ifdi_init, ixl_if_init), + DEVMETHOD(ifdi_stop, ixl_if_stop), + DEVMETHOD(ifdi_msix_intr_assign, ixl_if_msix_intr_assign), + DEVMETHOD(ifdi_intr_disable, ixl_if_intr_disable), + DEVMETHOD(ifdi_intr_enable, ixl_if_intr_enable), + DEVMETHOD(ifdi_queue_intr_enable, ixl_if_queue_intr_enable), + DEVMETHOD(ifdi_multi_set, ixl_if_multi_set), + DEVMETHOD(ifdi_tx_queues_alloc, ixl_if_tx_queues_alloc), + DEVMETHOD(ifdi_rx_queues_alloc, ixl_if_rx_queues_alloc), + DEVMETHOD(ifdi_update_admin_status, ixl_if_update_admin_status), + DEVMETHOD(ifdi_mtu_set, ixl_if_mtu_set), + DEVMETHOD(ifdi_media_status, ixl_if_media_status), + DEVMETHOD(ifdi_media_change, ixl_if_media_change), + DEVMETHOD(ifdi_timer, ixl_if_timer), + DEVMETHOD(ifdi_promisc_set, ixl_if_promisc_set), + DEVMETHOD(ifdi_vlan_register, ixl_if_vlan_register), + DEVMETHOD(ifdi_vlan_unregister, ixl_if_vlan_unregister), + DEVMETHOD(ifdi_queues_free, ixl_if_queues_free), +#ifdef PCI_IOV + DEVMETHOD(ifdi_vflr_handle, ixl_if_handle_vflr), + DEVMETHOD(ifdi_iov_init, ixl_if_iov_init), + DEVMETHOD(ifdi_iov_uninit, ixl_if_iov_uninit), + DEVMETHOD(ifdi_iov_vf_add, ixl_if_vf_add), +#endif + DEVMETHOD(ifdi_get_counter, ixl_if_get_counter), + DEVMETHOD_END +}; + +static driver_t ixl_if_driver = { + "ixl_if", ixl_if_methods, sizeof(struct ixl_pf), +}; + + +/* +** TUNEABLE PARAMETERS: +*/ + +static SYSCTL_NODE(_hw, OID_AUTO, ixl, CTLFLAG_RD, 0, + "IXL driver parameters"); + +/* + * MSIX should be the default for best performance, + * but this allows it to be forced off for testing. + */ +static int ixl_enable_msix = 1; +TUNABLE_INT("hw.ixl.enable_msix", &ixl_enable_msix); +SYSCTL_INT(_hw_ixl, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &ixl_enable_msix, 0, + "Enable MSI-X interrupts"); + +/* +** Number of descriptors per ring: +** - TX and RX are the same size +*/ +static int ixl_ringsz = DEFAULT_RING; +TUNABLE_INT("hw.ixl.ringsz", &ixl_ringsz); +SYSCTL_INT(_hw_ixl, OID_AUTO, ring_size, CTLFLAG_RDTUN, + &ixl_ringsz, 0, "Descriptor Ring Size"); + +/* +** Controls for Interrupt Throttling +** - true/false for dynamic adjustment +** - default values for static ITR +*/ +int ixl_dynamic_rx_itr = 0; +TUNABLE_INT("hw.ixl.dynamic_rx_itr", &ixl_dynamic_rx_itr); +SYSCTL_INT(_hw_ixl, OID_AUTO, dynamic_rx_itr, CTLFLAG_RDTUN, + &ixl_dynamic_rx_itr, 0, "Dynamic RX Interrupt Rate"); + +int ixl_dynamic_tx_itr = 0; +TUNABLE_INT("hw.ixl.dynamic_tx_itr", &ixl_dynamic_tx_itr); +SYSCTL_INT(_hw_ixl, OID_AUTO, dynamic_tx_itr, CTLFLAG_RDTUN, + &ixl_dynamic_tx_itr, 0, "Dynamic TX Interrupt Rate"); + +int ixl_rx_itr = IXL_ITR_8K; +TUNABLE_INT("hw.ixl.rx_itr", &ixl_rx_itr); +SYSCTL_INT(_hw_ixl, OID_AUTO, rx_itr, CTLFLAG_RDTUN, + &ixl_rx_itr, 0, "RX Interrupt Rate"); + +int ixl_tx_itr = IXL_ITR_4K; +TUNABLE_INT("hw.ixl.tx_itr", &ixl_tx_itr); +SYSCTL_INT(_hw_ixl, OID_AUTO, tx_itr, CTLFLAG_RDTUN, + &ixl_tx_itr, 0, "TX Interrupt Rate"); + +#ifdef IXL_FDIR +static int ixl_enable_fdir = 1; +TUNABLE_INT("hw.ixl.enable_fdir", &ixl_enable_fdir); +/* Rate at which we sample */ +int ixl_atr_rate = 20; +TUNABLE_INT("hw.ixl.atr_rate", &ixl_atr_rate); +#endif + +static char *ixl_fc_string[6] = { + "None", + "Rx", + "Tx", + "Full", + "Priority", + "Default" +}; + +extern struct if_txrx ixl_txrx; + +static struct if_shared_ctx ixl_sctx_init = { + .isc_magic = IFLIB_MAGIC, + .isc_q_align = PAGE_SIZE,/* max(DBA_ALIGN, PAGE_SIZE) */ + .isc_tx_maxsize = IXL_TSO_SIZE, + + .isc_tx_maxsegsize = PAGE_SIZE, + + .isc_rx_maxsize = PAGE_SIZE*4, + .isc_rx_nsegments = 1, + .isc_rx_maxsegsize = PAGE_SIZE*4, + .isc_ntxd = DEFAULT_RING, + .isc_nrxd = DEFAULT_RING, + .isc_nfl = 1, + .isc_txqsizes[0] = roundup2((DEFAULT_RING * sizeof(struct i40e_tx_desc)) + + sizeof(u32), DBA_ALIGN), + .isc_rxqsizes[0] = roundup2(DEFAULT_RING * + sizeof(union i40e_rx_desc), DBA_ALIGN), + .isc_ntxqs = 1, + .isc_nrxqs = 1, + .isc_admin_intrcnt = 1, + .isc_vendor_info = ixl_vendor_info_array, + .isc_driver_version = ixl_driver_version, + .isc_txrx = &ixl_txrx, + .isc_driver = &ixl_if_driver, +}; + +if_shared_ctx_t ixl_sctx = &ixl_sctx_init; +MALLOC_DEFINE(M_IXL, "ixl", "ixl driver allocations"); + +static uint8_t ixl_bcast_addr[ETHER_ADDR_LEN] = + {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; + +static void * +ixl_register(device_t dev) +{ + ixl_sctx->isc_ntxd = ixl_ringsz; + ixl_sctx->isc_nrxd = ixl_ringsz; + ixl_sctx->isc_txqsizes[0] = roundup2((ixl_ringsz * sizeof(struct i40e_tx_desc)) + + sizeof(u32), DBA_ALIGN); + ixl_sctx->isc_rxqsizes[1] = roundup2(ixl_ringsz * + sizeof(union i40e_rx_desc), DBA_ALIGN); + + + return (ixl_sctx); +} + + +/********************************************************************* + * Device initialization routine + * + * The attach entry point is called when the driver is being loaded. + * This routine identifies the type of hardware, allocates all resources + * and initializes the hardware. + * + * return 0 on success, positive on failure + *********************************************************************/ + +static int +ixl_if_attach_pre(if_ctx_t ctx) +{ + device_t dev; + struct ixl_pf *pf; + struct i40e_hw *hw; + struct ixl_vsi *vsi; + int error = 0; + + INIT_DEBUGOUT("ixl_attach: begin"); + + dev = iflib_get_dev(ctx); + pf = iflib_get_softc(ctx); + hw = &pf->hw; + vsi = &pf->vsi; + vsi->back = pf; + vsi->hw = &pf->hw; + vsi->id = 0; + vsi->num_vlans = 0; + vsi->ctx = ctx; + vsi->media = iflib_get_media(ctx); + vsi->shared = iflib_get_softc_ctx(ctx); + pf->dev = iflib_get_dev(ctx); + + /* + * These are the same across all current ixl models + */ + vsi->shared->isc_tx_nsegments = IXL_MAX_TX_SEGS; + vsi->shared->isc_msix_bar = PCIR_BAR(IXL_BAR); + + vsi->shared->isc_tx_tso_segments_max = IXL_MAX_TSO_SEGS; + vsi->shared->isc_tx_tso_size_max = IXL_TSO_SIZE; + vsi->shared->isc_tx_tso_segsize_max = PAGE_SIZE; + + /* + ** Note this assumes we have a single embedded VSI, + ** this could be enhanced later to allocate multiple + */ + vsi = &pf->vsi; + + /* Set up sysctls */ + SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW, + pf, 0, ixl_set_flowcntl, "I", "Flow Control"); + + SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW, + pf, 0, ixl_set_advertise, "I", "Advertised Speed"); + + SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "current_speed", CTLTYPE_STRING | CTLFLAG_RD, + pf, 0, ixl_current_speed, "A", "Current Port Speed"); + + SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "fw_version", CTLTYPE_STRING | CTLFLAG_RD, + pf, 0, ixl_sysctl_show_fw, "A", "Firmware version"); + + SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "rx_itr", CTLFLAG_RW, + &ixl_rx_itr, IXL_ITR_8K, "RX ITR"); + + SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "dynamic_rx_itr", CTLFLAG_RW, + &ixl_dynamic_rx_itr, 0, "Dynamic RX ITR"); + + SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "tx_itr", CTLFLAG_RW, + &ixl_tx_itr, IXL_ITR_4K, "TX ITR"); + + SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "dynamic_tx_itr", CTLFLAG_RW, + &ixl_dynamic_tx_itr, 0, "Dynamic TX ITR"); + +#ifdef IXL_DEBUG_SYSCTL + SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, pf, 0, + ixl_debug_info, "I", "Debug Information"); + + /* Debug shared-code message level */ + SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "debug_mask", CTLFLAG_RW, + &pf->hw.debug_mask, 0, "Debug Message Level"); + + SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "vc_debug_level", CTLFLAG_RW, &pf->vc_debug_lvl, + 0, "PF/VF Virtual Channel debug level"); + + SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "link_status", CTLTYPE_STRING | CTLFLAG_RD, + pf, 0, ixl_sysctl_link_status, "A", "Current Link Status"); + + SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "phy_abilities", CTLTYPE_STRING | CTLFLAG_RD, + pf, 0, ixl_sysctl_phy_abilities, "A", "PHY Abilities"); + + SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "filter_list", CTLTYPE_STRING | CTLFLAG_RD, + pf, 0, ixl_sysctl_sw_filter_list, "A", "SW Filter List"); + + SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "hw_res_alloc", CTLTYPE_STRING | CTLFLAG_RD, + pf, 0, ixl_sysctl_hw_res_alloc, "A", "HW Resource Allocation"); + + SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "switch_config", CTLTYPE_STRING | CTLFLAG_RD, + pf, 0, ixl_sysctl_switch_config, "A", "HW Switch Configuration"); +#endif + + /* Save off the PCI information */ + hw->vendor_id = pci_get_vendor(dev); + hw->device_id = pci_get_device(dev); + hw->revision_id = pci_read_config(dev, PCIR_REVID, 1); + hw->subsystem_vendor_id = + pci_read_config(dev, PCIR_SUBVEND_0, 2); + hw->subsystem_device_id = + pci_read_config(dev, PCIR_SUBDEV_0, 2); + + hw->bus.device = pci_get_slot(dev); + hw->bus.func = pci_get_function(dev); + + pf->vc_debug_lvl = 1; + hw->back = &pf->osdep; + pf->osdep.dev = dev; + + /* Do PCI setup - map BAR0, etc */ + if (ixl_allocate_pci_resources(pf)) { + device_printf(dev, "Allocation of PCI resources failed\n"); + error = ENXIO; + goto err_out; + } + + /* Establish a clean starting point */ + i40e_clear_hw(hw); + error = i40e_pf_reset(hw); + if (error) { + device_printf(dev,"PF reset failure %x\n", error); + error = EIO; + goto err_out; + } + + /* Set admin queue parameters */ + hw->aq.num_arq_entries = IXL_AQ_LEN; + hw->aq.num_asq_entries = IXL_AQ_LEN; + hw->aq.arq_buf_size = IXL_AQ_BUFSZ; + hw->aq.asq_buf_size = IXL_AQ_BUFSZ; + + /* Initialize the shared code */ + error = i40e_init_shared_code(hw); + if (error) { + device_printf(dev,"Unable to initialize the shared code\n"); + error = EIO; + goto err_out; + } + + /* Set up the admin queue */ + error = i40e_init_adminq(hw); + if (error) { + device_printf(dev, "The driver for the device stopped " + "because the NVM image is newer than expected.\n" + "You must install the most recent version of " + " the network driver.\n"); + goto err_out; + } + device_printf(dev, "%s\n", ixl_fw_version_str(hw)); + + if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR && + hw->aq.api_min_ver > I40E_FW_API_VERSION_MINOR) + device_printf(dev, "The driver for the device detected " + "a newer version of the NVM image than expected.\n" + "Please install the most recent version of the network driver.\n"); + else if (hw->aq.api_maj_ver < I40E_FW_API_VERSION_MAJOR || + hw->aq.api_min_ver < (I40E_FW_API_VERSION_MINOR - 1)) + device_printf(dev, "The driver for the device detected " + "an older version of the NVM image than expected.\n" + "Please update the NVM image.\n"); + + /* Clear PXE mode */ + i40e_clear_pxe_mode(hw); + + /* Get capabilities from the device */ + error = ixl_get_hw_capabilities(pf); + if (error) { + device_printf(dev, "HW capabilities failure!\n"); + goto err_get_cap; + } + + /* Set up host memory cache */ + error = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, + hw->func_caps.num_rx_qp, 0, 0); + if (error) { + device_printf(dev, "init_lan_hmc failed: %d\n", error); + goto err_get_cap; + } + + error = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); + if (error) { + device_printf(dev, "configure_lan_hmc failed: %d\n", error); + goto err_mac_hmc; + } + + /* Disable LLDP from the firmware */ + i40e_aq_stop_lldp(hw, TRUE, NULL); + + i40e_get_mac_addr(hw, hw->mac.addr); + error = i40e_validate_mac_addr(hw->mac.addr); + if (error) { + device_printf(dev, "validate_mac_addr failed: %d\n", error); + goto err_mac_hmc; + } + bcopy(hw->mac.addr, hw->mac.perm_addr, ETHER_ADDR_LEN); + iflib_set_mac(ctx, hw->mac.addr); + + i40e_get_port_mac_addr(hw, hw->mac.port_addr); + + /* Initialize mac filter list for VSI */ + SLIST_INIT(&vsi->ftl); + + /* + * Tell iflib what the correct modulus is + */ + vsi->shared->isc_rss_table_size = pf->hw.func_caps.rss_table_size; + return (0); + +err_mac_hmc: + i40e_shutdown_lan_hmc(hw); +err_get_cap: + i40e_shutdown_adminq(hw); +err_out: + ixl_free_pci_resources(pf); + ixl_free_mac_filters(vsi); + return (error); +} + +static int +ixl_if_attach_post(if_ctx_t ctx) +{ + device_t dev; + struct ixl_pf *pf; + struct i40e_hw *hw; + struct ixl_vsi *vsi; + int error = 0; + u16 bus; +#ifdef PCI_IOV + nvlist_t *pf_schema, *vf_schema; + int iov_error; +#endif + + INIT_DEBUGOUT("ixl_attach: begin"); + + dev = iflib_get_dev(ctx); + vsi = iflib_get_softc(ctx); + vsi->ifp = iflib_get_ifp(ctx); + pf = (struct ixl_pf *)vsi; + hw = &pf->hw; + + if (((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver < 33)) || + (hw->aq.fw_maj_ver < 4)) { + i40e_msec_delay(75); + error = i40e_aq_set_link_restart_an(hw, TRUE, NULL); + if (error) { + device_printf(dev, "link restart failed, aq_err=%d\n", + pf->hw.aq.asq_last_status); + goto err_mac_hmc; + } + } + + /* Determine link state */ + i40e_aq_get_link_info(hw, TRUE, NULL, NULL); + i40e_get_link_status(hw, &pf->link_up); + + if (ixl_setup_interface(dev, vsi) != 0) { + device_printf(dev, "interface setup failed!\n"); + error = EIO; + } + + if (error) { + device_printf(dev, "Interface setup failed: %d\n", error); + goto err_mac_hmc; + } else + device_printf(dev, "%s success!\n", __FUNCTION__); + + error = ixl_switch_config(pf); + if (error) { + device_printf(dev, "Initial switch config failed: %d\n", error); + goto err_mac_hmc; + } + + /* Limit phy interrupts to link and modules failure */ + error = i40e_aq_set_phy_int_mask(hw, ~(I40E_AQ_EVENT_LINK_UPDOWN | + I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL); + if (error) + device_printf(dev, "set phy mask failed: %d\n", error); + + /* Get the bus configuration and set the shared code */ + bus = ixl_get_bus_info(hw, dev); + i40e_set_pci_config_data(hw, bus); + + /* Initialize statistics */ + ixl_pf_reset_stats(pf); + ixl_update_stats_counters(pf); + ixl_add_hw_stats(pf); + +#ifdef PCI_IOV + /* SR-IOV is only supported when MSI-X is in use. */ + if (pf->msix > 1) { + pf_schema = pci_iov_schema_alloc_node(); + vf_schema = pci_iov_schema_alloc_node(); + pci_iov_schema_add_unicast_mac(vf_schema, "mac-addr", 0, NULL); + pci_iov_schema_add_bool(vf_schema, "mac-anti-spoof", + IOV_SCHEMA_HASDEFAULT, TRUE); + pci_iov_schema_add_bool(vf_schema, "allow-set-mac", + IOV_SCHEMA_HASDEFAULT, FALSE); + pci_iov_schema_add_bool(vf_schema, "allow-promisc", + IOV_SCHEMA_HASDEFAULT, FALSE); + + iov_error = pci_iov_attach(dev, pf_schema, vf_schema); + if (iov_error != 0) + device_printf(dev, + "Failed to initialize SR-IOV (error=%d)\n", + iov_error); + } +#endif + INIT_DEBUGOUT("ixl_attach: end"); + device_printf(dev, "%s success!\n", __FUNCTION__); + return (0); +err_mac_hmc: + i40e_shutdown_lan_hmc(hw); + i40e_shutdown_adminq(hw); + ixl_free_pci_resources(pf); + ixl_free_mac_filters(vsi); + return (error); +} + +/********************************************************************* + * Device removal routine + * + * The detach entry point is called when the driver is being removed. + * This routine stops the adapter and deallocates all the resources + * that were allocated for driver operation. + * + * return 0 on success, positive on failure + *********************************************************************/ + +static int +ixl_if_detach(if_ctx_t ctx) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + struct ixl_pf *pf = vsi->back; + struct i40e_hw *hw = &pf->hw; + i40e_status status; +#ifdef PCI_IOV + int error; +#endif + + INIT_DEBUGOUT("ixl_detach: begin"); + +#ifdef PCI_IOV + error = pci_iov_detach(iflib_get_dev(ctx)); + if (error != 0) { + device_printf(iflib_get_dev(ctx), "SR-IOV in use; detach first.\n"); + return (error); + } +#endif + + /* Shutdown LAN HMC */ + status = i40e_shutdown_lan_hmc(hw); + if (status) + device_printf(iflib_get_dev(ctx), + "Shutdown LAN HMC failed with code %d\n", status); + + /* Shutdown admin queue */ + status = i40e_shutdown_adminq(hw); + if (status) + device_printf(iflib_get_dev(ctx), + "Shutdown Admin queue failed with code %d\n", status); + else + device_printf(iflib_get_dev(ctx), + "Shutdown Admin queue success\n"); + ixl_free_pci_resources(pf); + ixl_free_mac_filters(vsi); + return (0); +} + +/********************************************************************* + * + * Get the hardware capabilities + * + **********************************************************************/ + +static int +ixl_get_hw_capabilities(struct ixl_pf *pf) +{ + struct i40e_aqc_list_capabilities_element_resp *buf; + struct i40e_hw *hw = &pf->hw; + device_t dev = iflib_get_dev(((struct ixl_vsi *)pf)->ctx); + int error, len; + u16 needed; + bool again = TRUE; + + len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp); +retry: + if (!(buf = (struct i40e_aqc_list_capabilities_element_resp *) + malloc(len, M_IXL, M_NOWAIT | M_ZERO))) { + device_printf(dev, "Unable to allocate cap memory\n"); + return (ENOMEM); + } + + /* This populates the hw struct */ + error = i40e_aq_discover_capabilities(hw, buf, len, + &needed, i40e_aqc_opc_list_func_capabilities, NULL); + free(buf, M_IXL); + if ((pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) && + (again == TRUE)) { + /* retry once with a larger buffer */ + again = FALSE; + len = needed; + goto retry; + } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) { + device_printf(dev, "capability discovery failed: %d\n", + pf->hw.aq.asq_last_status); + return (ENODEV); + } + + /* Capture this PF's starting queue pair */ + pf->qbase = hw->func_caps.base_queue; + +#ifdef IXL_DEBUG + device_printf(dev,"pf_id=%d, num_vfs=%d, msix_pf=%d, " + "msix_vf=%d, fd_g=%d, fd_b=%d, tx_qp=%d rx_qp=%d qbase=%d\n", + hw->pf_id, hw->func_caps.num_vfs, + hw->func_caps.num_msix_vectors, + hw->func_caps.num_msix_vectors_vf, + hw->func_caps.fd_filters_guaranteed, + hw->func_caps.fd_filters_best_effort, + hw->func_caps.num_tx_qp, + hw->func_caps.num_rx_qp, + hw->func_caps.base_queue); +#endif + return (error); +} + +/********************************************************************* + * Ioctl mtu entry point + * + * + * return 0 on success, EINVAL on failure + **********************************************************************/ +static int +ixl_if_mtu_set(if_ctx_t ctx, uint32_t mtu) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + + IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); + if (mtu > IXL_MAX_FRAME - ETHER_HDR_LEN - ETHER_CRC_LEN - + ETHER_VLAN_ENCAP_LEN) + return (EINVAL); + + vsi->max_frame_size = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN + + ETHER_VLAN_ENCAP_LEN; + return (0); +} + +/********************************************************************* + * Init entry point + * + * This routine is used in two ways. It is used by the stack as + * init entry point in network interface structure. It is also used + * by the driver as a hw/sw initialization routine to get to a + * consistent state. + * + * return 0 on success, positive on failure + **********************************************************************/ + +static void +ixl_if_init(if_ctx_t ctx) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + struct ixl_pf *pf = vsi->back; + device_t dev = iflib_get_dev(ctx); + + struct i40e_hw *hw = &pf->hw; + struct i40e_filter_control_settings filter; + u8 tmpaddr[ETHER_ADDR_LEN]; + int ret; + + INIT_DEBUGOUT("ixl_init: begin"); + + /* Get the latest mac address... User might use a LAA */ + bcopy(IF_LLADDR(iflib_get_ifp(ctx)), tmpaddr, + I40E_ETH_LENGTH_OF_ADDRESS); + if (!cmp_etheraddr(hw->mac.addr, tmpaddr) && + (i40e_validate_mac_addr(tmpaddr) == I40E_SUCCESS)) { + ixl_del_filter(vsi, hw->mac.addr, IXL_VLAN_ANY); + bcopy(tmpaddr, hw->mac.addr, + I40E_ETH_LENGTH_OF_ADDRESS); + ret = i40e_aq_mac_address_write(hw, + I40E_AQC_WRITE_TYPE_LAA_ONLY, + hw->mac.addr, NULL); + if (ret) { + device_printf(dev, "LLA address" + "change failed!!\n"); + return; + } else { + ixl_add_filter(vsi, hw->mac.addr, IXL_VLAN_ANY); + } + } + + /* Set up the device filtering */ + bzero(&filter, sizeof(filter)); + filter.enable_ethtype = TRUE; + filter.enable_macvlan = TRUE; +#ifdef IXL_FDIR + filter.enable_fdir = TRUE; +#endif + filter.hash_lut_size = I40E_HASH_LUT_SIZE_512; + if (i40e_set_filter_control(hw, &filter)) + device_printf(dev, "set_filter_control() failed\n"); + + /* Set up RSS */ + ixl_config_rss(vsi); + + /* + ** Prepare the VSI: rings, hmc contexts, etc... + */ + if (ixl_initialize_vsi(vsi)) { + device_printf(dev, "initialize vsi failed!!\n"); + return; + } + + /* Add protocol filters to list */ + ixl_init_filters(vsi); + + /* Setup vlan's if needed */ + ixl_setup_vlan_filters(vsi); + + /* Set up MSI/X routing and the ITR settings */ + if (ixl_enable_msix) { + ixl_configure_msix(pf); + ixl_configure_itr(pf); + } else + ixl_configure_legacy(pf); + + ixl_enable_rings(vsi); + + i40e_aq_set_default_vsi(hw, vsi->seid, NULL); + + ixl_reconfigure_filters(vsi); + + /* Set MTU in hardware*/ + int aq_error = i40e_aq_set_mac_config(hw, vsi->max_frame_size, + TRUE, 0, NULL); + if (aq_error) + device_printf(dev, + "aq_set_mac_config in init error, code %d\n", + aq_error); +} + +/********************************************************************* + * + * Legacy Interrupt Service routine + * + **********************************************************************/ +int +ixl_intr(void *arg) +{ + struct ixl_pf *pf = arg; + struct i40e_hw *hw = &pf->hw; + struct ixl_vsi *vsi = &pf->vsi; + struct ixl_rx_queue *que = vsi->rx_queues; + u32 reg, icr0, mask; + + ++que->irqs; + + icr0 = rd32(hw, I40E_PFINT_ICR0); + + reg = rd32(hw, I40E_PFINT_DYN_CTL0); + reg = reg | I40E_PFINT_DYN_CTL0_CLEARPBA_MASK; + wr32(hw, I40E_PFINT_DYN_CTL0, reg); + + mask = rd32(hw, I40E_PFINT_ICR0_ENA); + +#ifdef PCI_IOV + if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) + iflib_iov_intr_deferred(vsi->ctx); +#endif + + if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) { + iflib_admin_intr_deferred(vsi->ctx); + return (FILTER_HANDLED); + } + + /* re-enable other interrupt causes */ + wr32(hw, I40E_PFINT_ICR0_ENA, mask); + + /* And now the queues */ + reg = rd32(hw, I40E_QINT_RQCTL(0)); + reg |= I40E_QINT_RQCTL_CAUSE_ENA_MASK; + wr32(hw, I40E_QINT_RQCTL(0), reg); + + reg = rd32(hw, I40E_QINT_TQCTL(0)); + reg |= I40E_QINT_TQCTL_CAUSE_ENA_MASK; + reg &= ~I40E_PFINT_ICR0_INTEVENT_MASK; + wr32(hw, I40E_QINT_TQCTL(0), reg); + return (FILTER_SCHEDULE_THREAD); +} + + +/********************************************************************* + * + * MSIX VSI Interrupt Service routine + * + **********************************************************************/ +int +ixl_msix_que(void *arg) +{ + struct ixl_rx_queue *que = arg; + + ixl_set_queue_rx_itr(que); +#ifdef notyet + ixl_set_queue_tx_itr(que); +#endif + return (FILTER_SCHEDULE_THREAD); +} + + +/********************************************************************* + * + * MSIX Admin Queue Interrupt Service routine + * + **********************************************************************/ +int +ixl_msix_adminq(void *arg) +{ + struct ixl_pf *pf = arg; + struct i40e_hw *hw = &pf->hw; + u32 reg, mask; + + ++pf->admin_irq; + + reg = rd32(hw, I40E_PFINT_ICR0); + mask = rd32(hw, I40E_PFINT_ICR0_ENA); + + /* Check on the cause */ + if (reg & I40E_PFINT_ICR0_ADMINQ_MASK) + mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK; + + if (reg & I40E_PFINT_ICR0_MAL_DETECT_MASK) { + ixl_handle_mdd_event(pf); + mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; + } + +#ifdef PCI_IOV + if (reg & I40E_PFINT_ICR0_VFLR_MASK) { + mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK; + iflib_iov_intr_deferred(pf->vsi.ctx); + } +#endif + + reg = rd32(hw, I40E_PFINT_DYN_CTL0); + reg = reg | I40E_PFINT_DYN_CTL0_CLEARPBA_MASK; + wr32(hw, I40E_PFINT_DYN_CTL0, reg); + + iflib_admin_intr_deferred(pf->vsi.ctx); + return (FILTER_HANDLED); +} + +/********************************************************************* + * + * Setup MSIX Interrupt resources and handlers for the VSI + * + **********************************************************************/ +int +ixl_if_msix_intr_assign(if_ctx_t ctx, int msix) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + struct ixl_pf *pf = vsi->back; + struct ixl_rx_queue *que = vsi->rx_queues; + struct ixl_tx_queue *tx_que = vsi->tx_queues; + int err, i, rid, vector = 0; + char buf[16]; + + /* Admin Que is vector 0*/ + rid = vector + 1; + + err = iflib_irq_alloc_generic(ctx, &vsi->irq, rid, IFLIB_INTR_ADMIN, + ixl_msix_adminq, pf, 0, "aq"); + if (err) { + iflib_irq_free(ctx, &vsi->irq); + device_printf(iflib_get_dev(ctx), "Failed to register Admin que handler"); + return (err); + } + pf->admvec = vector; + ++vector; + iflib_softirq_alloc_generic(ctx, rid, IFLIB_INTR_IOV, pf, 0, "ixl_iov"); + + /* Now set up the stations */ + for (i = 0; i < vsi->num_rx_queues; i++, vector++, que++) { + rid = vector + 1; + + snprintf(buf, sizeof(buf), "rxq%d", i); + err = iflib_irq_alloc_generic(ctx, &que->que_irq, rid, IFLIB_INTR_RX, + ixl_msix_que, que, que->rxr.me, buf); + if (err) { + device_printf(iflib_get_dev(ctx), "Failed to allocate q int %d err: %d", i, err); + vsi->num_rx_queues = i + 1; + goto fail; + } + que->msix = vector; + } + + for (i = 0, tx_que = vsi->tx_queues; i < vsi->num_tx_queues; i++, tx_que++) { + snprintf(buf, sizeof(buf), "txq%d", i); + rid = que->msix + 1; + iflib_softirq_alloc_generic(ctx, rid, IFLIB_INTR_TX, tx_que, tx_que->txr.me, buf); + } + + return (0); +fail: + iflib_irq_free(ctx, &vsi->irq); + que = vsi->rx_queues; + for (int i = 0; i < vsi->num_rx_queues; i++, que++) + iflib_irq_free(ctx, &que->que_irq); + return (err); +} + +/********************************************************************* + * + * Media Ioctl callback + * + * This routine is called whenever the user queries the status of + * the interface using ifconfig. + * + **********************************************************************/ +static void +ixl_if_media_status(if_ctx_t ctx, struct ifmediareq * ifmr) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + struct ixl_pf *pf = (struct ixl_pf *)vsi->back; + struct i40e_hw *hw = &pf->hw; + + INIT_DEBUGOUT("ixl_media_status: begin"); + + hw->phy.get_link_info = TRUE; + i40e_get_link_status(hw, &pf->link_up); + ixl_update_link_status(pf); + + ifmr->ifm_status = IFM_AVALID; + ifmr->ifm_active = IFM_ETHER; + + if (!pf->link_up) { + return; + } + + ifmr->ifm_status |= IFM_ACTIVE; + /* Hardware is always full-duplex */ + ifmr->ifm_active |= IFM_FDX; + + switch (hw->phy.link_info.phy_type) { + /* 100 M */ + case I40E_PHY_TYPE_100BASE_TX: + ifmr->ifm_active |= IFM_100_TX; + break; + /* 1 G */ + case I40E_PHY_TYPE_1000BASE_T: + ifmr->ifm_active |= IFM_1000_T; + break; + case I40E_PHY_TYPE_1000BASE_SX: + ifmr->ifm_active |= IFM_1000_SX; + break; + case I40E_PHY_TYPE_1000BASE_LX: + ifmr->ifm_active |= IFM_1000_LX; + break; + /* 10 G */ + case I40E_PHY_TYPE_10GBASE_SFPP_CU: + ifmr->ifm_active |= IFM_10G_TWINAX; + break; + case I40E_PHY_TYPE_10GBASE_SR: + ifmr->ifm_active |= IFM_10G_SR; + break; + case I40E_PHY_TYPE_10GBASE_LR: + ifmr->ifm_active |= IFM_10G_LR; + break; + case I40E_PHY_TYPE_10GBASE_T: + ifmr->ifm_active |= IFM_10G_T; + break; + /* 40 G */ + case I40E_PHY_TYPE_40GBASE_CR4: + case I40E_PHY_TYPE_40GBASE_CR4_CU: + ifmr->ifm_active |= IFM_40G_CR4; + break; + case I40E_PHY_TYPE_40GBASE_SR4: + ifmr->ifm_active |= IFM_40G_SR4; + break; + case I40E_PHY_TYPE_40GBASE_LR4: + ifmr->ifm_active |= IFM_40G_LR4; + break; +#ifndef IFM_ETH_XTYPE + case I40E_PHY_TYPE_1000BASE_KX: + ifmr->ifm_active |= IFM_1000_CX; + break; + case I40E_PHY_TYPE_10GBASE_CR1_CU: + case I40E_PHY_TYPE_10GBASE_CR1: + ifmr->ifm_active |= IFM_10G_TWINAX; + break; + case I40E_PHY_TYPE_10GBASE_KX4: + ifmr->ifm_active |= IFM_10G_CX4; + break; + case I40E_PHY_TYPE_10GBASE_KR: + ifmr->ifm_active |= IFM_10G_SR; + break; + case I40E_PHY_TYPE_40GBASE_KR4: + case I40E_PHY_TYPE_XLPPI: + ifmr->ifm_active |= IFM_40G_SR4; + break; +#else + case I40E_PHY_TYPE_1000BASE_KX: + ifmr->ifm_active |= IFM_1000_KX; + break; + /* ERJ: What's the difference between these? */ + case I40E_PHY_TYPE_10GBASE_CR1_CU: + case I40E_PHY_TYPE_10GBASE_CR1: + ifmr->ifm_active |= IFM_10G_CR1; + break; + case I40E_PHY_TYPE_10GBASE_KX4: + ifmr->ifm_active |= IFM_10G_KX4; + break; + case I40E_PHY_TYPE_10GBASE_KR: + ifmr->ifm_active |= IFM_10G_KR; + break; + case I40E_PHY_TYPE_20GBASE_KR2: + ifmr->ifm_active |= IFM_20G_KR2; + break; + case I40E_PHY_TYPE_40GBASE_KR4: + ifmr->ifm_active |= IFM_40G_KR4; + break; + case I40E_PHY_TYPE_XLPPI: + ifmr->ifm_active |= IFM_40G_XLPPI; + break; +#endif + default: + ifmr->ifm_active |= IFM_UNKNOWN; + break; + } + /* Report flow control status as well */ + if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_TX) + ifmr->ifm_active |= IFM_ETH_TXPAUSE; + if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_RX) + ifmr->ifm_active |= IFM_ETH_RXPAUSE; + +} + + +#ifdef IXL_FDIR +/* +** ATR: Application Targetted Receive - creates a filter +** based on TX flow info that will keep the receive +** portion of the flow on the same queue. Based on the +** implementation this is only available for TCP connections +*/ +void +ixl_atr(struct ixl_tx_queue *que, int hflags, int etype) +{ + struct ixl_vsi *vsi = que->vsi; + if_shared_ctx_t sctx = ixl_sctx; + struct tx_ring *txr = &que->txr; + struct i40e_filter_program_desc *FDIR; + u32 ptype, dtype; + int idx; + + /* check if ATR is enabled and sample rate */ + if ((!ixl_enable_fdir) || (!txr->atr_rate)) + return; + /* + ** We sample all TCP SYN/FIN packets, + ** or at the selected sample rate + */ + txr->atr_count++; + if (((hflags & (TH_FIN | TH_SYN)) == 0) && + (txr->atr_count < txr->atr_rate)) + return; + txr->atr_count = 0; + + /* Get a descriptor to use */ + idx = txr->next_avail; + FDIR = (struct i40e_filter_program_desc *) &txr->base[idx]; + if (++idx == sctx->isc_ntxd) + idx = 0; + + ptype = (que->txr.me << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) & + I40E_TXD_FLTR_QW0_QINDEX_MASK; + + ptype |= (etype == ETHERTYPE_IP) ? + (I40E_FILTER_PCTYPE_NONF_IPV4_TCP << + I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) : + (I40E_FILTER_PCTYPE_NONF_IPV6_TCP << + I40E_TXD_FLTR_QW0_PCTYPE_SHIFT); + + ptype |= vsi->id << I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT; + + dtype = I40E_TX_DESC_DTYPE_FILTER_PROG; + + /* + ** We use the TCP TH_FIN as a trigger to remove + ** the filter, otherwise its an update. + */ + dtype |= (hflags & TH_FIN) ? + (I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE << + I40E_TXD_FLTR_QW1_PCMD_SHIFT) : + (I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE << + I40E_TXD_FLTR_QW1_PCMD_SHIFT); + + dtype |= I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX << + I40E_TXD_FLTR_QW1_DEST_SHIFT; + + dtype |= I40E_FILTER_PROGRAM_DESC_FD_STATUS_FD_ID << + I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT; + + FDIR->qindex_flex_ptype_vsi = htole32(ptype); + FDIR->dtype_cmd_cntindex = htole32(dtype); +} +#endif + +static int +ixl_if_promisc_set(if_ctx_t ctx, int flags) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + struct ifnet *ifp = iflib_get_ifp(ctx); + struct i40e_hw *hw = vsi->hw; + int err; + bool uni = FALSE, multi = FALSE; + + if (flags & IFF_ALLMULTI || + if_multiaddr_count(ifp, MAX_MULTICAST_ADDR) == MAX_MULTICAST_ADDR) + multi = TRUE; + if (flags & IFF_PROMISC) + uni = TRUE; + + err = i40e_aq_set_vsi_unicast_promiscuous(hw, + vsi->seid, uni, NULL); + if (err) + return (err); + err = i40e_aq_set_vsi_multicast_promiscuous(hw, + vsi->seid, multi, NULL); + return (err); +} + +/********************************************************************* + * Filter Routines + * + * Routines for multicast and vlan filter management. + * + *********************************************************************/ +static void +ixl_del_multi(struct ixl_vsi *vsi) +{ + struct ixl_mac_filter *f; + int mcnt = 0; + + IOCTL_DEBUGOUT("ixl_del_multi: begin"); + + SLIST_FOREACH(f, &vsi->ftl, next) { + if ((f->flags & (IXL_FILTER_USED|IXL_FILTER_MC)) == (IXL_FILTER_USED|IXL_FILTER_MC)){ + f->flags |= IXL_FILTER_DEL; + mcnt++; + } + } + if (mcnt > 0) + ixl_del_hw_filters(vsi, mcnt); +} + +static int +ixl_mc_filter_apply(void *arg, struct ifmultiaddr *ifma, int count __unused) +{ + struct ixl_vsi *vsi = arg; + + if (ifma->ifma_addr->sa_family != AF_LINK) + return (0); + ixl_add_mc_filter(vsi, + (u8*)LLADDR((struct sockaddr_dl *) ifma->ifma_addr)); + return (1); +} + +static void +ixl_if_multi_set(if_ctx_t ctx) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + struct i40e_hw *hw = vsi->hw; + int mcnt = 0, flags; + + IOCTL_DEBUGOUT("ixl_if_multi_set: begin"); + + mcnt = if_multiaddr_count(iflib_get_ifp(ctx), MAX_MULTICAST_ADDR); + /* delete existing MC filters */ + ixl_del_multi(vsi); + + if (__predict_false(mcnt == MAX_MULTICAST_ADDR)) { + i40e_aq_set_vsi_multicast_promiscuous(hw, + vsi->seid, TRUE, NULL); + return; + } + /* (re-)install filters for all mcast addresses */ + mcnt = if_multi_apply(iflib_get_ifp(ctx), ixl_mc_filter_apply, vsi); + + if (mcnt > 0) { + flags = (IXL_FILTER_ADD | IXL_FILTER_USED | IXL_FILTER_MC); + ixl_add_hw_filters(vsi, flags, mcnt); + } + + IOCTL_DEBUGOUT("ixl_if_multi_set: end"); +} + +/********************************************************************* + * Timer routine + * + * This routine checks for link status,updates statistics, + * and runs the watchdog check. + * + **********************************************************************/ + +static void +ixl_if_timer(if_ctx_t ctx, uint16_t qid) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + struct ixl_pf *pf = vsi->back; + struct i40e_hw *hw = &pf->hw; + struct ixl_tx_queue *que = &vsi->tx_queues[qid]; + u32 mask; + + /* + ** Check status of the queues + */ + mask = (I40E_PFINT_DYN_CTLN_INTENA_MASK | + I40E_PFINT_DYN_CTLN_SWINT_TRIG_MASK); + + /* Any queues with outstanding work get a sw irq */ + if (que->busy) + wr32(hw, I40E_PFINT_DYN_CTLN(que->txr.me), mask); + + if (qid != 0) + return; + + /* Fire off the adminq task */ + iflib_admin_intr_deferred(ctx); + + /* Update stats */ + ixl_update_stats_counters(pf); +} + +/* +** Note: this routine updates the OS on the link state +** the real check of the hardware only happens with +** a link interrupt. +*/ +static void +ixl_update_link_status(struct ixl_pf *pf) +{ + struct ixl_vsi *vsi = &pf->vsi; + struct i40e_hw *hw = &pf->hw; + device_t dev = iflib_get_dev(vsi->ctx); + + if (pf->link_up){ + if (vsi->link_active == FALSE) { + pf->fc = hw->fc.current_mode; + if (bootverbose) { + device_printf(dev,"Link is up %d Gbps %s," + " Flow Control: %s\n", + ((pf->link_speed == + I40E_LINK_SPEED_40GB)? 40:10), + "Full Duplex", ixl_fc_string[pf->fc]); + } + vsi->link_active = TRUE; + /* + ** Warn user if link speed on NPAR enabled + ** partition is not at least 10GB + */ + if (hw->func_caps.npar_enable && + (hw->phy.link_info.link_speed == I40E_LINK_SPEED_1GB || + hw->phy.link_info.link_speed == I40E_LINK_SPEED_100MB)) + device_printf(dev, "The partition detected link" + "speed that is less than 10Gbps\n"); + iflib_link_state_change(vsi->ctx, LINK_STATE_UP); + } + } else { /* Link down */ + if (vsi->link_active == TRUE) { + if (bootverbose) + device_printf(dev,"Link is Down\n"); + iflib_link_state_change(vsi->ctx, LINK_STATE_DOWN); + vsi->link_active = FALSE; + } + } +} + +/********************************************************************* + * + * This routine disables all traffic on the adapter by issuing a + * global reset on the MAC and deallocates TX/RX buffers. + * + **********************************************************************/ + +static void +ixl_if_stop(if_ctx_t ctx) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + + INIT_DEBUGOUT("ixl_if_stop: begin\n"); +#ifdef notyet + if (pf->num_vfs == 0) + ixl_disable_intr(vsi); + else + ixl_disable_rings_intr(vsi); +#endif + ixl_disable_rings(vsi); +} + +/* + * Plumb MSI/X vectors + */ +static void +ixl_configure_msix(struct ixl_pf *pf) +{ + struct i40e_hw *hw = &pf->hw; + struct ixl_vsi *vsi = &pf->vsi; + u32 reg; + u16 vector = 1; + + /* First set up the adminq - vector 0 */ + wr32(hw, I40E_PFINT_ICR0_ENA, 0); /* disable all */ + rd32(hw, I40E_PFINT_ICR0); /* read to clear */ + + reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | + I40E_PFINT_ICR0_ENA_GRST_MASK | + I40E_PFINT_ICR0_HMC_ERR_MASK | + I40E_PFINT_ICR0_ENA_ADMINQ_MASK | + I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | + I40E_PFINT_ICR0_ENA_VFLR_MASK | + I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK; + wr32(hw, I40E_PFINT_ICR0_ENA, reg); + + wr32(hw, I40E_PFINT_LNKLST0, 0x7FF); + wr32(hw, I40E_PFINT_ITR0(IXL_RX_ITR), 0x003E); + + wr32(hw, I40E_PFINT_DYN_CTL0, + I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK | + I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK); + + wr32(hw, I40E_PFINT_STAT_CTL0, 0); + + /* Next configure the queues */ + for (int i = 0; i < vsi->num_rx_queues; i++, vector++) { + wr32(hw, I40E_PFINT_DYN_CTLN(i), 0); + /* First queue type is RX / type 0 */ + wr32(hw, I40E_PFINT_LNKLSTN(i), i); + + reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK | + (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | + (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | + (i << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | + (I40E_QUEUE_TYPE_TX << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT); + wr32(hw, I40E_QINT_RQCTL(i), reg); + + reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK | + (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | + (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | + (IXL_QUEUE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | + (I40E_QUEUE_TYPE_RX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); + wr32(hw, I40E_QINT_TQCTL(i), reg); + } +} + +/* + * Configure for MSI single vector operation + */ +static void +ixl_configure_legacy(struct ixl_pf *pf) +{ + struct i40e_hw *hw = &pf->hw; + u32 reg; + + wr32(hw, I40E_PFINT_ITR0(0), 0); + wr32(hw, I40E_PFINT_ITR0(1), 0); + + /* Setup "other" causes */ + reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK + | I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK + | I40E_PFINT_ICR0_ENA_GRST_MASK + | I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK + | I40E_PFINT_ICR0_ENA_GPIO_MASK + | I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK + | I40E_PFINT_ICR0_ENA_HMC_ERR_MASK + | I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK + | I40E_PFINT_ICR0_ENA_VFLR_MASK + | I40E_PFINT_ICR0_ENA_ADMINQ_MASK + ; + wr32(hw, I40E_PFINT_ICR0_ENA, reg); + + /* SW_ITR_IDX = 0, but don't change INTENA */ + wr32(hw, I40E_PFINT_DYN_CTL0, + I40E_PFINT_DYN_CTLN_SW_ITR_INDX_MASK | + I40E_PFINT_DYN_CTLN_INTENA_MSK_MASK); + /* SW_ITR_IDX = 0, OTHER_ITR_IDX = 0 */ + wr32(hw, I40E_PFINT_STAT_CTL0, 0); + + /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */ + wr32(hw, I40E_PFINT_LNKLST0, 0); + + /* Associate the queue pair to the vector and enable the q int */ + reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK + | (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) + | (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); + wr32(hw, I40E_QINT_RQCTL(0), reg); + + reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK + | (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) + | (IXL_QUEUE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); + wr32(hw, I40E_QINT_TQCTL(0), reg); + + /* Next enable the queue pair */ + reg = rd32(hw, I40E_QTX_ENA(0)); + reg |= I40E_QTX_ENA_QENA_REQ_MASK; + wr32(hw, I40E_QTX_ENA(0), reg); + + reg = rd32(hw, I40E_QRX_ENA(0)); + reg |= I40E_QRX_ENA_QENA_REQ_MASK; + wr32(hw, I40E_QRX_ENA(0), reg); +} + + +/* + * Set the Initial ITR state + */ +static void +ixl_configure_itr(struct ixl_pf *pf) +{ + struct i40e_hw *hw = &pf->hw; + struct ixl_vsi *vsi = &pf->vsi; + struct ixl_rx_queue *rx_que = vsi->rx_queues; + struct ixl_tx_queue *tx_que = vsi->tx_queues; + + vsi->rx_itr_setting = ixl_rx_itr; + if (ixl_dynamic_rx_itr) + vsi->rx_itr_setting |= IXL_ITR_DYNAMIC; + vsi->tx_itr_setting = ixl_tx_itr; + if (ixl_dynamic_tx_itr) + vsi->tx_itr_setting |= IXL_ITR_DYNAMIC; + + for (int i = 0; i < vsi->num_tx_queues; i++, tx_que++) { + struct tx_ring *txr = &tx_que->txr; + + wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, i), + vsi->tx_itr_setting); + txr->itr = vsi->tx_itr_setting; + txr->latency = IXL_AVE_LATENCY; + } + for (int i = 0; i < vsi->num_rx_queues; i++, rx_que++) { + struct rx_ring *rxr = &rx_que->rxr; + + wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, i), + vsi->rx_itr_setting); + rxr->itr = vsi->rx_itr_setting; + rxr->latency = IXL_AVE_LATENCY; + } +} + + +static int +ixl_allocate_pci_resources(struct ixl_pf *pf) +{ + int rid; + device_t dev = iflib_get_dev(pf->vsi.ctx); + + rid = PCIR_BAR(0); + pf->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, + &rid, RF_ACTIVE); + + if (!(pf->pci_mem)) { + device_printf(dev,"Unable to allocate bus resource: memory\n"); + return (ENXIO); + } + + pf->osdep.mem_bus_space_tag = + rman_get_bustag(pf->pci_mem); + pf->osdep.mem_bus_space_handle = + rman_get_bushandle(pf->pci_mem); + pf->osdep.mem_bus_space_size = rman_get_size(pf->pci_mem); + pf->osdep.flush_reg = I40E_GLGEN_STAT; + pf->hw.hw_addr = (u8 *) &pf->osdep.mem_bus_space_handle; + + pf->hw.back = &pf->osdep; + + return (0); +} + +static void +ixl_free_pci_resources(struct ixl_pf * pf) +{ + struct ixl_vsi *vsi = &pf->vsi; + struct ixl_rx_queue *que = vsi->rx_queues; + device_t dev = iflib_get_dev(vsi->ctx); + + /* We may get here before stations are setup */ + if ((!ixl_enable_msix) || (que == NULL)) + goto early; + + + /* + ** Release all msix VSI resources: + */ + iflib_irq_free(vsi->ctx, &vsi->irq); + + for (int i = 0; i < vsi->num_rx_queues; i++, que++) + iflib_irq_free(vsi->ctx, &que->que_irq); +early: + if (pf->pci_mem != NULL) + bus_release_resource(dev, SYS_RES_MEMORY, + PCIR_BAR(0), pf->pci_mem); + +} + +static void +ixl_add_ifmedia(struct ixl_vsi *vsi, u32 phy_type) +{ + /* Display supported media types */ + if (phy_type & (1 << I40E_PHY_TYPE_100BASE_TX)) + ifmedia_add(vsi->media, IFM_ETHER | IFM_100_TX, 0, NULL); + + if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_T)) + ifmedia_add(vsi->media, IFM_ETHER | IFM_1000_T, 0, NULL); + if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_SX)) + ifmedia_add(vsi->media, IFM_ETHER | IFM_1000_SX, 0, NULL); + if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_LX)) + ifmedia_add(vsi->media, IFM_ETHER | IFM_1000_LX, 0, NULL); + + if (phy_type & (1 << I40E_PHY_TYPE_XAUI) || + phy_type & (1 << I40E_PHY_TYPE_XFI) || + phy_type & (1 << I40E_PHY_TYPE_10GBASE_SFPP_CU)) + ifmedia_add(vsi->media, IFM_ETHER | IFM_10G_TWINAX, 0, NULL); + + if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_SR)) + ifmedia_add(vsi->media, IFM_ETHER | IFM_10G_SR, 0, NULL); + if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_LR)) + ifmedia_add(vsi->media, IFM_ETHER | IFM_10G_LR, 0, NULL); + if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_T)) + ifmedia_add(vsi->media, IFM_ETHER | IFM_10G_T, 0, NULL); + + if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_CR4) || + phy_type & (1 << I40E_PHY_TYPE_40GBASE_CR4_CU) || + phy_type & (1 << I40E_PHY_TYPE_40GBASE_AOC) || + phy_type & (1 << I40E_PHY_TYPE_XLAUI) || + phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4)) + ifmedia_add(vsi->media, IFM_ETHER | IFM_40G_CR4, 0, NULL); + if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_SR4)) + ifmedia_add(vsi->media, IFM_ETHER | IFM_40G_SR4, 0, NULL); + if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_LR4)) + ifmedia_add(vsi->media, IFM_ETHER | IFM_40G_LR4, 0, NULL); + +#ifndef IFM_ETH_XTYPE + if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_KX)) + ifmedia_add(vsi->media, IFM_ETHER | IFM_1000_CX, 0, NULL); + + if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1_CU) || + phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1) || + phy_type & (1 << I40E_PHY_TYPE_10GBASE_AOC) || + phy_type & (1 << I40E_PHY_TYPE_SFI)) + ifmedia_add(vsi->media, IFM_ETHER | IFM_10G_TWINAX, 0, NULL); + if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KX4)) + ifmedia_add(vsi->media, IFM_ETHER | IFM_10G_CX4, 0, NULL); + if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KR)) + ifmedia_add(vsi->media, IFM_ETHER | IFM_10G_SR, 0, NULL); + + if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4)) + ifmedia_add(vsi->media, IFM_ETHER | IFM_40G_SR4, 0, NULL); + if (phy_type & (1 << I40E_PHY_TYPE_XLPPI)) + ifmedia_add(vsi->media, IFM_ETHER | IFM_40G_CR4, 0, NULL); +#else + if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_KX)) + ifmedia_add(vsi->media, IFM_ETHER | IFM_1000_KX, 0, NULL); + + if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1_CU) + || phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1)) + ifmedia_add(vsi->media, IFM_ETHER | IFM_10G_CR1, 0, NULL); + if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_AOC)) + ifmedia_add(vsi->media, IFM_ETHER | IFM_10G_TWINAX_LONG, 0, NULL); + if (phy_type & (1 << I40E_PHY_TYPE_SFI)) + ifmedia_add(vsi->media, IFM_ETHER | IFM_10G_SFI, 0, NULL); + if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KX4)) + ifmedia_add(vsi->media, IFM_ETHER | IFM_10G_KX4, 0, NULL); + if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KR)) + ifmedia_add(vsi->media, IFM_ETHER | IFM_10G_KR, 0, NULL); + + if (phy_type & (1 << I40E_PHY_TYPE_20GBASE_KR2)) + ifmedia_add(vsi->media, IFM_ETHER | IFM_20G_KR2, 0, NULL); + + if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4)) + ifmedia_add(vsi->media, IFM_ETHER | IFM_40G_KR4, 0, NULL); + if (phy_type & (1 << I40E_PHY_TYPE_XLPPI)) + ifmedia_add(vsi->media, IFM_ETHER | IFM_40G_XLPPI, 0, NULL); +#endif +} + +/********************************************************************* + * + * Setup networking device structure and register an interface. + * + **********************************************************************/ +static int +ixl_setup_interface(device_t dev, struct ixl_vsi *vsi) +{ + if_ctx_t ctx = vsi->ctx; + struct ixl_pf *pf = vsi->back; + struct i40e_hw *hw = &pf->hw; + struct ifnet *ifp = iflib_get_ifp(ctx); + struct i40e_aq_get_phy_abilities_resp abilities; + enum i40e_status_code aq_error = 0; + uint64_t cap; + + INIT_DEBUGOUT("ixl_setup_interface: begin"); + /* initialize fast path functions */ + + cap = IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6 | IFCAP_LRO | IFCAP_JUMBO_MTU | IFCAP_TSO; + cap |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWTSO | IFCAP_VLAN_MTU | IFCAP_VLAN_HWCSUM; + if_setifheaderlen(ifp, sizeof(struct ether_vlan_header)); + if_setcapabilitiesbit(ifp, cap, 0); + if_setcapenable(ifp, if_getcapabilities(ifp)); + if_setbaudrate(ifp, 4000000000); + vsi->max_frame_size = + ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN + + ETHER_VLAN_ENCAP_LEN; + + /* + ** Don't turn this on by default, if vlans are + ** created on another pseudo device (eg. lagg) + ** then vlan events are not passed thru, breaking + ** operation, but with HW FILTER off it works. If + ** using vlans directly on the ixl driver you can + ** enable this and get full hardware tag filtering. + */ + ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; + + aq_error = i40e_aq_get_phy_capabilities(hw, + FALSE, TRUE, &abilities, NULL); + /* May need delay to detect fiber correctly */ + if (aq_error == I40E_ERR_UNKNOWN_PHY) { + i40e_msec_delay(200); + aq_error = i40e_aq_get_phy_capabilities(hw, FALSE, + TRUE, &abilities, NULL); + } + if (aq_error) { + if (aq_error == I40E_ERR_UNKNOWN_PHY) + device_printf(dev, "Unknown PHY type detected!\n"); + else + device_printf(dev, + "Error getting supported media types, err %d," + " AQ error %d\n", aq_error, hw->aq.asq_last_status); + return (0); + } + + ixl_add_ifmedia(vsi, abilities.phy_type); + + /* Use autoselect media by default */ + ifmedia_add(vsi->media, IFM_ETHER | IFM_AUTO, 0, NULL); + ifmedia_set(vsi->media, IFM_ETHER | IFM_AUTO); + + return (0); +} + +/* +** Run when the Admin Queue gets a +** link transition interrupt. +*/ +static void +ixl_link_event(struct ixl_pf *pf, struct i40e_arq_event_info *e) +{ + struct i40e_hw *hw = &pf->hw; + struct i40e_aqc_get_link_status *status = + (struct i40e_aqc_get_link_status *)&e->desc.params.raw; + bool check; + + hw->phy.get_link_info = TRUE; + i40e_get_link_status(hw, &check); + pf->link_up = check; +#ifdef IXL_DEBUG + printf("Link is %s\n", check ? "up":"down"); +#endif + /* Report if Unqualified modules are found */ + if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) && + (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) && + (!(status->link_info & I40E_AQ_LINK_UP))) + device_printf(iflib_get_dev(pf->vsi.ctx), "Link failed because " + "an unqualified module was detected\n"); + + return; +} + +/********************************************************************* + * + * Get Firmware Switch configuration + * - this will need to be more robust when more complex + * switch configurations are enabled. + * + **********************************************************************/ +static int +ixl_switch_config(struct ixl_pf *pf) +{ + struct i40e_hw *hw = &pf->hw; + struct ixl_vsi *vsi = &pf->vsi; + device_t dev = iflib_get_dev(vsi->ctx); + struct i40e_aqc_get_switch_config_resp *sw_config; + u8 aq_buf[I40E_AQ_LARGE_BUF]; + int ret; + u16 next = 0; + + memset(&aq_buf, 0, sizeof(aq_buf)); + sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf; + ret = i40e_aq_get_switch_config(hw, sw_config, + sizeof(aq_buf), &next, NULL); + if (ret) { + device_printf(dev,"aq_get_switch_config failed (ret=%d)!!\n", + ret); + return (ret); + } +#ifdef IXL_DEBUG + device_printf(dev, + "Switch config: header reported: %d in structure, %d total\n", + sw_config->header.num_reported, sw_config->header.num_total); + for (int i = 0; i < sw_config->header.num_reported; i++) { + device_printf(dev, + "%d: type=%d seid=%d uplink=%d downlink=%d\n", i, + sw_config->element[i].element_type, + sw_config->element[i].seid, + sw_config->element[i].uplink_seid, + sw_config->element[i].downlink_seid); + } +#endif + /* Simplified due to a single VSI at the moment */ + vsi->uplink_seid = sw_config->element[0].uplink_seid; + vsi->downlink_seid = sw_config->element[0].downlink_seid; + vsi->seid = sw_config->element[0].seid; + return (ret); +} + +/********************************************************************* + * + * Initialize the VSI: this handles contexts, which means things + * like the number of descriptors, buffer size, + * plus we init the rings thru this function. + * + **********************************************************************/ +static int +ixl_initialize_vsi(struct ixl_vsi *vsi) +{ + if_shared_ctx_t sctx = ixl_sctx; + struct ixl_tx_queue *tx_que = vsi->tx_queues; + struct ixl_rx_queue *rx_que = vsi->rx_queues; + device_t dev = iflib_get_dev(vsi->ctx); + struct i40e_hw *hw = vsi->hw; + struct i40e_vsi_context ctxt; + int err = 0; + struct ifnet *ifp = iflib_get_ifp(vsi->ctx); + + memset(&ctxt, 0, sizeof(ctxt)); + ctxt.seid = vsi->seid; +#ifdef notyet + if (pf->veb_seid != 0) + ctxt.uplink_seid = pf->veb_seid; +#endif + ctxt.pf_num = hw->pf_id; + err = i40e_aq_get_vsi_params(hw, &ctxt, NULL); + if (err) { + device_printf(dev,"get vsi params failed %x!!\n", err); + return (err); + } +#ifdef IXL_DEBUG + printf("get_vsi_params: seid: %d, uplinkseid: %d, vsi_number: %d, " + "vsis_allocated: %d, vsis_unallocated: %d, flags: 0x%x, " + "pfnum: %d, vfnum: %d, stat idx: %d, enabled: %d\n", ctxt.seid, + ctxt.uplink_seid, ctxt.vsi_number, + ctxt.vsis_allocated, ctxt.vsis_unallocated, + ctxt.flags, ctxt.pf_num, ctxt.vf_num, + ctxt.info.stat_counter_idx, ctxt.info.up_enable_bits); +#endif + /* + ** Set the queue and traffic class bits + ** - when multiple traffic classes are supported + ** this will need to be more robust. + */ + ctxt.info.valid_sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; + ctxt.info.mapping_flags |= I40E_AQ_VSI_QUE_MAP_CONTIG; + /* In contig mode, que_mapping[0] is first queue index used by this VSI */ + ctxt.info.queue_mapping[0] = 0; + /* + * This VSI will only use traffic class 0; start traffic class 0's + * queue allocation at queue 0, and assign it 64 (2^6) queues (though + * the driver may not use all of them). + */ + ctxt.info.tc_mapping[0] = ((0 << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) + & I40E_AQ_VSI_TC_QUE_OFFSET_MASK) | + ((6 << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT) + & I40E_AQ_VSI_TC_QUE_NUMBER_MASK); + + /* Set VLAN receive stripping mode */ + ctxt.info.valid_sections |= I40E_AQ_VSI_PROP_VLAN_VALID; + ctxt.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL; + if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) + ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH; + else + ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_NOTHING; + + /* Keep copy of VSI info in VSI for statistic counters */ + memcpy(&vsi->info, &ctxt.info, sizeof(ctxt.info)); + + /* Reset VSI statistics */ + ixl_vsi_reset_stats(vsi); + vsi->hw_filters_add = 0; + vsi->hw_filters_del = 0; + + ctxt.flags = htole16(I40E_AQ_VSI_TYPE_PF); + + err = i40e_aq_update_vsi_params(hw, &ctxt, NULL); + if (err) { + device_printf(dev,"update vsi params failed %x!!\n", + hw->aq.asq_last_status); + return (err); + } + + for (int i = 0; i < vsi->num_tx_queues; i++, tx_que++) { + struct tx_ring *txr = &tx_que->txr; + struct i40e_hmc_obj_txq tctx; + u32 txctl; + u16 size; + + /* Setup the HMC TX Context */ + size = sctx->isc_ntxd * sizeof(struct i40e_tx_desc); + memset(&tctx, 0, sizeof(struct i40e_hmc_obj_txq)); + tctx.new_context = 1; + + tctx.base = (txr->tx_paddr/IXL_TX_CTX_BASE_UNITS); + tctx.qlen = sctx->isc_ntxd; + tctx.fc_ena = 0; + tctx.rdylist = vsi->info.qs_handle[0]; /* index is TC */ + /* Enable HEAD writeback */ + tctx.head_wb_ena = 1; + tctx.head_wb_addr = txr->tx_paddr + + (sctx->isc_ntxd * sizeof(struct i40e_tx_desc)); + tctx.rdylist_act = 0; + err = i40e_clear_lan_tx_queue_context(hw, i); + if (err) { + device_printf(dev, "Unable to clear TX context\n"); + break; + } + err = i40e_set_lan_tx_queue_context(hw, i, &tctx); + if (err) { + device_printf(dev, "Unable to set TX context\n"); + break; + } + /* Associate the ring with this PF */ + txctl = I40E_QTX_CTL_PF_QUEUE; + txctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) & + I40E_QTX_CTL_PF_INDX_MASK); + wr32(hw, I40E_QTX_CTL(i), txctl); + ixl_flush(hw); + + /* Do ring (re)init */ + ixl_init_tx_ring(vsi, tx_que); + } + for (int i = 0; i < vsi->num_rx_queues; i++, rx_que++) { + struct rx_ring *rxr = &rx_que->rxr; + struct i40e_hmc_obj_rxq rctx; + + /* Next setup the HMC RX Context */ + if (vsi->max_frame_size <= MCLBYTES) + rxr->mbuf_sz = MCLBYTES; + else + rxr->mbuf_sz = MJUMPAGESIZE; + + u16 max_rxmax = rxr->mbuf_sz * hw->func_caps.rx_buf_chain_len; + + /* Set up an RX context for the HMC */ + memset(&rctx, 0, sizeof(struct i40e_hmc_obj_rxq)); + rctx.dbuff = rxr->mbuf_sz >> I40E_RXQ_CTX_DBUFF_SHIFT; + /* ignore header split for now */ + rctx.hbuff = 0 >> I40E_RXQ_CTX_HBUFF_SHIFT; + rctx.rxmax = (vsi->max_frame_size < max_rxmax) ? + vsi->max_frame_size : max_rxmax; + rctx.dtype = 0; + rctx.dsize = 1; /* do 32byte descriptors */ + rctx.hsplit_0 = 0; /* no HDR split initially */ + rctx.base = (rxr->rx_paddr/IXL_RX_CTX_BASE_UNITS); + rctx.qlen = sctx->isc_nrxd; + rctx.tphrdesc_ena = 1; + rctx.tphwdesc_ena = 1; + rctx.tphdata_ena = 0; + rctx.tphhead_ena = 0; + rctx.lrxqthresh = 2; + rctx.crcstrip = 1; + rctx.l2tsel = 1; + rctx.showiv = 1; + rctx.fc_ena = 0; + rctx.prefena = 1; + + err = i40e_clear_lan_rx_queue_context(hw, i); + if (err) { + device_printf(dev, + "Unable to clear RX context %d\n", i); + break; + } + err = i40e_set_lan_rx_queue_context(hw, i, &rctx); + if (err) { + device_printf(dev, "Unable to set RX context %d\n", i); + break; + } + } + return (err); +} + +static void +ixl_free_mac_filters(struct ixl_vsi *vsi) +{ + struct ixl_mac_filter *f; + + while (!SLIST_EMPTY(&vsi->ftl)) { + f = SLIST_FIRST(&vsi->ftl); + SLIST_REMOVE_HEAD(&vsi->ftl, next); + free(f, M_IXL); + } +} + +/********************************************************************* + * + * Allocate memory for the VSI (virtual station interface) and their + * associated queues, rings and the descriptors associated with each, + * called only once at attach. + * + **********************************************************************/ + +/* +** Provide a update to the queue RX +** interrupt moderation value. +*/ +static void +ixl_set_queue_rx_itr(struct ixl_rx_queue *que) +{ + struct ixl_vsi *vsi = que->vsi; + struct i40e_hw *hw = vsi->hw; + struct rx_ring *rxr = &que->rxr; + u16 rx_itr; + u16 rx_latency = 0; + int rx_bytes; + + + /* Idle, do nothing */ + if (rxr->bytes == 0) + return; + + if (ixl_dynamic_rx_itr) { + rx_bytes = rxr->bytes/rxr->itr; + rx_itr = rxr->itr; + + /* Adjust latency range */ + switch (rxr->latency) { + case IXL_LOW_LATENCY: + if (rx_bytes > 10) { + rx_latency = IXL_AVE_LATENCY; + rx_itr = IXL_ITR_20K; + } + break; + case IXL_AVE_LATENCY: + if (rx_bytes > 20) { + rx_latency = IXL_BULK_LATENCY; + rx_itr = IXL_ITR_8K; + } else if (rx_bytes <= 10) { + rx_latency = IXL_LOW_LATENCY; + rx_itr = IXL_ITR_100K; + } + break; + case IXL_BULK_LATENCY: + if (rx_bytes <= 20) { + rx_latency = IXL_AVE_LATENCY; + rx_itr = IXL_ITR_20K; + } + break; + } + + rxr->latency = rx_latency; + + if (rx_itr != rxr->itr) { + /* do an exponential smoothing */ + rx_itr = (10 * rx_itr * rxr->itr) / + ((9 * rx_itr) + rxr->itr); + rxr->itr = rx_itr & IXL_MAX_ITR; + wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, + que->rxr.me), rxr->itr); + } + } else { /* We may have have toggled to non-dynamic */ + if (vsi->rx_itr_setting & IXL_ITR_DYNAMIC) + vsi->rx_itr_setting = ixl_rx_itr; + /* Update the hardware if needed */ + if (rxr->itr != vsi->rx_itr_setting) { + rxr->itr = vsi->rx_itr_setting; + wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, + que->rxr.me), rxr->itr); + } + } + rxr->bytes = 0; + rxr->packets = 0; +} + +#ifdef notyet +/* +** Provide a update to the queue TX +** interrupt moderation value. +*/ +static void +ixl_set_queue_tx_itr(struct ixl_tx_queue *que) +{ + struct ixl_vsi *vsi = que->vsi; + struct i40e_hw *hw = vsi->hw; + struct tx_ring *txr = &que->txr; + u16 tx_itr; + u16 tx_latency = 0; + int tx_bytes; + + + /* Idle, do nothing */ + if (txr->bytes == 0) + return; + + if (ixl_dynamic_tx_itr) { + tx_bytes = txr->bytes/txr->itr; + tx_itr = txr->itr; + + switch (txr->latency) { + case IXL_LOW_LATENCY: + if (tx_bytes > 10) { + tx_latency = IXL_AVE_LATENCY; + tx_itr = IXL_ITR_20K; + } + break; + case IXL_AVE_LATENCY: + if (tx_bytes > 20) { + tx_latency = IXL_BULK_LATENCY; + tx_itr = IXL_ITR_8K; + } else if (tx_bytes <= 10) { + tx_latency = IXL_LOW_LATENCY; + tx_itr = IXL_ITR_100K; + } + break; + case IXL_BULK_LATENCY: + if (tx_bytes <= 20) { + tx_latency = IXL_AVE_LATENCY; + tx_itr = IXL_ITR_20K; + } + break; + } + + txr->latency = tx_latency; + + if (tx_itr != txr->itr) { + /* do an exponential smoothing */ + tx_itr = (10 * tx_itr * txr->itr) / + ((9 * tx_itr) + txr->itr); + txr->itr = tx_itr & IXL_MAX_ITR; + wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, + que->txr.me), txr->itr); + } + + } else { /* We may have have toggled to non-dynamic */ + if (vsi->tx_itr_setting & IXL_ITR_DYNAMIC) + vsi->tx_itr_setting = ixl_tx_itr; + /* Update the hardware if needed */ + if (txr->itr != vsi->tx_itr_setting) { + txr->itr = vsi->tx_itr_setting; + wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, + que->txr.me), txr->itr); + } + } + txr->bytes = 0; + txr->packets = 0; +} +#endif + +#define QUEUE_NAME_LEN 32 + +static void +ixl_add_vsi_sysctls(struct ixl_pf *pf, struct ixl_vsi *vsi, + struct sysctl_ctx_list *ctx, const char *sysctl_name) +{ + struct sysctl_oid *tree; + struct sysctl_oid_list *child; + struct sysctl_oid_list *vsi_list; + + tree = device_get_sysctl_tree(iflib_get_dev(vsi->ctx)); + child = SYSCTL_CHILDREN(tree); + vsi->vsi_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, sysctl_name, + CTLFLAG_RD, NULL, "VSI Number"); + vsi_list = SYSCTL_CHILDREN(vsi->vsi_node); + + ixl_add_sysctls_eth_stats(ctx, vsi_list, &vsi->eth_stats); +} + +static void +ixl_add_hw_stats(struct ixl_pf *pf) +{ + struct ixl_vsi *vsi = &pf->vsi; + device_t dev = iflib_get_dev(vsi->ctx); + struct i40e_hw_port_stats *pf_stats = &pf->stats; + + struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); + struct sysctl_oid *tree = device_get_sysctl_tree(dev); + struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree); + struct sysctl_oid_list *vsi_list; + + /* Driver statistics */ + SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events", + CTLFLAG_RD, &pf->watchdog_events, + "Watchdog timeouts"); + SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "admin_irq", + CTLFLAG_RD, &pf->admin_irq, + "Admin Queue IRQ Handled"); + + ixl_add_vsi_sysctls(pf, &pf->vsi, ctx, "pf"); + vsi_list = SYSCTL_CHILDREN(pf->vsi.vsi_node); + + /* MAC stats */ + ixl_add_sysctls_mac_stats(ctx, child, pf_stats); +} + +static void +ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *ctx, + struct sysctl_oid_list *child, + struct i40e_eth_stats *eth_stats) +{ + struct ixl_sysctl_info ctls[] = + { + {ð_stats->rx_bytes, "good_octets_rcvd", "Good Octets Received"}, + {ð_stats->rx_unicast, "ucast_pkts_rcvd", + "Unicast Packets Received"}, + {ð_stats->rx_multicast, "mcast_pkts_rcvd", + "Multicast Packets Received"}, + {ð_stats->rx_broadcast, "bcast_pkts_rcvd", + "Broadcast Packets Received"}, + {ð_stats->rx_discards, "rx_discards", "Discarded RX packets"}, + {ð_stats->tx_bytes, "good_octets_txd", "Good Octets Transmitted"}, + {ð_stats->tx_unicast, "ucast_pkts_txd", "Unicast Packets Transmitted"}, + {ð_stats->tx_multicast, "mcast_pkts_txd", + "Multicast Packets Transmitted"}, + {ð_stats->tx_broadcast, "bcast_pkts_txd", + "Broadcast Packets Transmitted"}, + // end + {0,0,0} + }; + + struct ixl_sysctl_info *entry = ctls; + while (entry->stat != NULL) + { + SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, entry->name, + CTLFLAG_RD, entry->stat, + entry->description); + entry++; + } +} + +static void +ixl_add_sysctls_mac_stats(struct sysctl_ctx_list *ctx, + struct sysctl_oid_list *child, + struct i40e_hw_port_stats *stats) +{ + struct sysctl_oid *stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac", + CTLFLAG_RD, NULL, "Mac Statistics"); + struct sysctl_oid_list *stat_list = SYSCTL_CHILDREN(stat_node); + + struct i40e_eth_stats *eth_stats = &stats->eth; + ixl_add_sysctls_eth_stats(ctx, stat_list, eth_stats); + + struct ixl_sysctl_info ctls[] = + { + {&stats->crc_errors, "crc_errors", "CRC Errors"}, + {&stats->illegal_bytes, "illegal_bytes", "Illegal Byte Errors"}, + {&stats->mac_local_faults, "local_faults", "MAC Local Faults"}, + {&stats->mac_remote_faults, "remote_faults", "MAC Remote Faults"}, + {&stats->rx_length_errors, "rx_length_errors", "Receive Length Errors"}, + /* Packet Reception Stats */ + {&stats->rx_size_64, "rx_frames_64", "64 byte frames received"}, + {&stats->rx_size_127, "rx_frames_65_127", "65-127 byte frames received"}, + {&stats->rx_size_255, "rx_frames_128_255", "128-255 byte frames received"}, + {&stats->rx_size_511, "rx_frames_256_511", "256-511 byte frames received"}, + {&stats->rx_size_1023, "rx_frames_512_1023", "512-1023 byte frames received"}, + {&stats->rx_size_1522, "rx_frames_1024_1522", "1024-1522 byte frames received"}, + {&stats->rx_size_big, "rx_frames_big", "1523-9522 byte frames received"}, + {&stats->rx_undersize, "rx_undersize", "Undersized packets received"}, + {&stats->rx_fragments, "rx_fragmented", "Fragmented packets received"}, + {&stats->rx_oversize, "rx_oversized", "Oversized packets received"}, + {&stats->rx_jabber, "rx_jabber", "Received Jabber"}, + {&stats->checksum_error, "checksum_errors", "Checksum Errors"}, + /* Packet Transmission Stats */ + {&stats->tx_size_64, "tx_frames_64", "64 byte frames transmitted"}, + {&stats->tx_size_127, "tx_frames_65_127", "65-127 byte frames transmitted"}, + {&stats->tx_size_255, "tx_frames_128_255", "128-255 byte frames transmitted"}, + {&stats->tx_size_511, "tx_frames_256_511", "256-511 byte frames transmitted"}, + {&stats->tx_size_1023, "tx_frames_512_1023", "512-1023 byte frames transmitted"}, + {&stats->tx_size_1522, "tx_frames_1024_1522", "1024-1522 byte frames transmitted"}, + {&stats->tx_size_big, "tx_frames_big", "1523-9522 byte frames transmitted"}, + /* Flow control */ + {&stats->link_xon_tx, "xon_txd", "Link XON transmitted"}, + {&stats->link_xon_rx, "xon_recvd", "Link XON received"}, + {&stats->link_xoff_tx, "xoff_txd", "Link XOFF transmitted"}, + {&stats->link_xoff_rx, "xoff_recvd", "Link XOFF received"}, + /* End */ + {0,0,0} + }; + + struct ixl_sysctl_info *entry = ctls; + while (entry->stat != NULL) + { + SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, entry->name, + CTLFLAG_RD, entry->stat, + entry->description); + entry++; + } +} +/* +** ixl_config_rss - setup RSS +** - note this is done for the single vsi +*/ +static void ixl_config_rss(struct ixl_vsi *vsi) +{ + struct ixl_pf *pf = (struct ixl_pf *)vsi->back; + struct i40e_hw *hw = vsi->hw; + u32 lut = 0; + u32 qf_ctl; + u64 set_hena = 0, hena; + int i, j, que_id; +#ifdef RSS + u32 rss_hash_config; + u32 rss_seed[IXL_KEYSZ]; +#else + u32 rss_seed[IXL_KEYSZ] = {0x41b01687, + 0x183cfd8c, 0xce880440, 0x580cbc3c, + 0x35897377, 0x328b25e1, 0x4fa98922, + 0xb7d90c14, 0xd5bad70d, 0xcd15a2c1}; +#endif + +#ifdef RSS + /* Fetch the configured RSS key */ + rss_getkey((uint8_t *) &rss_seed); +#endif + + /* Fill out hash function seed */ + for (i = 0; i < IXL_KEYSZ; i++) + wr32(hw, I40E_PFQF_HKEY(i), rss_seed[i]); + + /* Enable PCTYPES for RSS: */ +#ifdef RSS + rss_hash_config = rss_gethashconfig(); + if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4) + set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER); + if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4) + set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP); + if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4) + set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP); + if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6) + set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER); + if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX) + set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6); + if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6) + set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP); + if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6) + set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP); +#else + set_hena = + ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP) | + ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP) | + ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) | + ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) | + ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV4) | + ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP) | + ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP) | + ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) | + ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) | + ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6) | + ((u64)1 << I40E_FILTER_PCTYPE_L2_PAYLOAD); +#endif + hena = (u64)rd32(hw, I40E_PFQF_HENA(0)) | + ((u64)rd32(hw, I40E_PFQF_HENA(1)) << 32); + hena |= set_hena; + wr32(hw, I40E_PFQF_HENA(0), (u32)hena); + wr32(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32)); + + /* configure the RSS table size */ + qf_ctl = rd32(hw, I40E_PFQF_CTL_0); + if (pf->hw.func_caps.rss_table_size == 512) + qf_ctl |= I40E_PFQF_CTL_0_HASHLUTSIZE_512; + else + qf_ctl &= ~I40E_PFQF_CTL_0_HASHLUTSIZE_512; + wr32(hw, I40E_PFQF_CTL_0, qf_ctl); + + /* Populate the LUT with max no. of queues in round robin fashion */ + for (i = j = 0; i < pf->hw.func_caps.rss_table_size; i++, j++) { + if (j == vsi->num_rx_queues) + j = 0; +#ifdef RSS + /* + * Fetch the RSS bucket id for the given indirection entry. + * Cap it at the number of configured buckets (which is + * num_rx_queues.) + */ + que_id = rss_get_indirection_to_bucket(i); + que_id = que_id % vsi->num_rx_queues; +#else + que_id = j; +#endif + /* lut = 4-byte sliding window of 4 lut entries */ + lut = (lut << 8) | (que_id & + ((0x1 << pf->hw.func_caps.rss_table_entry_width) - 1)); + /* On i = 3, we have 4 entries in lut; write to the register */ + if ((i & 3) == 3) + wr32(hw, I40E_PFQF_HLUT(i >> 2), lut); + } + ixl_flush(hw); +} + +/* +** This routine updates vlan filters, called by init +** it scans the filter table and then updates the hw +** after a soft reset. +*/ +static void +ixl_setup_vlan_filters(struct ixl_vsi *vsi) +{ + struct ixl_mac_filter *f; + int cnt = 0, flags; + + if (vsi->num_vlans == 0) + return; + /* + ** Scan the filter list for vlan entries, + ** mark them for addition and then call + ** for the AQ update. + */ + SLIST_FOREACH(f, &vsi->ftl, next) { + if (f->flags & IXL_FILTER_VLAN) { + f->flags |= + (IXL_FILTER_ADD | + IXL_FILTER_USED); + cnt++; + } + } + if (cnt == 0) { + printf("setup vlan: no filters found!\n"); + return; + } + flags = IXL_FILTER_VLAN; + flags |= (IXL_FILTER_ADD | IXL_FILTER_USED); + ixl_add_hw_filters(vsi, flags, cnt); +} + +/* +** Initialize filter list and add filters that the hardware +** needs to know about. +*/ +static void +ixl_init_filters(struct ixl_vsi *vsi) +{ + /* Add broadcast address */ + ixl_add_filter(vsi, ixl_bcast_addr, IXL_VLAN_ANY); +} + +/* +** This routine adds mulicast filters +*/ +static void +ixl_add_mc_filter(struct ixl_vsi *vsi, u8 *macaddr) +{ + struct ixl_mac_filter *f; + + /* Does one already exist */ + f = ixl_find_filter(vsi, macaddr, IXL_VLAN_ANY); + if (f != NULL) + return; + + f = ixl_get_filter(vsi); + if (f == NULL) { + printf("WARNING: no filter available!!\n"); + return; + } + bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN); + f->vlan = IXL_VLAN_ANY; + f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED + | IXL_FILTER_MC); +} + +static void +ixl_reconfigure_filters(struct ixl_vsi *vsi) +{ + + ixl_add_hw_filters(vsi, IXL_FILTER_USED, vsi->num_macs); +} + +/* +** This routine is run via an vlan config EVENT, +** it enables us to use the HW Filter table since +** we can get the vlan id. This just creates the +** entry in the soft version of the VFTA, init will +** repopulate the real table. +*/ +void +ixl_if_vlan_register(if_ctx_t ctx, u16 vtag) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + struct i40e_hw *hw = vsi->hw; + + if ((vtag == 0) || (vtag > 4095)) /* Invalid */ + return; + + ++vsi->num_vlans; + ixl_add_filter(vsi, hw->mac.addr, vtag); +} + +/* +** This routine is run via an vlan +** unconfig EVENT, remove our entry +** in the soft vfta. +*/ +void +ixl_if_vlan_unregister(if_ctx_t ctx, u16 vtag) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + struct i40e_hw *hw = vsi->hw; + + + if ((vtag == 0) || (vtag > 4095)) /* Invalid */ + return; + + --vsi->num_vlans; + ixl_del_filter(vsi, hw->mac.addr, vtag); +} + +/* +** This routine adds macvlan filters +*/ +void +ixl_add_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan) +{ + struct ixl_mac_filter *f, *tmp; + + DEBUGOUT("ixl_add_filter: begin"); + + /* Does one already exist */ + f = ixl_find_filter(vsi, macaddr, vlan); + if (f != NULL) + return; + /* + ** Is this the first vlan being registered, if so we + ** need to remove the ANY filter that indicates we are + ** not in a vlan, and replace that with a 0 filter. + */ + if ((vlan != IXL_VLAN_ANY) && (vsi->num_vlans == 1)) { + tmp = ixl_find_filter(vsi, macaddr, IXL_VLAN_ANY); + if (tmp != NULL) { + ixl_del_filter(vsi, macaddr, IXL_VLAN_ANY); + ixl_add_filter(vsi, macaddr, 0); + } + } + + f = ixl_get_filter(vsi); + if (f == NULL) { + device_printf(iflib_get_dev(vsi->ctx), "WARNING: no filter available!!\n"); + return; + } + bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN); + f->vlan = vlan; + f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED); + if (f->vlan != IXL_VLAN_ANY) + f->flags |= IXL_FILTER_VLAN; + else + vsi->num_macs++; + + ixl_add_hw_filters(vsi, f->flags, 1); + return; +} + +void +ixl_del_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan) +{ + struct ixl_mac_filter *f; + + f = ixl_find_filter(vsi, macaddr, vlan); + if (f == NULL) + return; + + f->flags |= IXL_FILTER_DEL; + ixl_del_hw_filters(vsi, 1); + vsi->num_macs--; + + /* Check if this is the last vlan removal */ + if (vlan != IXL_VLAN_ANY && vsi->num_vlans == 0) { + /* Switch back to a non-vlan filter */ + ixl_del_filter(vsi, macaddr, 0); + ixl_add_filter(vsi, macaddr, IXL_VLAN_ANY); + } + return; +} + +/* +** Find the filter with both matching mac addr and vlan id +*/ +static struct ixl_mac_filter * +ixl_find_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan) +{ + struct ixl_mac_filter *f; + bool match = FALSE; + + SLIST_FOREACH(f, &vsi->ftl, next) { + if (!cmp_etheraddr(f->macaddr, macaddr)) + continue; + if (f->vlan == vlan) { + match = TRUE; + break; + } + } + + if (!match) + f = NULL; + return (f); +} + +/* +** This routine takes additions to the vsi filter +** table and creates an Admin Queue call to create +** the filters in the hardware. +*/ +static void +ixl_add_hw_filters(struct ixl_vsi *vsi, int flags, int cnt) +{ + struct i40e_aqc_add_macvlan_element_data *a, *b; + struct ixl_mac_filter *f; + struct i40e_hw *hw = vsi->hw; + device_t dev = iflib_get_dev(vsi->ctx); + int err, j = 0; + + a = malloc(sizeof(struct i40e_aqc_add_macvlan_element_data) * cnt, + M_IXL, M_NOWAIT | M_ZERO); + if (a == NULL) { + device_printf(dev, "add_hw_filters failed to get memory\n"); + return; + } + + /* + ** Scan the filter list, each time we find one + ** we add it to the admin queue array and turn off + ** the add bit. + */ + SLIST_FOREACH(f, &vsi->ftl, next) { + if (f->flags == flags) { + b = &a[j]; // a pox on fvl long names :) + bcopy(f->macaddr, b->mac_addr, ETHER_ADDR_LEN); + if (f->vlan == IXL_VLAN_ANY) { + b->vlan_tag = 0; + b->flags = I40E_AQC_MACVLAN_ADD_IGNORE_VLAN; + } else { + b->vlan_tag = f->vlan; + b->flags = 0; + } + b->flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH; + f->flags &= ~IXL_FILTER_ADD; + j++; + } + if (j == cnt) + break; + } + if (j > 0) { + err = i40e_aq_add_macvlan(hw, vsi->seid, a, j, NULL); + if (err) + device_printf(dev, "aq_add_macvlan err %d, " + "aq_error %d\n", err, hw->aq.asq_last_status); + else + vsi->hw_filters_add += j; + } + free(a, M_IXL); + return; +} + +/* +** This routine takes removals in the vsi filter +** table and creates an Admin Queue call to delete +** the filters in the hardware. +*/ +static void +ixl_del_hw_filters(struct ixl_vsi *vsi, int cnt) +{ + struct i40e_aqc_remove_macvlan_element_data *d, *e; + struct i40e_hw *hw = vsi->hw; + device_t dev = iflib_get_dev(vsi->ctx); + struct ixl_mac_filter *f, *f_temp; + int err, j = 0; + + DEBUGOUT("ixl_del_hw_filters: begin\n"); + + d = malloc(sizeof(struct i40e_aqc_remove_macvlan_element_data) * cnt, + M_IXL, M_NOWAIT | M_ZERO); + if (d == NULL) { + printf("del hw filter failed to get memory\n"); + return; + } + + SLIST_FOREACH_SAFE(f, &vsi->ftl, next, f_temp) { + if (f->flags & IXL_FILTER_DEL) { + e = &d[j]; // a pox on fvl long names :) + bcopy(f->macaddr, e->mac_addr, ETHER_ADDR_LEN); + e->vlan_tag = (f->vlan == IXL_VLAN_ANY ? 0 : f->vlan); + e->flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; + /* delete entry from vsi list */ + SLIST_REMOVE(&vsi->ftl, f, ixl_mac_filter, next); + free(f, M_IXL); + j++; + } + if (j == cnt) + break; + } + if (j > 0) { + err = i40e_aq_remove_macvlan(hw, vsi->seid, d, j, NULL); + /* NOTE: returns ENOENT every time but seems to work fine, + so we'll ignore that specific error. */ + // TODO: Does this still occur on current firmwares? + if (err && hw->aq.asq_last_status != I40E_AQ_RC_ENOENT) { + int sc = 0; + for (int i = 0; i < j; i++) + sc += (!d[i].error_code); + vsi->hw_filters_del += sc; + device_printf(dev, + "Failed to remove %d/%d filters, aq error %d\n", + j - sc, j, hw->aq.asq_last_status); + } else + vsi->hw_filters_del += j; + } + free(d, M_IXL); + + DEBUGOUT("ixl_del_hw_filters: end\n"); + return; +} + +static int +ixl_enable_rings(struct ixl_vsi *vsi) +{ + struct ixl_pf *pf = vsi->back; + struct i40e_hw *hw = &pf->hw; + int index, error; + u32 reg; + + error = 0; + for (int i = 0; i < vsi->num_rx_queues; i++) { + index = vsi->first_queue + i; + i40e_pre_tx_queue_cfg(hw, index, TRUE); + + reg = rd32(hw, I40E_QTX_ENA(index)); + reg |= I40E_QTX_ENA_QENA_REQ_MASK | + I40E_QTX_ENA_QENA_STAT_MASK; + wr32(hw, I40E_QTX_ENA(index), reg); + /* Verify the enable took */ + for (int j = 0; j < 10; j++) { + reg = rd32(hw, I40E_QTX_ENA(index)); + if (reg & I40E_QTX_ENA_QENA_STAT_MASK) + break; + i40e_msec_delay(10); + } + if ((reg & I40E_QTX_ENA_QENA_STAT_MASK) == 0) { + device_printf(iflib_get_dev(vsi->ctx), "TX queue %d disabled!\n", + index); + error = ETIMEDOUT; + } + + reg = rd32(hw, I40E_QRX_ENA(index)); + reg |= I40E_QRX_ENA_QENA_REQ_MASK | + I40E_QRX_ENA_QENA_STAT_MASK; + wr32(hw, I40E_QRX_ENA(index), reg); + /* Verify the enable took */ + for (int j = 0; j < 10; j++) { + reg = rd32(hw, I40E_QRX_ENA(index)); + if (reg & I40E_QRX_ENA_QENA_STAT_MASK) + break; + i40e_msec_delay(10); + } + if ((reg & I40E_QRX_ENA_QENA_STAT_MASK) == 0) { + device_printf(iflib_get_dev(vsi->ctx), "RX queue %d disabled!\n", + index); + error = ETIMEDOUT; + } + } + + return (error); +} + +static void +ixl_if_intr_enable(if_ctx_t ctx) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + struct i40e_hw *hw = vsi->hw; + + if (ixl_enable_msix) { + ixl_enable_adminq(hw); + } else + ixl_enable_legacy(hw); +} + +static void +ixl_if_intr_disable(if_ctx_t ctx) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + struct i40e_hw *hw = vsi->hw; + + if (ixl_enable_msix) { + ixl_disable_adminq(hw); + } else + ixl_disable_legacy(hw); +} + +static void +ixl_if_queue_intr_enable(if_ctx_t ctx, uint16_t qid) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + u32 reg; + + reg = I40E_PFINT_DYN_CTLN_INTENA_MASK | + I40E_PFINT_DYN_CTLN_CLEARPBA_MASK | + (IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT); + wr32(vsi->hw, I40E_PFINT_DYN_CTLN(qid), reg); +} + +#if 0 +static void +ixl_if_queue_intr_disable(if_ctx_t ctx, uint16_t qid) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + struct i40e_hw *hw = vsi->hw; + u32 reg; + + reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT; + wr32(hw, I40E_PFINT_DYN_CTLN(qid), reg); +} +#endif + +static int +ixl_disable_rings(struct ixl_vsi *vsi) +{ + struct ixl_pf *pf = vsi->back; + struct i40e_hw *hw = &pf->hw; + int index, error; + u32 reg; + + error = 0; + for (int i = 0; i < vsi->num_rx_queues; i++) { + index = vsi->first_queue + i; + + i40e_pre_tx_queue_cfg(hw, index, FALSE); + i40e_usec_delay(500); + + reg = rd32(hw, I40E_QTX_ENA(index)); + reg &= ~I40E_QTX_ENA_QENA_REQ_MASK; + wr32(hw, I40E_QTX_ENA(index), reg); + /* Verify the disable took */ + for (int j = 0; j < 10; j++) { + reg = rd32(hw, I40E_QTX_ENA(index)); + if (!(reg & I40E_QTX_ENA_QENA_STAT_MASK)) + break; + i40e_msec_delay(10); + } + if (reg & I40E_QTX_ENA_QENA_STAT_MASK) { + device_printf(iflib_get_dev(vsi->ctx), "TX queue %d still enabled!\n", + index); + error = ETIMEDOUT; + } + + reg = rd32(hw, I40E_QRX_ENA(index)); + reg &= ~I40E_QRX_ENA_QENA_REQ_MASK; + wr32(hw, I40E_QRX_ENA(index), reg); + /* Verify the disable took */ + for (int j = 0; j < 10; j++) { + reg = rd32(hw, I40E_QRX_ENA(index)); + if (!(reg & I40E_QRX_ENA_QENA_STAT_MASK)) + break; + i40e_msec_delay(10); + } + if (reg & I40E_QRX_ENA_QENA_STAT_MASK) { + device_printf(iflib_get_dev(vsi->ctx), "RX queue %d still enabled!\n", + index); + error = ETIMEDOUT; + } + } + + return (error); +} + +/** + * ixl_handle_mdd_event + * + * Called from interrupt handler to identify possibly malicious vfs + * (But also detects events from the PF, as well) + **/ +static void ixl_handle_mdd_event(struct ixl_pf *pf) +{ + struct i40e_hw *hw = &pf->hw; + device_t dev = iflib_get_dev(pf->vsi.ctx); + bool mdd_detected = false; + bool pf_mdd_detected = false; + u32 reg; + + /* find what triggered the MDD event */ + reg = rd32(hw, I40E_GL_MDET_TX); + if (reg & I40E_GL_MDET_TX_VALID_MASK) { + u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >> + I40E_GL_MDET_TX_PF_NUM_SHIFT; + u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >> + I40E_GL_MDET_TX_EVENT_SHIFT; + u8 queue = (reg & I40E_GL_MDET_TX_QUEUE_MASK) >> + I40E_GL_MDET_TX_QUEUE_SHIFT; + device_printf(dev, + "Malicious Driver Detection event 0x%02x" + " on TX queue %d pf number 0x%02x\n", + event, queue, pf_num); + wr32(hw, I40E_GL_MDET_TX, 0xffffffff); + mdd_detected = true; + } + reg = rd32(hw, I40E_GL_MDET_RX); + if (reg & I40E_GL_MDET_RX_VALID_MASK) { + u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >> + I40E_GL_MDET_RX_FUNCTION_SHIFT; + u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >> + I40E_GL_MDET_RX_EVENT_SHIFT; + u8 queue = (reg & I40E_GL_MDET_RX_QUEUE_MASK) >> + I40E_GL_MDET_RX_QUEUE_SHIFT; + device_printf(dev, + "Malicious Driver Detection event 0x%02x" + " on RX queue %d of function 0x%02x\n", + event, queue, func); + wr32(hw, I40E_GL_MDET_RX, 0xffffffff); + mdd_detected = true; + } + + if (mdd_detected) { + reg = rd32(hw, I40E_PF_MDET_TX); + if (reg & I40E_PF_MDET_TX_VALID_MASK) { + wr32(hw, I40E_PF_MDET_TX, 0xFFFF); + device_printf(dev, + "MDD TX event is for this function 0x%08x", + reg); + pf_mdd_detected = true; + } + reg = rd32(hw, I40E_PF_MDET_RX); + if (reg & I40E_PF_MDET_RX_VALID_MASK) { + wr32(hw, I40E_PF_MDET_RX, 0xFFFF); + device_printf(dev, + "MDD RX event is for this function 0x%08x", + reg); + pf_mdd_detected = true; + } + } + + /* re-enable mdd interrupt cause */ + reg = rd32(hw, I40E_PFINT_ICR0_ENA); + reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; + wr32(hw, I40E_PFINT_ICR0_ENA, reg); + ixl_flush(hw); +} + +static void +ixl_enable_intr(struct ixl_vsi *vsi) +{ + struct i40e_hw *hw = vsi->hw; + struct ixl_rx_queue *que = vsi->rx_queues; + + if (ixl_enable_msix) { + ixl_enable_adminq(hw); + for (int i = 0; i < vsi->num_rx_queues; i++, que++) + ixl_if_queue_intr_enable(vsi->ctx, que->rxr.me); + } else + ixl_enable_legacy(hw); +} + +static void +ixl_enable_adminq(struct i40e_hw *hw) +{ + u32 reg; + + reg = I40E_PFINT_DYN_CTL0_INTENA_MASK | + I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | + (IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT); + wr32(hw, I40E_PFINT_DYN_CTL0, reg); + ixl_flush(hw); + return; +} + +static void +ixl_disable_adminq(struct i40e_hw *hw) +{ + u32 reg; + + reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT; + wr32(hw, I40E_PFINT_DYN_CTL0, reg); + + return; +} + +static void +ixl_enable_legacy(struct i40e_hw *hw) +{ + u32 reg; + reg = I40E_PFINT_DYN_CTL0_INTENA_MASK | + I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | + (IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT); + wr32(hw, I40E_PFINT_DYN_CTL0, reg); +} + +static void +ixl_disable_legacy(struct i40e_hw *hw) +{ + u32 reg; + + reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT; + wr32(hw, I40E_PFINT_DYN_CTL0, reg); + + return; +} + +static void +ixl_update_stats_counters(struct ixl_pf *pf) +{ + struct i40e_hw *hw = &pf->hw; + struct ixl_vsi *vsi = &pf->vsi; + struct ixl_vf *vf; + + struct i40e_hw_port_stats *nsd = &pf->stats; + struct i40e_hw_port_stats *osd = &pf->stats_offsets; + + /* Update hw stats */ + ixl_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port), + pf->stat_offsets_loaded, + &osd->crc_errors, &nsd->crc_errors); + ixl_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port), + pf->stat_offsets_loaded, + &osd->illegal_bytes, &nsd->illegal_bytes); + ixl_stat_update48(hw, I40E_GLPRT_GORCH(hw->port), + I40E_GLPRT_GORCL(hw->port), + pf->stat_offsets_loaded, + &osd->eth.rx_bytes, &nsd->eth.rx_bytes); + ixl_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port), + I40E_GLPRT_GOTCL(hw->port), + pf->stat_offsets_loaded, + &osd->eth.tx_bytes, &nsd->eth.tx_bytes); + ixl_stat_update32(hw, I40E_GLPRT_RDPC(hw->port), + pf->stat_offsets_loaded, + &osd->eth.rx_discards, + &nsd->eth.rx_discards); + ixl_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port), + I40E_GLPRT_UPRCL(hw->port), + pf->stat_offsets_loaded, + &osd->eth.rx_unicast, + &nsd->eth.rx_unicast); + ixl_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port), + I40E_GLPRT_UPTCL(hw->port), + pf->stat_offsets_loaded, + &osd->eth.tx_unicast, + &nsd->eth.tx_unicast); + ixl_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port), + I40E_GLPRT_MPRCL(hw->port), + pf->stat_offsets_loaded, + &osd->eth.rx_multicast, + &nsd->eth.rx_multicast); + ixl_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port), + I40E_GLPRT_MPTCL(hw->port), + pf->stat_offsets_loaded, + &osd->eth.tx_multicast, + &nsd->eth.tx_multicast); + ixl_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port), + I40E_GLPRT_BPRCL(hw->port), + pf->stat_offsets_loaded, + &osd->eth.rx_broadcast, + &nsd->eth.rx_broadcast); + ixl_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port), + I40E_GLPRT_BPTCL(hw->port), + pf->stat_offsets_loaded, + &osd->eth.tx_broadcast, + &nsd->eth.tx_broadcast); + + ixl_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port), + pf->stat_offsets_loaded, + &osd->tx_dropped_link_down, + &nsd->tx_dropped_link_down); + ixl_stat_update32(hw, I40E_GLPRT_MLFC(hw->port), + pf->stat_offsets_loaded, + &osd->mac_local_faults, + &nsd->mac_local_faults); + ixl_stat_update32(hw, I40E_GLPRT_MRFC(hw->port), + pf->stat_offsets_loaded, + &osd->mac_remote_faults, + &nsd->mac_remote_faults); + ixl_stat_update32(hw, I40E_GLPRT_RLEC(hw->port), + pf->stat_offsets_loaded, + &osd->rx_length_errors, + &nsd->rx_length_errors); + + /* Flow control (LFC) stats */ + ixl_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port), + pf->stat_offsets_loaded, + &osd->link_xon_rx, &nsd->link_xon_rx); + ixl_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port), + pf->stat_offsets_loaded, + &osd->link_xon_tx, &nsd->link_xon_tx); + ixl_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port), + pf->stat_offsets_loaded, + &osd->link_xoff_rx, &nsd->link_xoff_rx); + ixl_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port), + pf->stat_offsets_loaded, + &osd->link_xoff_tx, &nsd->link_xoff_tx); + + /* Packet size stats rx */ + ixl_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port), + I40E_GLPRT_PRC64L(hw->port), + pf->stat_offsets_loaded, + &osd->rx_size_64, &nsd->rx_size_64); + ixl_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port), + I40E_GLPRT_PRC127L(hw->port), + pf->stat_offsets_loaded, + &osd->rx_size_127, &nsd->rx_size_127); + ixl_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port), + I40E_GLPRT_PRC255L(hw->port), + pf->stat_offsets_loaded, + &osd->rx_size_255, &nsd->rx_size_255); + ixl_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port), + I40E_GLPRT_PRC511L(hw->port), + pf->stat_offsets_loaded, + &osd->rx_size_511, &nsd->rx_size_511); + ixl_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port), + I40E_GLPRT_PRC1023L(hw->port), + pf->stat_offsets_loaded, + &osd->rx_size_1023, &nsd->rx_size_1023); + ixl_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port), + I40E_GLPRT_PRC1522L(hw->port), + pf->stat_offsets_loaded, + &osd->rx_size_1522, &nsd->rx_size_1522); + ixl_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port), + I40E_GLPRT_PRC9522L(hw->port), + pf->stat_offsets_loaded, + &osd->rx_size_big, &nsd->rx_size_big); + + /* Packet size stats tx */ + ixl_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port), + I40E_GLPRT_PTC64L(hw->port), + pf->stat_offsets_loaded, + &osd->tx_size_64, &nsd->tx_size_64); + ixl_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port), + I40E_GLPRT_PTC127L(hw->port), + pf->stat_offsets_loaded, + &osd->tx_size_127, &nsd->tx_size_127); + ixl_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port), + I40E_GLPRT_PTC255L(hw->port), + pf->stat_offsets_loaded, + &osd->tx_size_255, &nsd->tx_size_255); + ixl_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port), + I40E_GLPRT_PTC511L(hw->port), + pf->stat_offsets_loaded, + &osd->tx_size_511, &nsd->tx_size_511); + ixl_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port), + I40E_GLPRT_PTC1023L(hw->port), + pf->stat_offsets_loaded, + &osd->tx_size_1023, &nsd->tx_size_1023); + ixl_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port), + I40E_GLPRT_PTC1522L(hw->port), + pf->stat_offsets_loaded, + &osd->tx_size_1522, &nsd->tx_size_1522); + ixl_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port), + I40E_GLPRT_PTC9522L(hw->port), + pf->stat_offsets_loaded, + &osd->tx_size_big, &nsd->tx_size_big); + + ixl_stat_update32(hw, I40E_GLPRT_RUC(hw->port), + pf->stat_offsets_loaded, + &osd->rx_undersize, &nsd->rx_undersize); + ixl_stat_update32(hw, I40E_GLPRT_RFC(hw->port), + pf->stat_offsets_loaded, + &osd->rx_fragments, &nsd->rx_fragments); + ixl_stat_update32(hw, I40E_GLPRT_ROC(hw->port), + pf->stat_offsets_loaded, + &osd->rx_oversize, &nsd->rx_oversize); + ixl_stat_update32(hw, I40E_GLPRT_RJC(hw->port), + pf->stat_offsets_loaded, + &osd->rx_jabber, &nsd->rx_jabber); + pf->stat_offsets_loaded = true; + /* End hw stats */ + + /* Update vsi stats */ + ixl_update_vsi_stats(vsi); + + for (int i = 0; i < pf->num_vfs; i++) { + vf = &pf->vfs[i]; + if (vf->vf_flags & VF_FLAG_ENABLED) + ixl_update_eth_stats(&pf->vfs[i].vsi); + } +} + +/* +** Tasklet handler for MSIX Adminq interrupts +** - do outside interrupt since it might sleep +*/ +static void +ixl_if_update_admin_status(if_ctx_t ctx) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + struct ixl_pf *pf = vsi->back; + struct i40e_hw *hw = &pf->hw; + struct i40e_arq_event_info event; + i40e_status ret; + u32 reg, loop = 0; + u16 opcode, result; + + event.buf_len = IXL_AQ_BUF_SZ; + event.msg_buf = malloc(event.buf_len, + M_IXL, M_NOWAIT | M_ZERO); + if (!event.msg_buf) { + printf("Unable to allocate adminq memory\n"); + return; + } + + /* clean and process any events */ + do { + ret = i40e_clean_arq_element(hw, &event, &result); + if (ret) + break; + opcode = LE16_TO_CPU(event.desc.opcode); + switch (opcode) { + case i40e_aqc_opc_get_link_status: + ixl_link_event(pf, &event); + ixl_update_link_status(pf); + break; + case i40e_aqc_opc_send_msg_to_pf: +#ifdef PCI_IOV + ixl_handle_vf_msg(pf, &event); +#endif + break; + case i40e_aqc_opc_event_lan_overflow: + break; + default: +#ifdef IXL_DEBUG + printf("AdminQ unknown event %x\n", opcode); +#endif + break; + } + + } while (result && (loop++ < IXL_ADM_LIMIT)); + + reg = rd32(hw, I40E_PFINT_ICR0_ENA); + reg |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK; + wr32(hw, I40E_PFINT_ICR0_ENA, reg); + free(event.msg_buf, M_IXL); + + /* + * If there are still messages to process, reschedule ourselves. + * Otherwise, re-enable our interrupt and go to sleep. + */ + if (result > 0) + iflib_admin_intr_deferred(ctx); + else + ixl_enable_intr(vsi); + +} + +#ifdef IXL_DEBUG_SYSCTL +static int +ixl_debug_info(SYSCTL_HANDLER_ARGS) +{ + struct ixl_pf *pf; + int error, input = 0; + + error = sysctl_handle_int(oidp, &input, 0, req); + + if (error || !req->newptr) + return (error); + + if (input == 1) { + pf = (struct ixl_pf *)arg1; + ixl_print_debug_info(pf); + } + + return (error); +} + +static void +ixl_print_debug_info(struct ixl_pf *pf) +{ + struct i40e_hw *hw = &pf->hw; + struct ixl_vsi *vsi = &pf->vsi; + struct ixl_rx_queue *que = vsi->rx_queues; + struct rx_ring *rxr = &que->rxr; + u32 reg; + + + printf("Queue irqs = %jx\n", (uintmax_t)que->irqs); + printf("AdminQ irqs = %jx\n", (uintmax_t)pf->admin_irq); + printf("RX packets = %jx\n", (uintmax_t)rxr->rx_packets); + + reg = rd32(hw, I40E_GLV_GORCL(0xc)); + printf("RX Bytes = %x\n", reg); + reg = rd32(hw, I40E_GLPRT_GORCL(hw->port)); + printf("Port RX Bytes = %x\n", reg); + reg = rd32(hw, I40E_GLV_RDPC(0xc)); + printf("RX discard = %x\n", reg); + reg = rd32(hw, I40E_GLPRT_RDPC(hw->port)); + printf("Port RX discard = %x\n", reg); + + reg = rd32(hw, I40E_GLV_TEPC(0xc)); + printf("TX errors = %x\n", reg); + reg = rd32(hw, I40E_GLV_GOTCL(0xc)); + printf("TX Bytes = %x\n", reg); + + reg = rd32(hw, I40E_GLPRT_RUC(hw->port)); + printf("RX undersize = %x\n", reg); + reg = rd32(hw, I40E_GLPRT_RFC(hw->port)); + printf("RX fragments = %x\n", reg); + reg = rd32(hw, I40E_GLPRT_ROC(hw->port)); + printf("RX oversize = %x\n", reg); + reg = rd32(hw, I40E_GLPRT_RLEC(hw->port)); + printf("RX length error = %x\n", reg); + reg = rd32(hw, I40E_GLPRT_MRFC(hw->port)); + printf("mac remote fault = %x\n", reg); + reg = rd32(hw, I40E_GLPRT_MLFC(hw->port)); + printf("mac local fault = %x\n", reg); +} +#endif + +/** + * Update VSI-specific ethernet statistics counters. + **/ +void ixl_update_eth_stats(struct ixl_vsi *vsi) +{ + struct ixl_pf *pf = (struct ixl_pf *)vsi->back; + struct i40e_hw *hw = &pf->hw; + struct i40e_eth_stats *es; + struct i40e_eth_stats *oes; + + struct i40e_hw_port_stats *nsd; + u16 stat_idx = vsi->info.stat_counter_idx; + + es = &vsi->eth_stats; + oes = &vsi->eth_stats_offsets; + nsd = &pf->stats; + + /* Gather up the stats that the hw collects */ + ixl_stat_update32(hw, I40E_GLV_TEPC(stat_idx), + vsi->stat_offsets_loaded, + &oes->tx_errors, &es->tx_errors); + ixl_stat_update32(hw, I40E_GLV_RDPC(stat_idx), + vsi->stat_offsets_loaded, + &oes->rx_discards, &es->rx_discards); + + ixl_stat_update48(hw, I40E_GLV_GORCH(stat_idx), + I40E_GLV_GORCL(stat_idx), + vsi->stat_offsets_loaded, + &oes->rx_bytes, &es->rx_bytes); + ixl_stat_update48(hw, I40E_GLV_UPRCH(stat_idx), + I40E_GLV_UPRCL(stat_idx), + vsi->stat_offsets_loaded, + &oes->rx_unicast, &es->rx_unicast); + ixl_stat_update48(hw, I40E_GLV_MPRCH(stat_idx), + I40E_GLV_MPRCL(stat_idx), + vsi->stat_offsets_loaded, + &oes->rx_multicast, &es->rx_multicast); + ixl_stat_update48(hw, I40E_GLV_BPRCH(stat_idx), + I40E_GLV_BPRCL(stat_idx), + vsi->stat_offsets_loaded, + &oes->rx_broadcast, &es->rx_broadcast); + + ixl_stat_update48(hw, I40E_GLV_GOTCH(stat_idx), + I40E_GLV_GOTCL(stat_idx), + vsi->stat_offsets_loaded, + &oes->tx_bytes, &es->tx_bytes); + ixl_stat_update48(hw, I40E_GLV_UPTCH(stat_idx), + I40E_GLV_UPTCL(stat_idx), + vsi->stat_offsets_loaded, + &oes->tx_unicast, &es->tx_unicast); + ixl_stat_update48(hw, I40E_GLV_MPTCH(stat_idx), + I40E_GLV_MPTCL(stat_idx), + vsi->stat_offsets_loaded, + &oes->tx_multicast, &es->tx_multicast); + ixl_stat_update48(hw, I40E_GLV_BPTCH(stat_idx), + I40E_GLV_BPTCL(stat_idx), + vsi->stat_offsets_loaded, + &oes->tx_broadcast, &es->tx_broadcast); + vsi->stat_offsets_loaded = true; +} + +static void +ixl_update_vsi_stats(struct ixl_vsi *vsi) +{ + struct ixl_pf *pf; + struct i40e_eth_stats *es; + u64 tx_discards; + + struct i40e_hw_port_stats *nsd; + + pf = vsi->back; + es = &vsi->eth_stats; + nsd = &pf->stats; + + ixl_update_eth_stats(vsi); + + tx_discards = es->tx_discards + nsd->tx_dropped_link_down; + + /* Update ifnet stats */ + IXL_SET_IPACKETS(vsi, es->rx_unicast + + es->rx_multicast + + es->rx_broadcast); + IXL_SET_OPACKETS(vsi, es->tx_unicast + + es->tx_multicast + + es->tx_broadcast); + IXL_SET_IBYTES(vsi, es->rx_bytes); + IXL_SET_OBYTES(vsi, es->tx_bytes); + IXL_SET_IMCASTS(vsi, es->rx_multicast); + IXL_SET_OMCASTS(vsi, es->tx_multicast); + + IXL_SET_IERRORS(vsi, nsd->crc_errors + nsd->illegal_bytes + + nsd->rx_undersize + nsd->rx_oversize + nsd->rx_fragments + + nsd->rx_jabber); + IXL_SET_OERRORS(vsi, es->tx_errors); + IXL_SET_IQDROPS(vsi, es->rx_discards + nsd->eth.rx_discards); + IXL_SET_OQDROPS(vsi, tx_discards); + IXL_SET_NOPROTO(vsi, es->rx_unknown_protocol); + IXL_SET_COLLISIONS(vsi, 0); +} + +/** + * Reset all of the stats for the given pf + **/ +void ixl_pf_reset_stats(struct ixl_pf *pf) +{ + bzero(&pf->stats, sizeof(struct i40e_hw_port_stats)); + bzero(&pf->stats_offsets, sizeof(struct i40e_hw_port_stats)); + pf->stat_offsets_loaded = false; +} + +/** + * Resets all stats of the given vsi + **/ +void ixl_vsi_reset_stats(struct ixl_vsi *vsi) +{ + bzero(&vsi->eth_stats, sizeof(struct i40e_eth_stats)); + bzero(&vsi->eth_stats_offsets, sizeof(struct i40e_eth_stats)); + vsi->stat_offsets_loaded = false; +} + +/** + * Read and update a 48 bit stat from the hw + * + * Since the device stats are not reset at PFReset, they likely will not + * be zeroed when the driver starts. We'll save the first values read + * and use them as offsets to be subtracted from the raw values in order + * to report stats that count from zero. + **/ +static void +ixl_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg, + bool offset_loaded, u64 *offset, u64 *stat) +{ + u64 new_data; + +#if defined(__FreeBSD__) && (__FreeBSD_version >= 1000000) && defined(__amd64__) + new_data = rd64(hw, loreg); +#else + /* + * Use two rd32's instead of one rd64; FreeBSD versions before + * 10 don't support 8 byte bus reads/writes. + */ + new_data = rd32(hw, loreg); + new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32; +#endif + + if (!offset_loaded) + *offset = new_data; + if (new_data >= *offset) + *stat = new_data - *offset; + else + *stat = (new_data + ((u64)1 << 48)) - *offset; + *stat &= 0xFFFFFFFFFFFFULL; +} + +/** + * Read and update a 32 bit stat from the hw + **/ +static void +ixl_stat_update32(struct i40e_hw *hw, u32 reg, + bool offset_loaded, u64 *offset, u64 *stat) +{ + u32 new_data; + + new_data = rd32(hw, reg); + if (!offset_loaded) + *offset = new_data; + if (new_data >= *offset) + *stat = (u32)(new_data - *offset); + else + *stat = (u32)((new_data + ((u64)1 << 32)) - *offset); +} + +/* +** Set flow control using sysctl: +** 0 - off +** 1 - rx pause +** 2 - tx pause +** 3 - full +*/ +static int +ixl_set_flowcntl(SYSCTL_HANDLER_ARGS) +{ + /* + * TODO: ensure flow control is disabled if + * priority flow control is enabled + * + * TODO: ensure tx CRC by hardware should be enabled + * if tx flow control is enabled. + */ + struct ixl_pf *pf = (struct ixl_pf *)arg1; + struct i40e_hw *hw = &pf->hw; + device_t dev = iflib_get_dev(pf->vsi.ctx); + int error = 0; + enum i40e_status_code aq_error = 0; + u8 fc_aq_err = 0; + + /* Get request */ + error = sysctl_handle_int(oidp, &pf->fc, 0, req); + if ((error) || (req->newptr == NULL)) + return (error); + if (pf->fc < 0 || pf->fc > 3) { + device_printf(dev, + "Invalid fc mode; valid modes are 0 through 3\n"); + return (EINVAL); + } + + /* + ** Changing flow control mode currently does not work on + ** 40GBASE-CR4 PHYs + */ + if (hw->phy.link_info.phy_type == I40E_PHY_TYPE_40GBASE_CR4 + || hw->phy.link_info.phy_type == I40E_PHY_TYPE_40GBASE_CR4_CU) { + device_printf(dev, "Changing flow control mode unsupported" + " on 40GBase-CR4 media.\n"); + return (ENODEV); + } + + /* Set fc ability for port */ + hw->fc.requested_mode = pf->fc; + aq_error = i40e_set_fc(hw, &fc_aq_err, TRUE); + if (aq_error) { + device_printf(dev, + "%s: Error setting new fc mode %d; fc_err %#x\n", + __func__, aq_error, fc_aq_err); + return (EAGAIN); + } + + return (0); +} + +static int +ixl_current_speed(SYSCTL_HANDLER_ARGS) +{ + struct ixl_pf *pf = (struct ixl_pf *)arg1; + struct i40e_hw *hw = &pf->hw; + int error = 0, index = 0; + + char *speeds[] = { + "Unknown", + "100M", + "1G", + "10G", + "40G", + "20G" + }; + + ixl_update_link_status(pf); + + switch (hw->phy.link_info.link_speed) { + case I40E_LINK_SPEED_100MB: + index = 1; + break; + case I40E_LINK_SPEED_1GB: + index = 2; + break; + case I40E_LINK_SPEED_10GB: + index = 3; + break; + case I40E_LINK_SPEED_40GB: + index = 4; + break; + case I40E_LINK_SPEED_20GB: + index = 5; + break; + case I40E_LINK_SPEED_UNKNOWN: + default: + index = 0; + break; + } + + error = sysctl_handle_string(oidp, speeds[index], + strlen(speeds[index]), req); + return (error); +} + +static int +ixl_set_advertised_speeds(struct ixl_pf *pf, int speeds) +{ + struct i40e_hw *hw = &pf->hw; + if_ctx_t ctx = ((struct ixl_vsi *)pf)->ctx; + device_t dev = iflib_get_dev(ctx); + struct ifnet *ifp = iflib_get_ifp(ctx); + + struct i40e_aq_get_phy_abilities_resp abilities; + struct i40e_aq_set_phy_config config; + enum i40e_status_code aq_error = 0; + + /* Get current capability information */ + aq_error = i40e_aq_get_phy_capabilities(hw, + FALSE, FALSE, &abilities, NULL); + if (aq_error) { + device_printf(dev, + "%s: Error getting phy capabilities %d," + " aq error: %d\n", __func__, aq_error, + hw->aq.asq_last_status); + return (EAGAIN); + } + + /* Prepare new config */ + bzero(&config, sizeof(config)); + config.phy_type = abilities.phy_type; + config.abilities = abilities.abilities + | I40E_AQ_PHY_ENABLE_ATOMIC_LINK; + config.eee_capability = abilities.eee_capability; + config.eeer = abilities.eeer_val; + config.low_power_ctrl = abilities.d3_lpan; + /* Translate into aq cmd link_speed */ + if (speeds & 0x8) + config.link_speed |= I40E_LINK_SPEED_20GB; + if (speeds & 0x4) + config.link_speed |= I40E_LINK_SPEED_10GB; + if (speeds & 0x2) + config.link_speed |= I40E_LINK_SPEED_1GB; + if (speeds & 0x1) + config.link_speed |= I40E_LINK_SPEED_100MB; + + /* Do aq command & restart link */ + aq_error = i40e_aq_set_phy_config(hw, &config, NULL); + if (aq_error) { + device_printf(dev, + "%s: Error setting new phy config %d," + " aq error: %d\n", __func__, aq_error, + hw->aq.asq_last_status); + return (EAGAIN); + } + + /* + ** This seems a bit heavy handed, but we + ** need to get a reinit on some devices + */ + ifp->if_init(ifp->if_softc); + return (0); +} + +/* +** Control link advertise speed: +** Flags: +** 0x1 - advertise 100 Mb +** 0x2 - advertise 1G +** 0x4 - advertise 10G +** 0x8 - advertise 20G +** +** Does not work on 40G devices. +*/ +static int +ixl_set_advertise(SYSCTL_HANDLER_ARGS) +{ + struct ixl_pf *pf = (struct ixl_pf *)arg1; + struct i40e_hw *hw = &pf->hw; + device_t dev = iflib_get_dev(pf->vsi.ctx); + int requested_ls = 0; + int error = 0; + + /* + ** FW doesn't support changing advertised speed + ** for 40G devices; speed is always 40G. + */ + if (i40e_is_40G_device(hw->device_id)) + return (ENODEV); + + /* Read in new mode */ + requested_ls = pf->advertised_speed; + error = sysctl_handle_int(oidp, &requested_ls, 0, req); + if ((error) || (req->newptr == NULL)) + return (error); + /* Check for sane value */ + if (requested_ls < 0x1 || requested_ls > 0xE) { + device_printf(dev, "Invalid advertised speed; " + "valid modes are 0x1 through 0xE\n"); + return (EINVAL); + } + /* Then check for validity based on adapter type */ + switch (hw->device_id) { + case I40E_DEV_ID_10G_BASE_T: + if (requested_ls & 0x8) { + device_printf(dev, + "20Gbs speed not supported on this device.\n"); + return (EINVAL); + } + break; + case I40E_DEV_ID_20G_KR2: + if (requested_ls & 0x1) { + device_printf(dev, + "100Mbs speed not supported on this device.\n"); + return (EINVAL); + } + break; + default: + if (requested_ls & ~0x6) { + device_printf(dev, + "Only 1/10Gbs speeds are supported on this device.\n"); + return (EINVAL); + } + break; + } + + /* Exit if no change */ + if (pf->advertised_speed == requested_ls) + return (0); + + error = ixl_set_advertised_speeds(pf, requested_ls); + if (error) + return (error); + + pf->advertised_speed = requested_ls; + ixl_update_link_status(pf); + return (0); +} + +/* +** Get the width and transaction speed of +** the bus this adapter is plugged into. +*/ +static u16 +ixl_get_bus_info(struct i40e_hw *hw, device_t dev) +{ + u16 link; + u32 offset; + + + /* Get the PCI Express Capabilities offset */ + pci_find_cap(dev, PCIY_EXPRESS, &offset); + + /* ...and read the Link Status Register */ + link = pci_read_config(dev, offset + PCIER_LINK_STA, 2); + + switch (link & I40E_PCI_LINK_WIDTH) { + case I40E_PCI_LINK_WIDTH_1: + hw->bus.width = i40e_bus_width_pcie_x1; + break; + case I40E_PCI_LINK_WIDTH_2: + hw->bus.width = i40e_bus_width_pcie_x2; + break; + case I40E_PCI_LINK_WIDTH_4: + hw->bus.width = i40e_bus_width_pcie_x4; + break; + case I40E_PCI_LINK_WIDTH_8: + hw->bus.width = i40e_bus_width_pcie_x8; + break; + default: + hw->bus.width = i40e_bus_width_unknown; + break; + } + + switch (link & I40E_PCI_LINK_SPEED) { + case I40E_PCI_LINK_SPEED_2500: + hw->bus.speed = i40e_bus_speed_2500; + break; + case I40E_PCI_LINK_SPEED_5000: + hw->bus.speed = i40e_bus_speed_5000; + break; + case I40E_PCI_LINK_SPEED_8000: + hw->bus.speed = i40e_bus_speed_8000; + break; + default: + hw->bus.speed = i40e_bus_speed_unknown; + break; + } + + + device_printf(dev,"PCI Express Bus: Speed %s %s\n", + ((hw->bus.speed == i40e_bus_speed_8000) ? "8.0GT/s": + (hw->bus.speed == i40e_bus_speed_5000) ? "5.0GT/s": + (hw->bus.speed == i40e_bus_speed_2500) ? "2.5GT/s":"Unknown"), + (hw->bus.width == i40e_bus_width_pcie_x8) ? "Width x8" : + (hw->bus.width == i40e_bus_width_pcie_x4) ? "Width x4" : + (hw->bus.width == i40e_bus_width_pcie_x1) ? "Width x1" : + ("Unknown")); + + if ((hw->bus.width <= i40e_bus_width_pcie_x8) && + (hw->bus.speed < i40e_bus_speed_8000)) { + device_printf(dev, "PCI-Express bandwidth available" + " for this device\n may be insufficient for" + " optimal performance.\n"); + device_printf(dev, "For expected performance a x8 " + "PCIE Gen3 slot is required.\n"); + } + + return (link); +} + +static int +ixl_sysctl_show_fw(SYSCTL_HANDLER_ARGS) +{ + struct ixl_pf *pf = (struct ixl_pf *)arg1; + struct i40e_hw *hw = &pf->hw; + char buf[32]; + + snprintf(buf, sizeof(buf), + "f%d.%d a%d.%d n%02x.%02x e%08x", + hw->aq.fw_maj_ver, hw->aq.fw_min_ver, + hw->aq.api_maj_ver, hw->aq.api_min_ver, + (hw->nvm.version & IXL_NVM_VERSION_HI_MASK) >> + IXL_NVM_VERSION_HI_SHIFT, + (hw->nvm.version & IXL_NVM_VERSION_LO_MASK) >> + IXL_NVM_VERSION_LO_SHIFT, + hw->nvm.eetrack); + return (sysctl_handle_string(oidp, buf, strlen(buf), req)); +} + + +#ifdef IXL_DEBUG_SYSCTL +static int +ixl_sysctl_link_status(SYSCTL_HANDLER_ARGS) +{ + struct ixl_pf *pf = (struct ixl_pf *)arg1; + struct i40e_hw *hw = &pf->hw; + struct i40e_link_status link_status; + char buf[512]; + + enum i40e_status_code aq_error = 0; + + aq_error = i40e_aq_get_link_info(hw, TRUE, &link_status, NULL); + if (aq_error) { + printf("i40e_aq_get_link_info() error %d\n", aq_error); + return (EPERM); + } + + sprintf(buf, "\n" + "PHY Type : %#04x\n" + "Speed : %#04x\n" + "Link info: %#04x\n" + "AN info : %#04x\n" + "Ext info : %#04x", + link_status.phy_type, link_status.link_speed, + link_status.link_info, link_status.an_info, + link_status.ext_info); + + return (sysctl_handle_string(oidp, buf, strlen(buf), req)); +} + +static int +ixl_sysctl_phy_abilities(SYSCTL_HANDLER_ARGS) +{ + struct ixl_pf *pf = (struct ixl_pf *)arg1; + struct i40e_hw *hw = &pf->hw; + char buf[512]; + enum i40e_status_code aq_error = 0; + + struct i40e_aq_get_phy_abilities_resp abilities; + + aq_error = i40e_aq_get_phy_capabilities(hw, + TRUE, FALSE, &abilities, NULL); + if (aq_error) { + printf("i40e_aq_get_phy_capabilities() error %d\n", aq_error); + return (EPERM); + } + + sprintf(buf, "\n" + "PHY Type : %#010x\n" + "Speed : %#04x\n" + "Abilities: %#04x\n" + "EEE cap : %#06x\n" + "EEER reg : %#010x\n" + "D3 Lpan : %#04x", + abilities.phy_type, abilities.link_speed, + abilities.abilities, abilities.eee_capability, + abilities.eeer_val, abilities.d3_lpan); + + return (sysctl_handle_string(oidp, buf, strlen(buf), req)); +} + +static int +ixl_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS) +{ + struct ixl_pf *pf = (struct ixl_pf *)arg1; + struct ixl_vsi *vsi = &pf->vsi; + struct ixl_mac_filter *f; + char *buf, *buf_i; + + int error = 0; + int ftl_len = 0; + int ftl_counter = 0; + int buf_len = 0; + int entry_len = 42; + + SLIST_FOREACH(f, &vsi->ftl, next) { + ftl_len++; + } + + if (ftl_len < 1) { + sysctl_handle_string(oidp, "(none)", 6, req); + return (0); + } + + buf_len = sizeof(char) * (entry_len + 1) * ftl_len + 2; + buf = buf_i = malloc(buf_len, M_IXL, M_NOWAIT); + + sprintf(buf_i++, "\n"); + SLIST_FOREACH(f, &vsi->ftl, next) { + sprintf(buf_i, + MAC_FORMAT ", vlan %4d, flags %#06x", + MAC_FORMAT_ARGS(f->macaddr), f->vlan, f->flags); + buf_i += entry_len; + /* don't print '\n' for last entry */ + if (++ftl_counter != ftl_len) { + sprintf(buf_i, "\n"); + buf_i++; + } + } + + error = sysctl_handle_string(oidp, buf, strlen(buf), req); + if (error) + printf("sysctl error: %d\n", error); + free(buf, M_IXL); + return error; +} + +#define IXL_SW_RES_SIZE 0x14 +static int +ixl_res_alloc_cmp(const void *a, const void *b) +{ + const struct i40e_aqc_switch_resource_alloc_element_resp *one, *two; + one = (const struct i40e_aqc_switch_resource_alloc_element_resp *)a; + two = (const struct i40e_aqc_switch_resource_alloc_element_resp *)b; + + return ((int)one->resource_type - (int)two->resource_type); +} + +static int +ixl_sysctl_hw_res_alloc(SYSCTL_HANDLER_ARGS) +{ + struct ixl_pf *pf = (struct ixl_pf *)arg1; + struct i40e_hw *hw = &pf->hw; + device_t dev = iflib_get_dev(pf->vsi.ctx); + struct sbuf *buf; + int error = 0; + + u8 num_entries; + struct i40e_aqc_switch_resource_alloc_element_resp resp[IXL_SW_RES_SIZE]; + + buf = sbuf_new(NULL, NULL, 1024, SBUF_AUTOEXTEND); + if (!buf) { + device_printf(dev, "Could not allocate sbuf for output.\n"); + return (ENOMEM); + } + + bzero(resp, sizeof(resp)); + error = i40e_aq_get_switch_resource_alloc(hw, &num_entries, + resp, + IXL_SW_RES_SIZE, + NULL); + if (error) { + device_printf(dev, + "%s: get_switch_resource_alloc() error %d, aq error %d\n", + __func__, error, hw->aq.asq_last_status); + sbuf_delete(buf); + return error; + } + + /* Sort entries by type for display */ + qsort(resp, num_entries, + sizeof(struct i40e_aqc_switch_resource_alloc_element_resp), + &ixl_res_alloc_cmp); + + sbuf_cat(buf, "\n"); + sbuf_printf(buf, "# of entries: %d\n", num_entries); + sbuf_printf(buf, + "Type | Guaranteed | Total | Used | Un-allocated\n" + " | (this) | (all) | (this) | (all) \n"); + for (int i = 0; i < num_entries; i++) { + sbuf_printf(buf, + "%#4x | %10d %5d %6d %12d", + resp[i].resource_type, + resp[i].guaranteed, + resp[i].total, + resp[i].used, + resp[i].total_unalloced); + if (i < num_entries - 1) + sbuf_cat(buf, "\n"); + } + + sbuf_trim(buf); + error = sbuf_finish(buf); + sbuf_delete(buf); + + return (error); +} + +/* +** Caller must init and delete sbuf; this function will clear and +** finish it for caller. +*/ +static char * +ixl_switch_element_string(struct sbuf *s, u16 seid, bool uplink) +{ + sbuf_clear(s); + + if (seid == 0 && uplink) + sbuf_cat(s, "Network"); + else if (seid == 0) + sbuf_cat(s, "Host"); + else if (seid == 1) + sbuf_cat(s, "EMP"); + else if (seid <= 5) + sbuf_printf(s, "MAC %d", seid - 2); + else if (seid <= 15) + sbuf_cat(s, "Reserved"); + else if (seid <= 31) + sbuf_printf(s, "PF %d", seid - 16); + else if (seid <= 159) + sbuf_printf(s, "VF %d", seid - 32); + else if (seid <= 287) + sbuf_cat(s, "Reserved"); + else if (seid <= 511) + sbuf_cat(s, "Other"); // for other structures + else if (seid <= 895) + sbuf_printf(s, "VSI %d", seid - 512); + else if (seid <= 1023) + sbuf_printf(s, "Reserved"); + else + sbuf_cat(s, "Invalid"); + + sbuf_finish(s); + return sbuf_data(s); +} + +static int +ixl_sysctl_switch_config(SYSCTL_HANDLER_ARGS) +{ + struct ixl_pf *pf = (struct ixl_pf *)arg1; + struct i40e_hw *hw = &pf->hw; + device_t dev = iflib_get_dev(pf->vsi.ctx); + struct sbuf *buf; + struct sbuf *nmbuf; + int error = 0; + u8 aq_buf[I40E_AQ_LARGE_BUF]; + + u16 next = 0; + struct i40e_aqc_get_switch_config_resp *sw_config; + sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf; + + buf = sbuf_new(NULL, NULL, 1024, SBUF_AUTOEXTEND); + if (!buf) { + device_printf(dev, "Could not allocate sbuf for sysctl output.\n"); + return (ENOMEM); + } + + error = i40e_aq_get_switch_config(hw, sw_config, + sizeof(aq_buf), &next, NULL); + if (error) { + device_printf(dev, + "%s: aq_get_switch_config() error %d, aq error %d\n", + __func__, error, hw->aq.asq_last_status); + sbuf_delete(buf); + return error; + } + + nmbuf = sbuf_new_auto(); + if (!nmbuf) { + device_printf(dev, "Could not allocate sbuf for name output.\n"); + return (ENOMEM); + } + + sbuf_cat(buf, "\n"); + // Assuming <= 255 elements in switch + sbuf_printf(buf, "# of elements: %d\n", sw_config->header.num_reported); + /* Exclude: + ** Revision -- all elements are revision 1 for now + */ + sbuf_printf(buf, + "SEID ( Name ) | Uplink | Downlink | Conn Type\n" + " | | | (uplink)\n"); + for (int i = 0; i < sw_config->header.num_reported; i++) { + // "%4d (%8s) | %8s %8s %#8x", + sbuf_printf(buf, "%4d", sw_config->element[i].seid); + sbuf_cat(buf, " "); + sbuf_printf(buf, "(%8s)", ixl_switch_element_string(nmbuf, + sw_config->element[i].seid, false)); + sbuf_cat(buf, " | "); + sbuf_printf(buf, "%8s", ixl_switch_element_string(nmbuf, + sw_config->element[i].uplink_seid, true)); + sbuf_cat(buf, " "); + sbuf_printf(buf, "%8s", ixl_switch_element_string(nmbuf, + sw_config->element[i].downlink_seid, false)); + sbuf_cat(buf, " "); + sbuf_printf(buf, "%#8x", sw_config->element[i].connection_type); + if (i < sw_config->header.num_reported - 1) + sbuf_cat(buf, "\n"); + } + sbuf_delete(nmbuf); + + error = sbuf_finish(buf); + sbuf_delete(buf); + + return (error); +} +#endif /* IXL_DEBUG_SYSCTL */ + + +#ifdef PCI_IOV +static int +ixl_vf_alloc_vsi(struct ixl_pf *pf, struct ixl_vf *vf) +{ + struct i40e_hw *hw; + struct ixl_vsi *vsi; + struct i40e_vsi_context vsi_ctx; + int i; + uint16_t first_queue; + enum i40e_status_code code; + + hw = &pf->hw; + vsi = &pf->vsi; + + vsi_ctx.pf_num = hw->pf_id; + vsi_ctx.uplink_seid = pf->veb_seid; + vsi_ctx.connection_type = IXL_VSI_DATA_PORT; + vsi_ctx.vf_num = hw->func_caps.vf_base_id + vf->vf_num; + vsi_ctx.flags = I40E_AQ_VSI_TYPE_VF; + + bzero(&vsi_ctx.info, sizeof(vsi_ctx.info)); + + vsi_ctx.info.valid_sections = htole16(I40E_AQ_VSI_PROP_SWITCH_VALID); + vsi_ctx.info.switch_id = htole16(0); + + vsi_ctx.info.valid_sections |= htole16(I40E_AQ_VSI_PROP_SECURITY_VALID); + vsi_ctx.info.sec_flags = 0; + if (vf->vf_flags & VF_FLAG_MAC_ANTI_SPOOF) + vsi_ctx.info.sec_flags |= I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK; + + vsi_ctx.info.valid_sections |= htole16(I40E_AQ_VSI_PROP_VLAN_VALID); + vsi_ctx.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | + I40E_AQ_VSI_PVLAN_EMOD_NOTHING; + + vsi_ctx.info.valid_sections |= + htole16(I40E_AQ_VSI_PROP_QUEUE_MAP_VALID); + vsi_ctx.info.mapping_flags = htole16(I40E_AQ_VSI_QUE_MAP_NONCONTIG); + first_queue = vsi->num_rx_queues + vf->vf_num * IXLV_MAX_QUEUES; + for (i = 0; i < IXLV_MAX_QUEUES; i++) + vsi_ctx.info.queue_mapping[i] = htole16(first_queue + i); + for (; i < nitems(vsi_ctx.info.queue_mapping); i++) + vsi_ctx.info.queue_mapping[i] = htole16(I40E_AQ_VSI_QUEUE_MASK); + + vsi_ctx.info.tc_mapping[0] = htole16( + (0 << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | + (1 << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT)); + + code = i40e_aq_add_vsi(hw, &vsi_ctx, NULL); + if (code != I40E_SUCCESS) + return (ixl_adminq_err_to_errno(hw->aq.asq_last_status)); + vf->vsi.seid = vsi_ctx.seid; + vf->vsi.vsi_num = vsi_ctx.vsi_number; + vf->vsi.first_queue = first_queue; + vf->vsi.num_tx_queues = IXLV_MAX_QUEUES; + vf->vsi.num_rx_queues = IXLV_MAX_QUEUES; + + code = i40e_aq_get_vsi_params(hw, &vsi_ctx, NULL); + if (code != I40E_SUCCESS) + return (ixl_adminq_err_to_errno(hw->aq.asq_last_status)); + + code = i40e_aq_config_vsi_bw_limit(hw, vf->vsi.seid, 0, 0, NULL); + if (code != I40E_SUCCESS) { + device_printf(iflib_get_dev(pf->vsi.ctx), "Failed to disable BW limit: %d\n", + ixl_adminq_err_to_errno(hw->aq.asq_last_status)); + return (ixl_adminq_err_to_errno(hw->aq.asq_last_status)); + } + + memcpy(&vf->vsi.info, &vsi_ctx.info, sizeof(vf->vsi.info)); + return (0); +} + +static int +ixl_vf_setup_vsi(struct ixl_pf *pf, struct ixl_vf *vf) +{ + struct i40e_hw *hw; + int error; + + hw = &pf->hw; + + error = ixl_vf_alloc_vsi(pf, vf); + if (error != 0) + return (error); + + vf->vsi.hw_filters_add = 0; + vf->vsi.hw_filters_del = 0; + ixl_add_filter(&vf->vsi, ixl_bcast_addr, IXL_VLAN_ANY); + ixl_reconfigure_filters(&vf->vsi); + + return (0); +} + +static void +ixl_vf_map_vsi_queue(struct i40e_hw *hw, struct ixl_vf *vf, int qnum, + uint32_t val) +{ + uint32_t qtable; + int index, shift; + + /* + * Two queues are mapped in a single register, so we have to do some + * gymnastics to convert the queue number into a register index and + * shift. + */ + index = qnum / 2; + shift = (qnum % 2) * I40E_VSILAN_QTABLE_QINDEX_1_SHIFT; + + qtable = rd32(hw, I40E_VSILAN_QTABLE(index, vf->vsi.vsi_num)); + qtable &= ~(I40E_VSILAN_QTABLE_QINDEX_0_MASK << shift); + qtable |= val << shift; + wr32(hw, I40E_VSILAN_QTABLE(index, vf->vsi.vsi_num), qtable); +} + +static void +ixl_vf_map_queues(struct ixl_pf *pf, struct ixl_vf *vf) +{ + struct i40e_hw *hw; + uint32_t qtable; + int i; + + hw = &pf->hw; + + /* + * Contiguous mappings aren't actually supported by the hardware, + * so we have to use non-contiguous mappings. + */ + wr32(hw, I40E_VSILAN_QBASE(vf->vsi.vsi_num), + I40E_VSILAN_QBASE_VSIQTABLE_ENA_MASK); + + wr32(hw, I40E_VPLAN_MAPENA(vf->vf_num), + I40E_VPLAN_MAPENA_TXRX_ENA_MASK); + + for (i = 0; i < vf->vsi.num_tx_queues; i++) { + qtable = (vf->vsi.first_queue + i) << + I40E_VPLAN_QTABLE_QINDEX_SHIFT; + + wr32(hw, I40E_VPLAN_QTABLE(i, vf->vf_num), qtable); + } + + /* Map queues allocated to VF to its VSI. */ + for (i = 0; i < vf->vsi.num_tx_queues; i++) + ixl_vf_map_vsi_queue(hw, vf, i, vf->vsi.first_queue + i); + + /* Set rest of VSI queues as unused. */ + for (; i < IXL_MAX_VSI_QUEUES; i++) + ixl_vf_map_vsi_queue(hw, vf, i, + I40E_VSILAN_QTABLE_QINDEX_0_MASK); + + ixl_flush(hw); +} + +static void +ixl_vf_vsi_release(struct ixl_pf *pf, struct ixl_vsi *vsi) +{ + struct i40e_hw *hw; + + hw = &pf->hw; + + if (vsi->seid == 0) + return; + + i40e_aq_delete_element(hw, vsi->seid, NULL); +} + +static void +ixl_vf_disable_queue_intr(struct i40e_hw *hw, uint32_t vfint_reg) +{ + + wr32(hw, vfint_reg, I40E_VFINT_DYN_CTLN_CLEARPBA_MASK); + ixl_flush(hw); +} + +static void +ixl_vf_unregister_intr(struct i40e_hw *hw, uint32_t vpint_reg) +{ + + wr32(hw, vpint_reg, I40E_VPINT_LNKLSTN_FIRSTQ_TYPE_MASK | + I40E_VPINT_LNKLSTN_FIRSTQ_INDX_MASK); + ixl_flush(hw); +} + +static void +ixl_vf_release_resources(struct ixl_pf *pf, struct ixl_vf *vf) +{ + struct i40e_hw *hw; + uint32_t vfint_reg, vpint_reg; + int i; + + hw = &pf->hw; + + ixl_vf_vsi_release(pf, &vf->vsi); + + /* Index 0 has a special register. */ + ixl_vf_disable_queue_intr(hw, I40E_VFINT_DYN_CTL0(vf->vf_num)); + + for (i = 1; i < hw->func_caps.num_msix_vectors_vf; i++) { + vfint_reg = IXL_VFINT_DYN_CTLN_REG(hw, i , vf->vf_num); + ixl_vf_disable_queue_intr(hw, vfint_reg); + } + + /* Index 0 has a special register. */ + ixl_vf_unregister_intr(hw, I40E_VPINT_LNKLST0(vf->vf_num)); + + for (i = 1; i < hw->func_caps.num_msix_vectors_vf; i++) { + vpint_reg = IXL_VPINT_LNKLSTN_REG(hw, i, vf->vf_num); + ixl_vf_unregister_intr(hw, vpint_reg); + } + + vf->vsi.num_tx_queues = 0; + vf->vsi.num_rx_queues = 0; +} + +static int +ixl_flush_pcie(struct ixl_pf *pf, struct ixl_vf *vf) +{ + struct i40e_hw *hw; + int i; + uint16_t global_vf_num; + uint32_t ciad; + + hw = &pf->hw; + global_vf_num = hw->func_caps.vf_base_id + vf->vf_num; + + wr32(hw, I40E_PF_PCI_CIAA, IXL_PF_PCI_CIAA_VF_DEVICE_STATUS | + (global_vf_num << I40E_PF_PCI_CIAA_VF_NUM_SHIFT)); + for (i = 0; i < IXL_VF_RESET_TIMEOUT; i++) { + ciad = rd32(hw, I40E_PF_PCI_CIAD); + if ((ciad & IXL_PF_PCI_CIAD_VF_TRANS_PENDING_MASK) == 0) + return (0); + DELAY(1); + } + + return (ETIMEDOUT); +} + +static void +ixl_reset_vf(struct ixl_pf *pf, struct ixl_vf *vf) +{ + struct i40e_hw *hw; + uint32_t vfrtrig; + + hw = &pf->hw; + + vfrtrig = rd32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num)); + vfrtrig |= I40E_VPGEN_VFRTRIG_VFSWR_MASK; + wr32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num), vfrtrig); + ixl_flush(hw); + + ixl_reinit_vf(pf, vf); +} + +static void +ixl_reinit_vf(struct ixl_pf *pf, struct ixl_vf *vf) +{ + struct i40e_hw *hw; + uint32_t vfrstat, vfrtrig; + int i, error; + + hw = &pf->hw; + + error = ixl_flush_pcie(pf, vf); + if (error != 0) + device_printf(iflib_get_dev(pf->vsi.ctx), + "Timed out waiting for PCIe activity to stop on VF-%d\n", + vf->vf_num); + + for (i = 0; i < IXL_VF_RESET_TIMEOUT; i++) { + DELAY(10); + + vfrstat = rd32(hw, I40E_VPGEN_VFRSTAT(vf->vf_num)); + if (vfrstat & I40E_VPGEN_VFRSTAT_VFRD_MASK) + break; + } + + if (i == IXL_VF_RESET_TIMEOUT) + device_printf(iflib_get_dev(pf->vsi.ctx), "VF %d failed to reset\n", vf->vf_num); + + wr32(hw, I40E_VFGEN_RSTAT1(vf->vf_num), I40E_VFR_COMPLETED); + + vfrtrig = rd32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num)); + vfrtrig &= ~I40E_VPGEN_VFRTRIG_VFSWR_MASK; + wr32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num), vfrtrig); + + if (vf->vsi.seid != 0) + ixl_disable_rings(&vf->vsi); + + ixl_vf_release_resources(pf, vf); + ixl_vf_setup_vsi(pf, vf); + ixl_vf_map_queues(pf, vf); + + wr32(hw, I40E_VFGEN_RSTAT1(vf->vf_num), I40E_VFR_VFACTIVE); + ixl_flush(hw); +} + +static const char * +ixl_vc_opcode_str(uint16_t op) +{ + + switch (op) { + case I40E_VIRTCHNL_OP_VERSION: + return ("VERSION"); + case I40E_VIRTCHNL_OP_RESET_VF: + return ("RESET_VF"); + case I40E_VIRTCHNL_OP_GET_VF_RESOURCES: + return ("GET_VF_RESOURCES"); + case I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE: + return ("CONFIG_TX_QUEUE"); + case I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE: + return ("CONFIG_RX_QUEUE"); + case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES: + return ("CONFIG_VSI_QUEUES"); + case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP: + return ("CONFIG_IRQ_MAP"); + case I40E_VIRTCHNL_OP_ENABLE_QUEUES: + return ("ENABLE_QUEUES"); + case I40E_VIRTCHNL_OP_DISABLE_QUEUES: + return ("DISABLE_QUEUES"); + case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS: + return ("ADD_ETHER_ADDRESS"); + case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS: + return ("DEL_ETHER_ADDRESS"); + case I40E_VIRTCHNL_OP_ADD_VLAN: + return ("ADD_VLAN"); + case I40E_VIRTCHNL_OP_DEL_VLAN: + return ("DEL_VLAN"); + case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE: + return ("CONFIG_PROMISCUOUS_MODE"); + case I40E_VIRTCHNL_OP_GET_STATS: + return ("GET_STATS"); + case I40E_VIRTCHNL_OP_FCOE: + return ("FCOE"); + case I40E_VIRTCHNL_OP_EVENT: + return ("EVENT"); + default: + return ("UNKNOWN"); + } +} + +static int +ixl_vc_opcode_level(uint16_t opcode) +{ + + switch (opcode) { + case I40E_VIRTCHNL_OP_GET_STATS: + return (10); + default: + return (5); + } +} + +static void +ixl_send_vf_msg(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op, + enum i40e_status_code status, void *msg, uint16_t len) +{ + struct i40e_hw *hw; + int global_vf_id; + + hw = &pf->hw; + global_vf_id = hw->func_caps.vf_base_id + vf->vf_num; + + I40E_VC_DEBUG(pf, ixl_vc_opcode_level(op), + "Sending msg (op=%s[%d], status=%d) to VF-%d\n", + ixl_vc_opcode_str(op), op, status, vf->vf_num); + + i40e_aq_send_msg_to_vf(hw, global_vf_id, op, status, msg, len, NULL); +} + +static void +ixl_send_vf_ack(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op) +{ + + ixl_send_vf_msg(pf, vf, op, I40E_SUCCESS, NULL, 0); +} + +static void +ixl_send_vf_nack_msg(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op, + enum i40e_status_code status, const char *file, int line) +{ + + I40E_VC_DEBUG(pf, 1, + "Sending NACK (op=%s[%d], err=%d) to VF-%d from %s:%d\n", + ixl_vc_opcode_str(op), op, status, vf->vf_num, file, line); + ixl_send_vf_msg(pf, vf, op, status, NULL, 0); +} + +static void +ixl_vf_version_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, + uint16_t msg_size) +{ + struct i40e_virtchnl_version_info reply; + + if (msg_size != sizeof(struct i40e_virtchnl_version_info)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_VERSION, + I40E_ERR_PARAM); + return; + } + + reply.major = I40E_VIRTCHNL_VERSION_MAJOR; + reply.minor = I40E_VIRTCHNL_VERSION_MINOR; + ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_VERSION, I40E_SUCCESS, &reply, + sizeof(reply)); +} + +static void +ixl_vf_reset_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, + uint16_t msg_size) +{ + + if (msg_size != 0) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_RESET_VF, + I40E_ERR_PARAM); + return; + } + + ixl_reset_vf(pf, vf); + + /* No response to a reset message. */ +} + +static void +ixl_vf_get_resources_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, + uint16_t msg_size) +{ + struct i40e_virtchnl_vf_resource reply; + + if (msg_size != 0) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_VF_RESOURCES, + I40E_ERR_PARAM); + return; + } + + bzero(&reply, sizeof(reply)); + + reply.vf_offload_flags = I40E_VIRTCHNL_VF_OFFLOAD_L2; + + reply.num_vsis = 1; + reply.num_queue_pairs = vf->vsi.num_tx_queues; + reply.max_vectors = pf->hw.func_caps.num_msix_vectors_vf; + reply.vsi_res[0].vsi_id = vf->vsi.vsi_num; + reply.vsi_res[0].vsi_type = I40E_VSI_SRIOV; + reply.vsi_res[0].num_queue_pairs = vf->vsi.num_tx_queues; + memcpy(reply.vsi_res[0].default_mac_addr, vf->mac, ETHER_ADDR_LEN); + + ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_GET_VF_RESOURCES, + I40E_SUCCESS, &reply, sizeof(reply)); +} + +static int +ixl_vf_config_tx_queue(struct ixl_pf *pf, struct ixl_vf *vf, + struct i40e_virtchnl_txq_info *info) +{ + struct i40e_hw *hw; + struct i40e_hmc_obj_txq txq; + uint16_t global_queue_num, global_vf_num; + enum i40e_status_code status; + uint32_t qtx_ctl; + + hw = &pf->hw; + global_queue_num = vf->vsi.first_queue + info->queue_id; + global_vf_num = hw->func_caps.vf_base_id + vf->vf_num; + bzero(&txq, sizeof(txq)); + + status = i40e_clear_lan_tx_queue_context(hw, global_queue_num); + if (status != I40E_SUCCESS) + return (EINVAL); + + txq.base = info->dma_ring_addr / IXL_TX_CTX_BASE_UNITS; + + txq.head_wb_ena = info->headwb_enabled; + txq.head_wb_addr = info->dma_headwb_addr; + txq.qlen = info->ring_len; + txq.rdylist = le16_to_cpu(vf->vsi.info.qs_handle[0]); + txq.rdylist_act = 0; + + status = i40e_set_lan_tx_queue_context(hw, global_queue_num, &txq); + if (status != I40E_SUCCESS) + return (EINVAL); + + qtx_ctl = I40E_QTX_CTL_VF_QUEUE | + (hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) | + (global_vf_num << I40E_QTX_CTL_VFVM_INDX_SHIFT); + wr32(hw, I40E_QTX_CTL(global_queue_num), qtx_ctl); + ixl_flush(hw); + + return (0); +} + +static int +ixl_vf_config_rx_queue(struct ixl_pf *pf, struct ixl_vf *vf, + struct i40e_virtchnl_rxq_info *info) +{ + struct i40e_hw *hw; + struct i40e_hmc_obj_rxq rxq; + uint16_t global_queue_num; + enum i40e_status_code status; + + hw = &pf->hw; + global_queue_num = vf->vsi.first_queue + info->queue_id; + bzero(&rxq, sizeof(rxq)); + + if (info->databuffer_size > IXL_VF_MAX_BUFFER) + return (EINVAL); + + if (info->max_pkt_size > IXL_VF_MAX_FRAME || + info->max_pkt_size < ETHER_MIN_LEN) + return (EINVAL); + + if (info->splithdr_enabled) { + if (info->hdr_size > IXL_VF_MAX_HDR_BUFFER) + return (EINVAL); + + rxq.hsplit_0 = info->rx_split_pos & + (I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_L2 | + I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_IP | + I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_TCP_UDP | + I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_SCTP); + rxq.hbuff = info->hdr_size >> I40E_RXQ_CTX_HBUFF_SHIFT; + + rxq.dtype = 2; + } + + status = i40e_clear_lan_rx_queue_context(hw, global_queue_num); + if (status != I40E_SUCCESS) + return (EINVAL); + + rxq.base = info->dma_ring_addr / IXL_RX_CTX_BASE_UNITS; + rxq.qlen = info->ring_len; + + rxq.dbuff = info->databuffer_size >> I40E_RXQ_CTX_DBUFF_SHIFT; + + rxq.dsize = 1; + rxq.crcstrip = 1; + rxq.l2tsel = 1; + + rxq.rxmax = info->max_pkt_size; + rxq.tphrdesc_ena = 1; + rxq.tphwdesc_ena = 1; + rxq.tphdata_ena = 1; + rxq.tphhead_ena = 1; + rxq.lrxqthresh = 2; + rxq.prefena = 1; + + status = i40e_set_lan_rx_queue_context(hw, global_queue_num, &rxq); + if (status != I40E_SUCCESS) + return (EINVAL); + + return (0); +} + +static void +ixl_vf_config_vsi_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, + uint16_t msg_size) +{ + struct i40e_virtchnl_vsi_queue_config_info *info; + struct i40e_virtchnl_queue_pair_info *pair; + int i; + + if (msg_size < sizeof(*info)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, + I40E_ERR_PARAM); + return; + } + + info = msg; + if (info->num_queue_pairs == 0) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, + I40E_ERR_PARAM); + return; + } + + if (msg_size != sizeof(*info) + info->num_queue_pairs * sizeof(*pair)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, + I40E_ERR_PARAM); + return; + } + + if (info->vsi_id != vf->vsi.vsi_num) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, + I40E_ERR_PARAM); + return; + } + + for (i = 0; i < info->num_queue_pairs; i++) { + pair = &info->qpair[i]; + + if (pair->txq.vsi_id != vf->vsi.vsi_num || + pair->rxq.vsi_id != vf->vsi.vsi_num || + pair->txq.queue_id != pair->rxq.queue_id || + pair->txq.queue_id >= vf->vsi.num_tx_queues) { + + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM); + return; + } + + if (ixl_vf_config_tx_queue(pf, vf, &pair->txq) != 0) { + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM); + return; + } + + if (ixl_vf_config_rx_queue(pf, vf, &pair->rxq) != 0) { + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM); + return; + } + } + + ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES); +} + +static void +ixl_vf_set_qctl(struct ixl_pf *pf, + const struct i40e_virtchnl_vector_map *vector, + enum i40e_queue_type cur_type, uint16_t cur_queue, + enum i40e_queue_type *last_type, uint16_t *last_queue) +{ + uint32_t offset, qctl; + uint16_t itr_indx; + + if (cur_type == I40E_QUEUE_TYPE_RX) { + offset = I40E_QINT_RQCTL(cur_queue); + itr_indx = vector->rxitr_idx; + } else { + offset = I40E_QINT_TQCTL(cur_queue); + itr_indx = vector->txitr_idx; + } + + qctl = htole32((vector->vector_id << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | + (*last_type << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) | + (*last_queue << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | + I40E_QINT_RQCTL_CAUSE_ENA_MASK | + (itr_indx << I40E_QINT_RQCTL_ITR_INDX_SHIFT)); + + wr32(&pf->hw, offset, qctl); + + *last_type = cur_type; + *last_queue = cur_queue; +} + +static void +ixl_vf_config_vector(struct ixl_pf *pf, struct ixl_vf *vf, + const struct i40e_virtchnl_vector_map *vector) +{ + struct i40e_hw *hw; + u_int qindex; + enum i40e_queue_type type, last_type; + uint32_t lnklst_reg; + uint16_t rxq_map, txq_map, cur_queue, last_queue; + + hw = &pf->hw; + + rxq_map = vector->rxq_map; + txq_map = vector->txq_map; + + last_queue = IXL_END_OF_INTR_LNKLST; + last_type = I40E_QUEUE_TYPE_RX; + + /* + * The datasheet says to optimize performance, RX queues and TX queues + * should be interleaved in the interrupt linked list, so we process + * both at once here. + */ + while ((rxq_map != 0) || (txq_map != 0)) { + if (txq_map != 0) { + qindex = ffs(txq_map) - 1; + type = I40E_QUEUE_TYPE_TX; + cur_queue = vf->vsi.first_queue + qindex; + ixl_vf_set_qctl(pf, vector, type, cur_queue, + &last_type, &last_queue); + txq_map &= ~(1 << qindex); + } + + if (rxq_map != 0) { + qindex = ffs(rxq_map) - 1; + type = I40E_QUEUE_TYPE_RX; + cur_queue = vf->vsi.first_queue + qindex; + ixl_vf_set_qctl(pf, vector, type, cur_queue, + &last_type, &last_queue); + rxq_map &= ~(1 << qindex); + } + } + + if (vector->vector_id == 0) + lnklst_reg = I40E_VPINT_LNKLST0(vf->vf_num); + else + lnklst_reg = IXL_VPINT_LNKLSTN_REG(hw, vector->vector_id, + vf->vf_num); + wr32(hw, lnklst_reg, + (last_queue << I40E_VPINT_LNKLST0_FIRSTQ_INDX_SHIFT) | + (last_type << I40E_VPINT_LNKLST0_FIRSTQ_TYPE_SHIFT)); + + ixl_flush(hw); +} + +static void +ixl_vf_config_irq_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, + uint16_t msg_size) +{ + struct i40e_virtchnl_irq_map_info *map; + struct i40e_virtchnl_vector_map *vector; + struct i40e_hw *hw; + int i, largest_txq, largest_rxq; + + hw = &pf->hw; + + if (msg_size < sizeof(*map)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, + I40E_ERR_PARAM); + return; + } + + map = msg; + if (map->num_vectors == 0) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, + I40E_ERR_PARAM); + return; + } + + if (msg_size != sizeof(*map) + map->num_vectors * sizeof(*vector)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, + I40E_ERR_PARAM); + return; + } + + for (i = 0; i < map->num_vectors; i++) { + vector = &map->vecmap[i]; + + if ((vector->vector_id >= hw->func_caps.num_msix_vectors_vf) || + vector->vsi_id != vf->vsi.vsi_num) { + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, I40E_ERR_PARAM); + return; + } + + if (vector->rxq_map != 0) { + largest_rxq = fls(vector->rxq_map) - 1; + if (largest_rxq >= vf->vsi.num_rx_queues) { + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, + I40E_ERR_PARAM); + return; + } + } + + if (vector->txq_map != 0) { + largest_txq = fls(vector->txq_map) - 1; + if (largest_txq >= vf->vsi.num_tx_queues) { + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, + I40E_ERR_PARAM); + return; + } + } + + if (vector->rxitr_idx > IXL_MAX_ITR_IDX || + vector->txitr_idx > IXL_MAX_ITR_IDX) { + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, + I40E_ERR_PARAM); + return; + } + + ixl_vf_config_vector(pf, vf, vector); + } + + ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP); +} + +static void +ixl_vf_enable_queues_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, + uint16_t msg_size) +{ + struct i40e_virtchnl_queue_select *select; + int error; + + if (msg_size != sizeof(*select)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES, + I40E_ERR_PARAM); + return; + } + + select = msg; + if (select->vsi_id != vf->vsi.vsi_num || + select->rx_queues == 0 || select->tx_queues == 0) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES, + I40E_ERR_PARAM); + return; + } + + error = ixl_enable_rings(&vf->vsi); + if (error) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES, + I40E_ERR_TIMEOUT); + return; + } + + ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES); +} + +static void +ixl_vf_disable_queues_msg(struct ixl_pf *pf, struct ixl_vf *vf, + void *msg, uint16_t msg_size) +{ + struct i40e_virtchnl_queue_select *select; + int error; + + if (msg_size != sizeof(*select)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES, + I40E_ERR_PARAM); + return; + } + + select = msg; + if (select->vsi_id != vf->vsi.vsi_num || + select->rx_queues == 0 || select->tx_queues == 0) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES, + I40E_ERR_PARAM); + return; + } + + error = ixl_disable_rings(&vf->vsi); + if (error) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES, + I40E_ERR_TIMEOUT); + return; + } + + ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES); +} + +static boolean_t +ixl_zero_mac(const uint8_t *addr) +{ + uint8_t zero[ETHER_ADDR_LEN] = {0, 0, 0, 0, 0, 0}; + + return (cmp_etheraddr(addr, zero)); +} + +static boolean_t +ixl_bcast_mac(const uint8_t *addr) +{ + + return (cmp_etheraddr(addr, ixl_bcast_addr)); +} + +static int +ixl_vf_mac_valid(struct ixl_vf *vf, const uint8_t *addr) +{ + + if (ixl_zero_mac(addr) || ixl_bcast_mac(addr)) + return (EINVAL); + + /* + * If the VF is not allowed to change its MAC address, don't let it + * set a MAC filter for an address that is not a multicast address and + * is not its assigned MAC. + */ + if (!(vf->vf_flags & VF_FLAG_SET_MAC_CAP) && + !(ETHER_IS_MULTICAST(addr) || cmp_etheraddr(addr, vf->mac))) + return (EPERM); + + return (0); +} + +static void +ixl_vf_add_mac_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, + uint16_t msg_size) +{ + struct i40e_virtchnl_ether_addr_list *addr_list; + struct i40e_virtchnl_ether_addr *addr; + struct ixl_vsi *vsi; + int i; + size_t expected_size; + + vsi = &vf->vsi; + + if (msg_size < sizeof(*addr_list)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, + I40E_ERR_PARAM); + return; + } + + addr_list = msg; + expected_size = sizeof(*addr_list) + + addr_list->num_elements * sizeof(*addr); + + if (addr_list->num_elements == 0 || + addr_list->vsi_id != vsi->vsi_num || + msg_size != expected_size) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, + I40E_ERR_PARAM); + return; + } + + for (i = 0; i < addr_list->num_elements; i++) { + if (ixl_vf_mac_valid(vf, addr_list->list[i].addr) != 0) { + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, I40E_ERR_PARAM); + return; + } + } + + for (i = 0; i < addr_list->num_elements; i++) { + addr = &addr_list->list[i]; + ixl_add_filter(vsi, addr->addr, IXL_VLAN_ANY); + } + + ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS); +} + +static void +ixl_vf_del_mac_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, + uint16_t msg_size) +{ + struct i40e_virtchnl_ether_addr_list *addr_list; + struct i40e_virtchnl_ether_addr *addr; + size_t expected_size; + int i; + + if (msg_size < sizeof(*addr_list)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, + I40E_ERR_PARAM); + return; + } + + addr_list = msg; + expected_size = sizeof(*addr_list) + + addr_list->num_elements * sizeof(*addr); + + if (addr_list->num_elements == 0 || + addr_list->vsi_id != vf->vsi.vsi_num || + msg_size != expected_size) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, + I40E_ERR_PARAM); + return; + } + + for (i = 0; i < addr_list->num_elements; i++) { + addr = &addr_list->list[i]; + if (ixl_zero_mac(addr->addr) || ixl_bcast_mac(addr->addr)) { + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, I40E_ERR_PARAM); + return; + } + } + + for (i = 0; i < addr_list->num_elements; i++) { + addr = &addr_list->list[i]; + ixl_del_filter(&vf->vsi, addr->addr, IXL_VLAN_ANY); + } + + ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS); +} + +static enum i40e_status_code +ixl_vf_enable_vlan_strip(struct ixl_pf *pf, struct ixl_vf *vf) +{ + struct i40e_vsi_context vsi_ctx; + + vsi_ctx.seid = vf->vsi.seid; + + bzero(&vsi_ctx.info, sizeof(vsi_ctx.info)); + vsi_ctx.info.valid_sections = htole16(I40E_AQ_VSI_PROP_VLAN_VALID); + vsi_ctx.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | + I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH; + return (i40e_aq_update_vsi_params(&pf->hw, &vsi_ctx, NULL)); +} + +static void +ixl_vf_add_vlan_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, + uint16_t msg_size) +{ + struct i40e_virtchnl_vlan_filter_list *filter_list; + enum i40e_status_code code; + size_t expected_size; + int i; + + if (msg_size < sizeof(*filter_list)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, + I40E_ERR_PARAM); + return; + } + + filter_list = msg; + expected_size = sizeof(*filter_list) + + filter_list->num_elements * sizeof(uint16_t); + if (filter_list->num_elements == 0 || + filter_list->vsi_id != vf->vsi.vsi_num || + msg_size != expected_size) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, + I40E_ERR_PARAM); + return; + } + + if (!(vf->vf_flags & VF_FLAG_VLAN_CAP)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, + I40E_ERR_PARAM); + return; + } + + for (i = 0; i < filter_list->num_elements; i++) { + if (filter_list->vlan_id[i] > EVL_VLID_MASK) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, + I40E_ERR_PARAM); + return; + } + } + + code = ixl_vf_enable_vlan_strip(pf, vf); + if (code != I40E_SUCCESS) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, + I40E_ERR_PARAM); + } + + for (i = 0; i < filter_list->num_elements; i++) + ixl_add_filter(&vf->vsi, vf->mac, filter_list->vlan_id[i]); + + ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN); +} + +static void +ixl_vf_del_vlan_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, + uint16_t msg_size) +{ + struct i40e_virtchnl_vlan_filter_list *filter_list; + int i; + size_t expected_size; + + if (msg_size < sizeof(*filter_list)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN, + I40E_ERR_PARAM); + return; + } + + filter_list = msg; + expected_size = sizeof(*filter_list) + + filter_list->num_elements * sizeof(uint16_t); + if (filter_list->num_elements == 0 || + filter_list->vsi_id != vf->vsi.vsi_num || + msg_size != expected_size) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN, + I40E_ERR_PARAM); + return; + } + + for (i = 0; i < filter_list->num_elements; i++) { + if (filter_list->vlan_id[i] > EVL_VLID_MASK) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, + I40E_ERR_PARAM); + return; + } + } + + if (!(vf->vf_flags & VF_FLAG_VLAN_CAP)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, + I40E_ERR_PARAM); + return; + } + + for (i = 0; i < filter_list->num_elements; i++) + ixl_del_filter(&vf->vsi, vf->mac, filter_list->vlan_id[i]); + + ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN); +} + +static void +ixl_vf_config_promisc_msg(struct ixl_pf *pf, struct ixl_vf *vf, + void *msg, uint16_t msg_size) +{ + struct i40e_virtchnl_promisc_info *info; + enum i40e_status_code code; + + if (msg_size != sizeof(*info)) { + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM); + return; + } + + if (!(vf->vf_flags & VF_FLAG_PROMISC_CAP)) { + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM); + return; + } + + info = msg; + if (info->vsi_id != vf->vsi.vsi_num) { + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM); + return; + } + + code = i40e_aq_set_vsi_unicast_promiscuous(&pf->hw, info->vsi_id, + info->flags & I40E_FLAG_VF_UNICAST_PROMISC, NULL); + if (code != I40E_SUCCESS) { + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, code); + return; + } + + code = i40e_aq_set_vsi_multicast_promiscuous(&pf->hw, info->vsi_id, + info->flags & I40E_FLAG_VF_MULTICAST_PROMISC, NULL); + if (code != I40E_SUCCESS) { + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, code); + return; + } + + ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE); +} + +static void +ixl_vf_get_stats_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, + uint16_t msg_size) +{ + struct i40e_virtchnl_queue_select *queue; + + if (msg_size != sizeof(*queue)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_STATS, + I40E_ERR_PARAM); + return; + } + + queue = msg; + if (queue->vsi_id != vf->vsi.vsi_num) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_STATS, + I40E_ERR_PARAM); + return; + } + + ixl_update_eth_stats(&vf->vsi); + + ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_GET_STATS, + I40E_SUCCESS, &vf->vsi.eth_stats, sizeof(vf->vsi.eth_stats)); +} + +static void +ixl_handle_vf_msg(struct ixl_pf *pf, struct i40e_arq_event_info *event) +{ + struct ixl_vf *vf; + void *msg; + uint16_t vf_num, msg_size; + uint32_t opcode; + + vf_num = le16toh(event->desc.retval) - pf->hw.func_caps.vf_base_id; + opcode = le32toh(event->desc.cookie_high); + + if (vf_num >= pf->num_vfs) { + device_printf(iflib_get_dev(pf->vsi.ctx), "Got msg from illegal VF: %d\n", vf_num); + return; + } + + vf = &pf->vfs[vf_num]; + msg = event->msg_buf; + msg_size = event->msg_len; + + I40E_VC_DEBUG(pf, ixl_vc_opcode_level(opcode), + "Got msg %s(%d) from VF-%d of size %d\n", + ixl_vc_opcode_str(opcode), opcode, vf_num, msg_size); + + switch (opcode) { + case I40E_VIRTCHNL_OP_VERSION: + ixl_vf_version_msg(pf, vf, msg, msg_size); + break; + case I40E_VIRTCHNL_OP_RESET_VF: + ixl_vf_reset_msg(pf, vf, msg, msg_size); + break; + case I40E_VIRTCHNL_OP_GET_VF_RESOURCES: + ixl_vf_get_resources_msg(pf, vf, msg, msg_size); + break; + case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES: + ixl_vf_config_vsi_msg(pf, vf, msg, msg_size); + break; + case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP: + ixl_vf_config_irq_msg(pf, vf, msg, msg_size); + break; + case I40E_VIRTCHNL_OP_ENABLE_QUEUES: + ixl_vf_enable_queues_msg(pf, vf, msg, msg_size); + break; + case I40E_VIRTCHNL_OP_DISABLE_QUEUES: + ixl_vf_disable_queues_msg(pf, vf, msg, msg_size); + break; + case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS: + ixl_vf_add_mac_msg(pf, vf, msg, msg_size); + break; + case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS: + ixl_vf_del_mac_msg(pf, vf, msg, msg_size); + break; + case I40E_VIRTCHNL_OP_ADD_VLAN: + ixl_vf_add_vlan_msg(pf, vf, msg, msg_size); + break; + case I40E_VIRTCHNL_OP_DEL_VLAN: + ixl_vf_del_vlan_msg(pf, vf, msg, msg_size); + break; + case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE: + ixl_vf_config_promisc_msg(pf, vf, msg, msg_size); + break; + case I40E_VIRTCHNL_OP_GET_STATS: + ixl_vf_get_stats_msg(pf, vf, msg, msg_size); + break; + + /* These two opcodes have been superseded by CONFIG_VSI_QUEUES. */ + case I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE: + case I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE: + default: + i40e_send_vf_nack(pf, vf, opcode, I40E_ERR_NOT_IMPLEMENTED); + break; + } +} + +/* Handle any VFs that have reset themselves via a Function Level Reset(FLR). */ +static void +ixl_if_handle_vflr(if_ctx_t ctx) +{ + struct ixl_pf *pf; + struct i40e_hw *hw; + uint16_t global_vf_num; + uint32_t vflrstat_index, vflrstat_mask, vflrstat, icr0; + int i; + + pf = iflib_get_softc(ctx); + hw = &pf->hw; + + for (i = 0; i < pf->num_vfs; i++) { + global_vf_num = hw->func_caps.vf_base_id + i; + + vflrstat_index = IXL_GLGEN_VFLRSTAT_INDEX(global_vf_num); + vflrstat_mask = IXL_GLGEN_VFLRSTAT_MASK(global_vf_num); + vflrstat = rd32(hw, I40E_GLGEN_VFLRSTAT(vflrstat_index)); + if (vflrstat & vflrstat_mask) { + wr32(hw, I40E_GLGEN_VFLRSTAT(vflrstat_index), + vflrstat_mask); + + ixl_reinit_vf(pf, &pf->vfs[i]); + } + } + + icr0 = rd32(hw, I40E_PFINT_ICR0_ENA); + icr0 |= I40E_PFINT_ICR0_ENA_VFLR_MASK; + wr32(hw, I40E_PFINT_ICR0_ENA, icr0); + ixl_flush(hw); +} + +static int +ixl_adminq_err_to_errno(enum i40e_admin_queue_err err) +{ + + switch (err) { + case I40E_AQ_RC_EPERM: + return (EPERM); + case I40E_AQ_RC_ENOENT: + return (ENOENT); + case I40E_AQ_RC_ESRCH: + return (ESRCH); + case I40E_AQ_RC_EINTR: + return (EINTR); + case I40E_AQ_RC_EIO: + return (EIO); + case I40E_AQ_RC_ENXIO: + return (ENXIO); + case I40E_AQ_RC_E2BIG: + return (E2BIG); + case I40E_AQ_RC_EAGAIN: + return (EAGAIN); + case I40E_AQ_RC_ENOMEM: + return (ENOMEM); + case I40E_AQ_RC_EACCES: + return (EACCES); + case I40E_AQ_RC_EFAULT: + return (EFAULT); + case I40E_AQ_RC_EBUSY: + return (EBUSY); + case I40E_AQ_RC_EEXIST: + return (EEXIST); + case I40E_AQ_RC_EINVAL: + return (EINVAL); + case I40E_AQ_RC_ENOTTY: + return (ENOTTY); + case I40E_AQ_RC_ENOSPC: + return (ENOSPC); + case I40E_AQ_RC_ENOSYS: + return (ENOSYS); + case I40E_AQ_RC_ERANGE: + return (ERANGE); + case I40E_AQ_RC_EFLUSHED: + return (EINVAL); /* No exact equivalent in errno.h */ + case I40E_AQ_RC_BAD_ADDR: + return (EFAULT); + case I40E_AQ_RC_EMODE: + return (EPERM); + case I40E_AQ_RC_EFBIG: + return (EFBIG); + default: + return (EINVAL); + } +} + +static int +ixl_if_iov_init(if_ctx_t ctx, uint16_t num_vfs, const nvlist_t *params) +{ + device_t dev; + struct ixl_pf *pf; + struct i40e_hw *hw; + struct ixl_vsi *pf_vsi; + enum i40e_status_code ret; + int i, error; + + dev = iflib_get_dev(ctx); + pf = iflib_get_softc(ctx); + hw = &pf->hw; + pf_vsi = &pf->vsi; + + pf->vfs = malloc(sizeof(struct ixl_vf) * num_vfs, M_IXL, M_NOWAIT | + M_ZERO); + + if (pf->vfs == NULL) { + error = ENOMEM; + goto fail; + } + + for (i = 0; i < num_vfs; i++) + sysctl_ctx_init(&pf->vfs[i].ctx); + + ret = i40e_aq_add_veb(hw, pf_vsi->uplink_seid, pf_vsi->seid, + 1, FALSE, FALSE, &pf->veb_seid, NULL); + if (ret != I40E_SUCCESS) { + error = ixl_adminq_err_to_errno(hw->aq.asq_last_status); + device_printf(dev, "add_veb failed; code=%d error=%d", ret, + error); + goto fail; + } + + ixl_configure_msix(pf); + ixl_enable_adminq(hw); + + pf->num_vfs = num_vfs; + return (0); + +fail: + free(pf->vfs, M_IXL); + pf->vfs = NULL; + return (error); +} + +static void +ixl_if_iov_uninit(if_ctx_t ctx) +{ + struct ixl_pf *pf; + struct i40e_hw *hw; + struct ixl_vsi *vsi; + struct ifnet *ifp; + struct ixl_vf *vfs; + device_t dev; + int i, num_vfs; + + dev = iflib_get_dev(ctx); + pf = iflib_get_softc(ctx); + hw = &pf->hw; + vsi = &pf->vsi; + ifp = vsi->ifp; + + for (i = 0; i < pf->num_vfs; i++) { + if (pf->vfs[i].vsi.seid != 0) + i40e_aq_delete_element(hw, pf->vfs[i].vsi.seid, NULL); + } + + if (pf->veb_seid != 0) { + i40e_aq_delete_element(hw, pf->veb_seid, NULL); + pf->veb_seid = 0; + } + + if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) + ixl_if_intr_disable(ctx); + + vfs = pf->vfs; + num_vfs = pf->num_vfs; + + pf->vfs = NULL; + pf->num_vfs = 0; + + /* Do this after the unlock as sysctl_ctx_free might sleep. */ + for (i = 0; i < num_vfs; i++) + sysctl_ctx_free(&vfs[i].ctx); + free(vfs, M_IXL); +} + +static int +ixl_if_vf_add(if_ctx_t ctx, uint16_t vfnum, const nvlist_t *params) +{ + char sysctl_name[QUEUE_NAME_LEN]; + struct ixl_pf *pf; + struct ixl_vf *vf; + device_t dev; + const void *mac; + size_t size; + int error; + + dev = iflib_get_dev(ctx); + pf = iflib_get_softc(ctx); + vf = &pf->vfs[vfnum]; + + vf->vf_num = vfnum; + + vf->vsi.back = pf; + vf->vf_flags = VF_FLAG_ENABLED; + SLIST_INIT(&vf->vsi.ftl); + + error = ixl_vf_setup_vsi(pf, vf); + if (error != 0) + goto out; + + if (nvlist_exists_binary(params, "mac-addr")) { + mac = nvlist_get_binary(params, "mac-addr", &size); + bcopy(mac, vf->mac, ETHER_ADDR_LEN); + + if (nvlist_get_bool(params, "allow-set-mac")) + vf->vf_flags |= VF_FLAG_SET_MAC_CAP; + } else + /* + * If the administrator has not specified a MAC address then + * we must allow the VF to choose one. + */ + vf->vf_flags |= VF_FLAG_SET_MAC_CAP; + + if (nvlist_get_bool(params, "mac-anti-spoof")) + vf->vf_flags |= VF_FLAG_MAC_ANTI_SPOOF; + + if (nvlist_get_bool(params, "allow-promisc")) + vf->vf_flags |= VF_FLAG_PROMISC_CAP; + + vf->vf_flags |= VF_FLAG_VLAN_CAP; + + ixl_reset_vf(pf, vf); +out: + if (error == 0) { + snprintf(sysctl_name, sizeof(sysctl_name), "vf%d", vfnum); + ixl_add_vsi_sysctls(pf, &vf->vsi, &vf->ctx, sysctl_name); + } + + return (error); +} +#endif /* PCI_IOV */ + +static uint64_t +ixl_if_get_counter(if_ctx_t ctx, ift_counter cnt) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + if_t ifp = iflib_get_ifp(ctx); + + switch (cnt) { + case IFCOUNTER_IPACKETS: + return (vsi->ipackets); + case IFCOUNTER_IERRORS: + return (vsi->ierrors); + case IFCOUNTER_OPACKETS: + return (vsi->opackets); + case IFCOUNTER_OERRORS: + return (vsi->oerrors); + case IFCOUNTER_COLLISIONS: + /* Collisions are by standard impossible in 40G/10G Ethernet */ + return (0); + case IFCOUNTER_IBYTES: + return (vsi->ibytes); + case IFCOUNTER_OBYTES: + return (vsi->obytes); + case IFCOUNTER_IMCASTS: + return (vsi->imcasts); + case IFCOUNTER_OMCASTS: + return (vsi->omcasts); + case IFCOUNTER_IQDROPS: + return (vsi->iqdrops); + case IFCOUNTER_OQDROPS: + return (vsi->oqdrops); + case IFCOUNTER_NOPROTO: + return (vsi->noproto); + default: + return (if_get_counter_default(ifp, cnt)); + } +} + Index: sys/dev/ixl/iflib_if_ixlv.c =================================================================== --- /dev/null +++ sys/dev/ixl/iflib_if_ixlv.c @@ -0,0 +1,2389 @@ +/****************************************************************************** + + Copyright (c) 2013-2015, Intel Corporation + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of the Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +******************************************************************************/ +/*$FreeBSD$*/ + +#ifndef IXL_STANDALONE_BUILD +#include "opt_inet.h" +#include "opt_inet6.h" +#include "opt_rss.h" +#endif + +#include "ixl.h" +#include "ixlv.h" + +#ifdef RSS +#include +#endif + +/********************************************************************* + * Driver version + *********************************************************************/ +char ixlv_driver_version[] = "1.2.6"; + +/********************************************************************* + * PCI Device ID Table + * + * Used by probe to select devices to load on + * Last field stores an index into ixlv_strings + * Last entry must be all 0s + * + * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } + *********************************************************************/ + +static struct pci_vendor_info ixlv_vendor_info_array[] = +{ + PVID(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_VF, "Intel(R) Ethernet Connection XL710 VF Driver"), + PVID(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_VF_HV, "Intel(R) Ethernet Connection XL710 VF HV Driver"), + /* required last entry */ + PVID_END +}; + +/********************************************************************* + * Function prototypes + *********************************************************************/ +static void *ixlv_register(device_t); +static int ixlv_if_attach_pre(if_ctx_t); +static int ixlv_if_attach_post(if_ctx_t); +static int ixlv_if_detach(if_ctx_t); +static int ixlv_if_shutdown(if_ctx_t); +static void ixlv_if_init(if_ctx_t); +static void ixlv_if_stop(if_ctx_t); +static void ixlv_if_intr_enable(if_ctx_t ctx); +static void ixlv_if_intr_disable(if_ctx_t ctx); +static void ixlv_if_queue_intr_enable(if_ctx_t ctx, uint16_t qid); +static void ixlv_if_queue_intr_disable(if_ctx_t ctx, uint16_t qid); +static void ixlv_if_media_status(if_ctx_t, struct ifmediareq *); +static void ixlv_if_timer(if_ctx_t ctx, uint16_t qid); +static int ixlv_if_msix_intr_assign(if_ctx_t ctx, int msix); +static void ixlv_if_vlan_register(if_ctx_t ctx, u16 vtag); +static void ixlv_if_vlan_unregister(if_ctx_t ctx, u16 vtag); +static void ixlv_if_multi_set(if_ctx_t ctx); +static void ixlv_if_update_admin_status(if_ctx_t ctx); +static int ixlv_if_promisc_set(if_ctx_t ctx, int flags); +static int ixlv_if_mtu_set(if_ctx_t ctx, uint32_t mtu); + +static int ixlv_allocate_pci_resources(struct ixlv_sc *); +static void ixlv_free_pci_resources(struct ixlv_sc *); +static void ixlv_config_rss(struct ixlv_sc *); +static int ixlv_setup_interface(if_ctx_t ctx); + +static void ixlv_init_internal(if_ctx_t ctx); +static void ixlv_init_multi(struct ixl_vsi *vsi); +static void ixlv_del_multi(struct ixl_vsi *vsi); + +static int ixlv_add_mac_filter(struct ixlv_sc *, u8 *, u16); +static int ixlv_del_mac_filter(struct ixlv_sc *sc, u8 *macaddr); +static void ixlv_init_filters(struct ixlv_sc *); +static void ixlv_free_filters(struct ixlv_sc *); + +static int ixlv_msix_que(void *); +static int ixlv_msix_adminq(void *); +static void ixlv_do_adminq_locked(struct ixlv_sc *sc); +static int ixlv_reset(struct ixlv_sc *); +static int ixlv_reset_complete(struct i40e_hw *); +static void ixlv_set_queue_rx_itr(struct ixl_rx_queue *); +#ifdef notyet +static void ixlv_set_queue_tx_itr(struct ixl_tx_queue *); +#endif +static void ixl_init_cmd_complete(struct ixl_vc_cmd *, void *, + enum i40e_status_code); + +static void ixlv_enable_adminq_irq(struct i40e_hw *); +static void ixlv_disable_adminq_irq(struct i40e_hw *); + +static void ixlv_setup_vlan_filters(struct ixlv_sc *); + +static void ixlv_init_hw(struct ixlv_sc *); +static int ixlv_setup_vc(struct ixlv_sc *); +static int ixlv_vf_config(struct ixlv_sc *); +#if 0 +static void ixlv_cap_txcsum_tso(struct ixl_vsi *, + struct ifnet *, int); +#endif +static void ixlv_add_sysctls(struct ixlv_sc *); +static int ixlv_sysctl_qtx_tail_handler(SYSCTL_HANDLER_ARGS); +static int ixlv_sysctl_qrx_tail_handler(SYSCTL_HANDLER_ARGS); + +/********************************************************************* + * FreeBSD Device Interface Entry Points + *********************************************************************/ + +static device_method_t ixlv_methods[] = { + /* Device interface */ + DEVMETHOD(device_register, ixlv_register), + DEVMETHOD(device_probe, iflib_device_probe), + DEVMETHOD(device_attach, iflib_device_attach), + DEVMETHOD(device_detach, iflib_device_detach), + DEVMETHOD(device_shutdown, iflib_device_shutdown), + DEVMETHOD_END +}; + +static driver_t ixlv_driver = { + "ixlv", ixlv_methods, sizeof(struct ixlv_sc), +}; + +devclass_t ixlv_devclass; +DRIVER_MODULE(ixlv, pci, ixlv_driver, ixlv_devclass, 0, 0); + +MODULE_DEPEND(ixlv, iflib, 1, 1, 1); +MODULE_DEPEND(ixlv, pci, 1, 1, 1); +MODULE_DEPEND(ixlv, ether, 1, 1, 1); + +static device_method_t ixlv_if_methods[] = { + DEVMETHOD(ifdi_attach_pre, ixlv_if_attach_pre), + DEVMETHOD(ifdi_attach_post, ixlv_if_attach_post), + DEVMETHOD(ifdi_detach, ixlv_if_detach), + DEVMETHOD(ifdi_detach, ixlv_if_shutdown), + DEVMETHOD(ifdi_init, ixlv_if_init), + DEVMETHOD(ifdi_stop, ixlv_if_stop), + DEVMETHOD(ifdi_intr_disable, ixlv_if_intr_disable), + DEVMETHOD(ifdi_intr_enable, ixlv_if_intr_enable), + DEVMETHOD(ifdi_queue_intr_enable, ixlv_if_queue_intr_enable), + DEVMETHOD(ifdi_multi_set, ixlv_if_multi_set), + DEVMETHOD(ifdi_update_admin_status, ixlv_if_update_admin_status), + DEVMETHOD(ifdi_mtu_set, ixlv_if_mtu_set), + DEVMETHOD(ifdi_media_status, ixlv_if_media_status), + DEVMETHOD(ifdi_timer, ixlv_if_timer), + DEVMETHOD(ifdi_promisc_set, ixlv_if_promisc_set), + DEVMETHOD(ifdi_msix_intr_assign, ixlv_if_msix_intr_assign), + DEVMETHOD(ifdi_vlan_register, ixlv_if_vlan_register), + DEVMETHOD(ifdi_vlan_unregister, ixlv_if_vlan_unregister), + + DEVMETHOD(ifdi_media_change, ixl_if_media_change), + DEVMETHOD(ifdi_tx_queues_alloc, ixl_if_tx_queues_alloc), + DEVMETHOD(ifdi_rx_queues_alloc, ixl_if_rx_queues_alloc), + DEVMETHOD(ifdi_queues_free, ixl_if_queues_free), + DEVMETHOD_END +}; + + +static driver_t ixlv_if_driver = { + "ixlv_if", ixlv_if_methods, sizeof(struct ixlv_sc), +}; + + +/* +** TUNEABLE PARAMETERS: +*/ + +static SYSCTL_NODE(_hw, OID_AUTO, ixlv, CTLFLAG_RD, 0, + "IXLV driver parameters"); + +/* +** Number of descriptors per ring: +** - TX and RX are the same size +*/ +static int ixlv_ringsz = DEFAULT_RING; +TUNABLE_INT("hw.ixlv.ringsz", &ixlv_ringsz); +SYSCTL_INT(_hw_ixlv, OID_AUTO, ring_size, CTLFLAG_RDTUN, + &ixlv_ringsz, 0, "Descriptor Ring Size"); + +/* Set to zero to auto calculate */ +int ixlv_max_queues = 0; +TUNABLE_INT("hw.ixlv.max_queues", &ixlv_max_queues); +SYSCTL_INT(_hw_ixlv, OID_AUTO, max_queues, CTLFLAG_RDTUN, + &ixlv_max_queues, 0, "Number of Queues"); + +/* +** Number of entries in Tx queue buf_ring. +** Increasing this will reduce the number of +** errors when transmitting fragmented UDP +** packets. +*/ +static int ixlv_txbrsz = DEFAULT_TXBRSZ; +TUNABLE_INT("hw.ixlv.txbrsz", &ixlv_txbrsz); +SYSCTL_INT(_hw_ixlv, OID_AUTO, txbr_size, CTLFLAG_RDTUN, + &ixlv_txbrsz, 0, "TX Buf Ring Size"); + +/* +** Controls for Interrupt Throttling +** - true/false for dynamic adjustment +** - default values for static ITR +*/ +int ixlv_dynamic_rx_itr = 0; +TUNABLE_INT("hw.ixlv.dynamic_rx_itr", &ixlv_dynamic_rx_itr); +SYSCTL_INT(_hw_ixlv, OID_AUTO, dynamic_rx_itr, CTLFLAG_RDTUN, + &ixlv_dynamic_rx_itr, 0, "Dynamic RX Interrupt Rate"); + +int ixlv_dynamic_tx_itr = 0; +TUNABLE_INT("hw.ixlv.dynamic_tx_itr", &ixlv_dynamic_tx_itr); +SYSCTL_INT(_hw_ixlv, OID_AUTO, dynamic_tx_itr, CTLFLAG_RDTUN, + &ixlv_dynamic_tx_itr, 0, "Dynamic TX Interrupt Rate"); + +int ixlv_rx_itr = IXL_ITR_8K; +TUNABLE_INT("hw.ixlv.rx_itr", &ixlv_rx_itr); +SYSCTL_INT(_hw_ixlv, OID_AUTO, rx_itr, CTLFLAG_RDTUN, + &ixlv_rx_itr, 0, "RX Interrupt Rate"); + +int ixlv_tx_itr = IXL_ITR_4K; +TUNABLE_INT("hw.ixlv.tx_itr", &ixlv_tx_itr); +SYSCTL_INT(_hw_ixlv, OID_AUTO, tx_itr, CTLFLAG_RDTUN, + &ixlv_tx_itr, 0, "TX Interrupt Rate"); + +extern struct if_txrx ixl_txrx; + +static struct if_shared_ctx ixlv_sctx_init = { + .isc_magic = IFLIB_MAGIC, + .isc_q_align = PAGE_SIZE,/* max(DBA_ALIGN, PAGE_SIZE) */ + .isc_tx_maxsize = IXL_TSO_SIZE, + + .isc_tx_maxsegsize = PAGE_SIZE*4, + + .isc_rx_maxsize = PAGE_SIZE*4, + .isc_rx_nsegments = 1, + .isc_rx_maxsegsize = PAGE_SIZE*4, + .isc_ntxd = DEFAULT_RING, + .isc_nrxd = DEFAULT_RING, + .isc_nfl = 1, + .isc_txqsizes[0] = roundup2((DEFAULT_RING * sizeof(struct i40e_tx_desc)) + + sizeof(u32), DBA_ALIGN), + .isc_rxqsizes[0] = roundup2(DEFAULT_RING * + sizeof(union i40e_rx_desc), DBA_ALIGN), + .isc_ntxqs = 1, + .isc_nrxqs = 1, + .isc_admin_intrcnt = 1, + .isc_vendor_info = ixlv_vendor_info_array, + .isc_driver_version = ixlv_driver_version, + .isc_txrx = &ixl_txrx, + .isc_driver = &ixlv_if_driver, +}; + +if_shared_ctx_t ixlv_sctx = &ixlv_sctx_init; + +static void * +ixlv_register(device_t dev) +{ + ixlv_sctx->isc_ntxd = ixlv_ringsz; + ixlv_sctx->isc_nrxd = ixlv_ringsz; + ixlv_sctx->isc_txqsizes[0] = roundup2((ixlv_ringsz * sizeof(struct i40e_tx_desc)) + + sizeof(u32), DBA_ALIGN); + ixlv_sctx->isc_rxqsizes[0] = roundup2(ixlv_ringsz * + sizeof(union i40e_rx_desc), DBA_ALIGN); + + + return (ixlv_sctx); +} + +/********************************************************************* + * Device initialization routine + * + * The attach entry point is called when the driver is being loaded. + * This routine identifies the type of hardware, allocates all resources + * and initializes the hardware. + * + * return 0 on success, positive on failure + *********************************************************************/ + +/* XXX We fail without MSIX support */ +static int +ixlv_if_attach_pre(if_ctx_t ctx) +{ + struct ixlv_sc *sc; + struct i40e_hw *hw; + struct ixl_vsi *vsi; + int error = 0; + device_t dev; + + INIT_DBG_DEV(dev, "begin"); + + /* Allocate, clear, and link in our primary soft structure */ + dev = iflib_get_dev(ctx); + sc = iflib_get_softc(ctx); + sc->dev = sc->osdep.dev = dev; + hw = &sc->hw; + vsi = &sc->vsi; + + /* Initialize hw struct */ + ixlv_init_hw(sc); + + /* Allocate filter lists */ + ixlv_init_filters(sc); + + /* Do PCI setup - map BAR0, etc */ + if (ixlv_allocate_pci_resources(sc)) { + device_printf(dev, "%s: Allocation of PCI resources failed\n", + __func__); + error = ENXIO; + goto err_early; + } + + INIT_DBG_DEV(dev, "Allocated PCI resources and MSIX vectors"); + + error = i40e_set_mac_type(hw); + if (error) { + device_printf(dev, "%s: set_mac_type failed: %d\n", + __func__, error); + goto err_pci_res; + } + + error = ixlv_reset_complete(hw); + if (error) { + device_printf(dev, "%s: Device is still being reset\n", + __func__); + goto err_pci_res; + } + + INIT_DBG_DEV(dev, "VF Device is ready for configuration"); + + error = ixlv_setup_vc(sc); + if (error) { + device_printf(dev, "%s: Error setting up PF comms, %d\n", + __func__, error); + goto err_pci_res; + } + + INIT_DBG_DEV(dev, "PF API version verified"); + + /* TODO: Figure out why MDD events occur when this reset is removed. */ + /* Need API version before sending reset message */ + error = ixlv_reset(sc); + if (error) { + device_printf(dev, "VF reset failed; reload the driver\n"); + goto err_aq; + } + + INIT_DBG_DEV(dev, "VF reset complete"); + + /* Ask for VF config from PF */ + error = ixlv_vf_config(sc); + if (error) { + device_printf(dev, "Error getting configuration from PF: %d\n", + error); + goto err_aq; + } + + INIT_DBG_DEV(dev, "VF config from PF:"); + INIT_DBG_DEV(dev, "VSIs %d, Queues %d, Max Vectors %d, Max MTU %d", + sc->vf_res->num_vsis, + sc->vf_res->num_queue_pairs, + sc->vf_res->max_vectors, + sc->vf_res->max_mtu); + INIT_DBG_DEV(dev, "Offload flags: %#010x", + sc->vf_res->vf_offload_flags); + + // TODO: Move this into ixlv_vf_config? + /* got VF config message back from PF, now we can parse it */ + for (int i = 0; i < sc->vf_res->num_vsis; i++) { + if (sc->vf_res->vsi_res[i].vsi_type == I40E_VSI_SRIOV) + sc->vsi_res = &sc->vf_res->vsi_res[i]; + } + if (!sc->vsi_res) { + device_printf(dev, "%s: no LAN VSI found\n", __func__); + error = EIO; + goto err_res_buf; + } + + INIT_DBG_DEV(dev, "Resource Acquisition complete"); + + /* If no mac address was assigned just make a random one */ + if (!ixlv_check_ether_addr(hw->mac.addr)) { + u8 addr[ETHER_ADDR_LEN]; + arc4rand(&addr, sizeof(addr), 0); + addr[0] &= 0xFE; + addr[0] |= 0x02; + bcopy(addr, hw->mac.addr, sizeof(addr)); + } + + vsi->id = sc->vsi_res->vsi_id; + vsi->back = (void *)sc; + sc->link_up = TRUE; + /* ATTACH_PRE end */ + + return (error); + +err_res_buf: + free(sc->vf_res, M_IXL); +err_aq: + i40e_shutdown_adminq(hw); +err_pci_res: + ixlv_free_pci_resources(sc); +err_early: + ixlv_free_filters(sc); + INIT_DBG_DEV(dev, "end: error %d", error); + return (error); +} + +static int +ixlv_if_attach_post(if_ctx_t ctx) +{ + struct ixlv_sc *sc; + int error = 0; + device_t dev; + struct i40e_hw *hw; + + INIT_DBG_DEV(dev, "begin"); + + /* Allocate, clear, and link in our primary soft structure */ + dev = iflib_get_dev(ctx); + sc = iflib_get_softc(ctx); + hw = &sc->hw; + /* Setup the stack interface */ + if (ixlv_setup_interface(ctx) != 0) { + device_printf(dev, "%s: setup interface failed!\n", + __func__); + error = EIO; + goto out; + } + + INIT_DBG_DEV(dev, "Queue memory and interface setup"); + + /* Initialize stats */ + bzero(&sc->vsi.eth_stats, sizeof(struct i40e_eth_stats)); + ixlv_add_sysctls(sc); + + /* We want AQ enabled early */ + ixlv_enable_adminq_irq(hw); + + /* Set things up to run init */ + sc->init_state = IXLV_INIT_READY; + + ixl_vc_init_mgr(sc, &sc->vc_mgr); + + INIT_DBG_DEV(dev, "end"); + return (0); +out: + ixlv_if_detach(ctx); + return (error); +} + + +/********************************************************************* + * Device removal routine + * + * The detach entry point is called when the driver is being removed. + * This routine stops the adapter and deallocates all the resources + * that were allocated for driver operation. + * + * return 0 on success, positive on failure + *********************************************************************/ + +static int +ixlv_if_detach(if_ctx_t ctx) +{ + struct ixlv_sc *sc = iflib_get_softc(ctx); +#ifdef IXL_DEBUG + device_t dev = iflib_get_dev(ctx); +#endif + INIT_DBG_DEV(dev, "begin"); + + i40e_shutdown_adminq(&sc->hw); + free(sc->vf_res, M_IXL); + ixlv_free_pci_resources(sc); + ixlv_free_filters(sc); + + INIT_DBG_DEV(dev, "end"); + return (0); +} + +/********************************************************************* + * + * Shutdown entry point + * + **********************************************************************/ + +static int +ixlv_if_shutdown(if_ctx_t ctx) +{ +#ifdef IXL_DEBUG + device_t dev = iflib_get_dev(ctx); +#endif + INIT_DBG_DEV(dev, "begin"); + + ixlv_if_stop(ctx); + + INIT_DBG_DEV(dev, "end"); + return (0); +} + +#ifdef notyet +/* + * Configure TXCSUM(IPV6) and TSO(4/6) + * - the hardware handles these together so we + * need to tweak them + */ +static void +ixlv_cap_txcsum_tso(struct ixl_vsi *vsi, struct ifnet *ifp, int mask) +{ + /* Enable/disable TXCSUM/TSO4 */ + if (!(ifp->if_capenable & IFCAP_TXCSUM) + && !(ifp->if_capenable & IFCAP_TSO4)) { + if (mask & IFCAP_TXCSUM) { + ifp->if_capenable |= IFCAP_TXCSUM; + /* enable TXCSUM, restore TSO if previously enabled */ + if (vsi->flags & IXL_FLAGS_KEEP_TSO4) { + vsi->flags &= ~IXL_FLAGS_KEEP_TSO4; + ifp->if_capenable |= IFCAP_TSO4; + } + } + else if (mask & IFCAP_TSO4) { + ifp->if_capenable |= (IFCAP_TXCSUM | IFCAP_TSO4); + vsi->flags &= ~IXL_FLAGS_KEEP_TSO4; + if_printf(ifp, + "TSO4 requires txcsum, enabling both...\n"); + } + } else if((ifp->if_capenable & IFCAP_TXCSUM) + && !(ifp->if_capenable & IFCAP_TSO4)) { + if (mask & IFCAP_TXCSUM) + ifp->if_capenable &= ~IFCAP_TXCSUM; + else if (mask & IFCAP_TSO4) + ifp->if_capenable |= IFCAP_TSO4; + } else if((ifp->if_capenable & IFCAP_TXCSUM) + && (ifp->if_capenable & IFCAP_TSO4)) { + if (mask & IFCAP_TXCSUM) { + vsi->flags |= IXL_FLAGS_KEEP_TSO4; + ifp->if_capenable &= ~(IFCAP_TXCSUM | IFCAP_TSO4); + if_printf(ifp, + "TSO4 requires txcsum, disabling both...\n"); + } else if (mask & IFCAP_TSO4) + ifp->if_capenable &= ~IFCAP_TSO4; + } + + /* Enable/disable TXCSUM_IPV6/TSO6 */ + if (!(ifp->if_capenable & IFCAP_TXCSUM_IPV6) + && !(ifp->if_capenable & IFCAP_TSO6)) { + if (mask & IFCAP_TXCSUM_IPV6) { + ifp->if_capenable |= IFCAP_TXCSUM_IPV6; + if (vsi->flags & IXL_FLAGS_KEEP_TSO6) { + vsi->flags &= ~IXL_FLAGS_KEEP_TSO6; + ifp->if_capenable |= IFCAP_TSO6; + } + } else if (mask & IFCAP_TSO6) { + ifp->if_capenable |= (IFCAP_TXCSUM_IPV6 | IFCAP_TSO6); + vsi->flags &= ~IXL_FLAGS_KEEP_TSO6; + if_printf(ifp, + "TSO6 requires txcsum6, enabling both...\n"); + } + } else if((ifp->if_capenable & IFCAP_TXCSUM_IPV6) + && !(ifp->if_capenable & IFCAP_TSO6)) { + if (mask & IFCAP_TXCSUM_IPV6) + ifp->if_capenable &= ~IFCAP_TXCSUM_IPV6; + else if (mask & IFCAP_TSO6) + ifp->if_capenable |= IFCAP_TSO6; + } else if ((ifp->if_capenable & IFCAP_TXCSUM_IPV6) + && (ifp->if_capenable & IFCAP_TSO6)) { + if (mask & IFCAP_TXCSUM_IPV6) { + vsi->flags |= IXL_FLAGS_KEEP_TSO6; + ifp->if_capenable &= ~(IFCAP_TXCSUM_IPV6 | IFCAP_TSO6); + if_printf(ifp, + "TSO6 requires txcsum6, disabling both...\n"); + } else if (mask & IFCAP_TSO6) + ifp->if_capenable &= ~IFCAP_TSO6; + } +} + +#endif + +/********************************************************************* + * Ioctl entry point + * + * ixlv_ioctl is called when the user wants to configure the + * interface. + * + * return 0 on success, positive on failure + **********************************************************************/ +#if 0 +static int +ixlv_ioctl(struct ifnet *ifp, u_long command, caddr_t data) +{ + struct ixl_vsi *vsi = ifp->if_softc; + struct ixlv_sc *sc = vsi->back; + struct ifreq *ifr = (struct ifreq *)data; +#if defined(INET) || defined(INET6) + struct ifaddr *ifa = (struct ifaddr *)data; + bool avoid_reset = FALSE; +#endif + int error = 0; + + + switch (command) { + + case SIOCSIFADDR: +#ifdef INET + if (ifa->ifa_addr->sa_family == AF_INET) + avoid_reset = TRUE; +#endif +#ifdef INET6 + if (ifa->ifa_addr->sa_family == AF_INET6) + avoid_reset = TRUE; +#endif +#if defined(INET) || defined(INET6) + /* + ** Calling init results in link renegotiation, + ** so we avoid doing it when possible. + */ + if (avoid_reset) { + ifp->if_flags |= IFF_UP; + if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) + ixlv_init(vsi); +#ifdef INET + if (!(ifp->if_flags & IFF_NOARP)) + arp_ifinit(ifp, ifa); +#endif + } else + error = ether_ioctl(ifp, command, data); + break; +#endif + case SIOCSIFMTU: + IOCTL_DBG_IF2(ifp, "SIOCSIFMTU (Set Interface MTU)"); + + break; + + break; + case SIOCADDMULTI: +#if 0 + IOCTL_DBG_IF2(ifp, "SIOCADDMULTI"); + if (ifp->if_drv_flags & IFF_DRV_RUNNING) { + + ixlv_add_multi(vsi); + + } + break; +#endif + case SIOCDELMULTI: + IOCTL_DBG_IF2(ifp, "SIOCDELMULTI"); + if (sc->init_state == IXLV_RUNNING) { + + ixlv_del_multi(vsi); + + } + break; + case SIOCSIFMEDIA: + case SIOCGIFMEDIA: + IOCTL_DBG_IF2(ifp, "SIOCxIFMEDIA (Get/Set Interface Media)"); + error = ifmedia_ioctl(ifp, ifr, sc->media, command); + break; + case SIOCSIFCAP: + { + int mask = ifr->ifr_reqcap ^ ifp->if_capenable; + IOCTL_DBG_IF2(ifp, "SIOCSIFCAP (Set Capabilities)"); + + ixlv_cap_txcsum_tso(vsi, ifp, mask); + + if (mask & IFCAP_RXCSUM) + ifp->if_capenable ^= IFCAP_RXCSUM; + if (mask & IFCAP_RXCSUM_IPV6) + ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; + if (mask & IFCAP_LRO) + ifp->if_capenable ^= IFCAP_LRO; + if (mask & IFCAP_VLAN_HWTAGGING) + ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; + if (mask & IFCAP_VLAN_HWFILTER) + ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; + if (mask & IFCAP_VLAN_HWTSO) + ifp->if_capenable ^= IFCAP_VLAN_HWTSO; + if (ifp->if_drv_flags & IFF_DRV_RUNNING) { + ixlv_init(vsi); + } + VLAN_CAPABILITIES(ifp); + + break; + } + + default: + IOCTL_DBG_IF2(ifp, "UNKNOWN (0x%X)", (int)command); + error = ether_ioctl(ifp, command, data); + break; + } + + return (error); +} +#endif + +/* +** To do a reinit on the VF is unfortunately more complicated +** than a physical device, we must have the PF more or less +** completely recreate our memory, so many things that were +** done only once at attach in traditional drivers now must be +** redone at each reinitialization. This function does that +** 'prelude' so we can then call the normal locked init code. +*/ +int +ixlv_reinit_locked(struct ixlv_sc *sc) +{ + struct i40e_hw *hw = &sc->hw; + struct ixl_vsi *vsi = &sc->vsi; + struct ifnet *ifp = vsi->ifp; + struct ixlv_mac_filter *mf, *mf_temp; + struct ixlv_vlan_filter *vf; + int error = 0; + + INIT_DBG_IF(ifp, "begin"); + + if (ifp->if_drv_flags & IFF_DRV_RUNNING) + ixlv_if_stop(sc->vsi.ctx); + + error = ixlv_reset(sc); + + INIT_DBG_IF(ifp, "VF was reset"); + + /* set the state in case we went thru RESET */ + sc->init_state = IXLV_RUNNING; + + /* + ** Resetting the VF drops all filters from hardware; + ** we need to mark them to be re-added in init. + */ + SLIST_FOREACH_SAFE(mf, sc->mac_filters, next, mf_temp) { + if (mf->flags & IXL_FILTER_DEL) { + SLIST_REMOVE(sc->mac_filters, mf, + ixlv_mac_filter, next); + free(mf, M_IXL); + } else + mf->flags |= IXL_FILTER_ADD; + } + if (vsi->num_vlans != 0) + SLIST_FOREACH(vf, sc->vlan_filters, next) + vf->flags = IXL_FILTER_ADD; + else { /* clean any stale filters */ + while (!SLIST_EMPTY(sc->vlan_filters)) { + vf = SLIST_FIRST(sc->vlan_filters); + SLIST_REMOVE_HEAD(sc->vlan_filters, next); + free(vf, M_IXL); + } + } + + ixlv_enable_adminq_irq(hw); + ixl_vc_flush(&sc->vc_mgr); + + INIT_DBG_IF(ifp, "end"); + return (error); +} + +static void +ixl_init_cmd_complete(struct ixl_vc_cmd *cmd, void *arg, + enum i40e_status_code code) +{ + struct ixlv_sc *sc; + + sc = arg; + + /* + * Ignore "Adapter Stopped" message as that happens if an ifconfig down + * happens while a command is in progress, so we don't print an error + * in that case. + */ + if (code != I40E_SUCCESS && code != I40E_ERR_ADAPTER_STOPPED) { + if_printf(sc->vsi.ifp, + "Error %d waiting for PF to complete operation %d\n", + code, cmd->request); + } +} + +static void +ixlv_init_internal(if_ctx_t ctx) +{ + struct ixlv_sc *sc = iflib_get_softc(ctx); + struct i40e_hw *hw = &sc->hw; + struct ixl_vsi *vsi = &sc->vsi; + struct ixl_tx_queue *tx_que = vsi->tx_queues; + struct ifnet *ifp = vsi->ifp; + int error = 0; + + INIT_DBG_IF(ifp, "begin"); + + /* Do a reinit first if an init has already been done */ + if ((sc->init_state == IXLV_RUNNING) || + (sc->init_state == IXLV_RESET_REQUIRED) || + (sc->init_state == IXLV_RESET_PENDING)) + error = ixlv_reinit_locked(sc); + /* Don't bother with init if we failed reinit */ + if (error) + goto init_done; + + /* Remove existing MAC filter if new MAC addr is set */ + if (bcmp(IF_LLADDR(ifp), hw->mac.addr, ETHER_ADDR_LEN) != 0) { + error = ixlv_del_mac_filter(sc, hw->mac.addr); + if (error == 0) + ixl_vc_enqueue(&sc->vc_mgr, &sc->del_mac_cmd, + IXLV_FLAG_AQ_DEL_MAC_FILTER, ixl_init_cmd_complete, + sc); + } + + /* Check for an LAA mac address... */ + bcopy(IF_LLADDR(ifp), hw->mac.addr, ETHER_ADDR_LEN); + + ifp->if_hwassist = 0; + if (ifp->if_capenable & IFCAP_TSO) + ifp->if_hwassist |= CSUM_TSO; + if (ifp->if_capenable & IFCAP_TXCSUM) + ifp->if_hwassist |= (CSUM_OFFLOAD_IPV4 & ~CSUM_IP); + if (ifp->if_capenable & IFCAP_TXCSUM_IPV6) + ifp->if_hwassist |= CSUM_OFFLOAD_IPV6; + + /* Add mac filter for this VF to PF */ + if (i40e_validate_mac_addr(hw->mac.addr) == I40E_SUCCESS) { + error = ixlv_add_mac_filter(sc, hw->mac.addr, 0); + if (!error || error == EEXIST) + ixl_vc_enqueue(&sc->vc_mgr, &sc->add_mac_cmd, + IXLV_FLAG_AQ_ADD_MAC_FILTER, ixl_init_cmd_complete, + sc); + } + + /* Setup vlan's if needed */ + ixlv_setup_vlan_filters(sc); + + ixlv_init_multi(&sc->vsi); + + /* Prepare the queues for operation */ + for (int i = 0; i < vsi->num_tx_queues; i++, tx_que++) { + ixl_init_tx_ring(vsi, tx_que); + } + + /* Configure queues */ + ixl_vc_enqueue(&sc->vc_mgr, &sc->config_queues_cmd, + IXLV_FLAG_AQ_CONFIGURE_QUEUES, ixl_init_cmd_complete, sc); + + /* Set up RSS */ + ixlv_config_rss(sc); + + /* Map vectors */ + ixl_vc_enqueue(&sc->vc_mgr, &sc->map_vectors_cmd, + IXLV_FLAG_AQ_MAP_VECTORS, ixl_init_cmd_complete, sc); + + /* Enable queues */ + ixl_vc_enqueue(&sc->vc_mgr, &sc->enable_queues_cmd, + IXLV_FLAG_AQ_ENABLE_QUEUES, ixl_init_cmd_complete, sc); + + sc->init_state = IXLV_RUNNING; + +init_done: + INIT_DBG_IF(ifp, "end"); +} + +static void +ixlv_if_init(if_ctx_t ctx) +{ + struct ifnet *ifp; + int retries; + + ixlv_init_internal(ctx); + + ifp = iflib_get_ifp(ctx); + retries = 0; + /* Wait for init to finish */ + while (!(ifp->if_drv_flags & IFF_DRV_RUNNING) + && ++retries < 100) { + i40e_msec_delay(10); + } + if (retries >= IXLV_AQ_MAX_ERR) + if_printf(ifp, + "Init failed to complete in alloted time!\n"); + +} +void +ixlv_init(void *arg) +{ + struct ixlv_sc *sc = arg; + + ixlv_if_init(sc->vsi.ctx); +} +/* + * ixlv_attach() helper function; gathers information about + * the (virtual) hardware for use elsewhere in the driver. + */ +static void +ixlv_init_hw(struct ixlv_sc *sc) +{ + struct i40e_hw *hw = &sc->hw; + device_t dev = sc->dev; + + /* Save off the information about this board */ + hw->vendor_id = pci_get_vendor(dev); + hw->device_id = pci_get_device(dev); + hw->revision_id = pci_read_config(dev, PCIR_REVID, 1); + hw->subsystem_vendor_id = + pci_read_config(dev, PCIR_SUBVEND_0, 2); + hw->subsystem_device_id = + pci_read_config(dev, PCIR_SUBDEV_0, 2); + + hw->bus.device = pci_get_slot(dev); + hw->bus.func = pci_get_function(dev); +} + +/* + * ixlv_attach() helper function; initalizes the admin queue + * and attempts to establish contact with the PF by + * retrying the initial "API version" message several times + * or until the PF responds. + */ +static int +ixlv_setup_vc(struct ixlv_sc *sc) +{ + struct i40e_hw *hw = &sc->hw; + device_t dev = sc->dev; + int error = 0, ret_error = 0, asq_retries = 0; + bool send_api_ver_retried = 0; + + /* Need to set these AQ paramters before initializing AQ */ + hw->aq.num_arq_entries = IXL_AQ_LEN; + hw->aq.num_asq_entries = IXL_AQ_LEN; + hw->aq.arq_buf_size = IXL_AQ_BUFSZ; + hw->aq.asq_buf_size = IXL_AQ_BUFSZ; + + for (int i = 0; i < IXLV_AQ_MAX_ERR; i++) { + /* Initialize admin queue */ + error = i40e_init_adminq(hw); + if (error) { + device_printf(dev, "%s: init_adminq failed: %d\n", + __func__, error); + ret_error = 1; + continue; + } + + INIT_DBG_DEV(dev, "Initialized Admin Queue, attempt %d", i+1); + +retry_send: + /* Send VF's API version */ + error = ixlv_send_api_ver(sc); + if (error) { + i40e_shutdown_adminq(hw); + ret_error = 2; + device_printf(dev, "%s: unable to send api" + " version to PF on attempt %d, error %d\n", + __func__, i+1, error); + } + + asq_retries = 0; + while (!i40e_asq_done(hw)) { + if (++asq_retries > IXLV_AQ_MAX_ERR) { + i40e_shutdown_adminq(hw); + DDPRINTF(dev, "Admin Queue timeout " + "(waiting for send_api_ver), %d more retries...", + IXLV_AQ_MAX_ERR - (i + 1)); + ret_error = 3; + break; + } + i40e_msec_delay(10); + } + if (asq_retries > IXLV_AQ_MAX_ERR) + continue; + + INIT_DBG_DEV(dev, "Sent API version message to PF"); + + /* Verify that the VF accepts the PF's API version */ + error = ixlv_verify_api_ver(sc); + if (error == ETIMEDOUT) { + if (!send_api_ver_retried) { + /* Resend message, one more time */ + send_api_ver_retried++; + device_printf(dev, + "%s: Timeout while verifying API version on first" + " try!\n", __func__); + goto retry_send; + } else { + device_printf(dev, + "%s: Timeout while verifying API version on second" + " try!\n", __func__); + ret_error = 4; + break; + } + } + if (error) { + device_printf(dev, + "%s: Unable to verify API version," + " error %d\n", __func__, error); + ret_error = 5; + } + break; + } + + if (ret_error >= 4) + i40e_shutdown_adminq(hw); + return (ret_error); +} + +/* + * ixlv_attach() helper function; asks the PF for this VF's + * configuration, and saves the information if it receives it. + */ +static int +ixlv_vf_config(struct ixlv_sc *sc) +{ + struct i40e_hw *hw = &sc->hw; + device_t dev = sc->dev; + int bufsz, error = 0, ret_error = 0; + int asq_retries, retried = 0; + +retry_config: + error = ixlv_send_vf_config_msg(sc); + if (error) { + device_printf(dev, + "%s: Unable to send VF config request, attempt %d," + " error %d\n", __func__, retried + 1, error); + ret_error = 2; + } + + asq_retries = 0; + while (!i40e_asq_done(hw)) { + if (++asq_retries > IXLV_AQ_MAX_ERR) { + device_printf(dev, "%s: Admin Queue timeout " + "(waiting for send_vf_config_msg), attempt %d\n", + __func__, retried + 1); + ret_error = 3; + goto fail; + } + i40e_msec_delay(10); + } + + INIT_DBG_DEV(dev, "Sent VF config message to PF, attempt %d", + retried + 1); + + if (!sc->vf_res) { + bufsz = sizeof(struct i40e_virtchnl_vf_resource) + + (I40E_MAX_VF_VSI * sizeof(struct i40e_virtchnl_vsi_resource)); + sc->vf_res = malloc(bufsz, M_IXL, M_NOWAIT); + if (!sc->vf_res) { + device_printf(dev, + "%s: Unable to allocate memory for VF configuration" + " message from PF on attempt %d\n", __func__, retried + 1); + ret_error = 1; + goto fail; + } + } + + /* Check for VF config response */ + error = ixlv_get_vf_config(sc); + if (error == ETIMEDOUT) { + /* The 1st time we timeout, send the configuration message again */ + if (!retried) { + retried++; + goto retry_config; + } + } + if (error) { + device_printf(dev, + "%s: Unable to get VF configuration from PF after %d tries!\n", + __func__, retried + 1); + ret_error = 4; + } + goto done; + +fail: + free(sc->vf_res, M_IXL); +done: + return (ret_error); +} + +#if 0 + /* Enforce the VF max value */ + if (queues > IXLV_MAX_QUEUES) + queues = IXLV_MAX_QUEUES; +#endif + +static int +ixlv_allocate_pci_resources(struct ixlv_sc *sc) +{ + int rid; + device_t dev = sc->dev; + + rid = PCIR_BAR(0); + sc->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, + &rid, RF_ACTIVE); + + if (!(sc->pci_mem)) { + device_printf(dev,"Unable to allocate bus resource: memory\n"); + return (ENXIO); + } + + sc->osdep.mem_bus_space_tag = rman_get_bustag(sc->pci_mem); + sc->osdep.mem_bus_space_handle = rman_get_bushandle(sc->pci_mem); + sc->osdep.mem_bus_space_size = rman_get_size(sc->pci_mem); + sc->osdep.flush_reg = I40E_VFGEN_RSTAT; + sc->hw.hw_addr = (u8 *) &sc->osdep.mem_bus_space_handle; + + sc->hw.back = &sc->osdep; + + /* Disable adminq interrupts */ + ixlv_disable_adminq_irq(&sc->hw); + + return (0); +} + +static void +ixlv_free_pci_resources(struct ixlv_sc *sc) +{ + device_t dev = sc->dev; + + if (sc->pci_mem != NULL) + bus_release_resource(dev, SYS_RES_MEMORY, + PCIR_BAR(0), sc->pci_mem); + +} + +/* +** Requests a VF reset from the PF. +** +** Requires the VF's Admin Queue to be initialized. +*/ +static int +ixlv_reset(struct ixlv_sc *sc) +{ + struct i40e_hw *hw = &sc->hw; + device_t dev = sc->dev; + int error = 0; + + /* Ask the PF to reset us if we are initiating */ + if (sc->init_state != IXLV_RESET_PENDING) + ixlv_request_reset(sc); + + i40e_msec_delay(100); + error = ixlv_reset_complete(hw); + if (error) { + device_printf(dev, "%s: VF reset failed\n", + __func__); + return (error); + } + + error = i40e_shutdown_adminq(hw); + if (error) { + device_printf(dev, "%s: shutdown_adminq failed: %d\n", + __func__, error); + return (error); + } + + error = i40e_init_adminq(hw); + if (error) { + device_printf(dev, "%s: init_adminq failed: %d\n", + __func__, error); + return(error); + } + + return (0); +} + +static int +ixlv_reset_complete(struct i40e_hw *hw) +{ + u32 reg; + + for (int i = 0; i < 100; i++) { + reg = rd32(hw, I40E_VFGEN_RSTAT) & + I40E_VFGEN_RSTAT_VFR_STATE_MASK; + + if ((reg == I40E_VFR_VFACTIVE) || + (reg == I40E_VFR_COMPLETED)) + return (0); + i40e_msec_delay(100); + } + + return (EBUSY); +} + +/********************************************************************* + * + * Setup MSIX Interrupt resources and handlers for the VSI + * + **********************************************************************/ +static int +ixlv_if_msix_intr_assign(if_ctx_t ctx, int msix) +{ + struct ixl_vsi *vsi = iflib_get_softc(ctx); + struct ixl_pf *pf = vsi->back; + struct ixl_rx_queue *que = vsi->rx_queues; + struct ixl_tx_queue *tx_que = vsi->tx_queues; + int err, rid, vector = 0; + char buf[16]; + + /* Admin Que is vector 0*/ + rid = vector + 1; + + err = iflib_irq_alloc_generic(ctx, &vsi->irq, rid, IFLIB_INTR_ADMIN, + ixlv_msix_adminq, pf, 0, "aq"); + if (err) { + iflib_irq_free(ctx, &vsi->irq); + device_printf(iflib_get_dev(ctx), "Failed to register Admin que handler"); + return (err); + } + ++vector; + iflib_softirq_alloc_generic(ctx, rid, IFLIB_INTR_IOV, pf, 0, "ixl_iov"); + + /* Now set up the stations */ + for (int i = 0; i < vsi->num_rx_queues; i++, vector++, que++) { + rid = vector + 1; + + snprintf(buf, sizeof(buf), "rxq%d", i); + err = iflib_irq_alloc_generic(ctx, &que->que_irq, rid, IFLIB_INTR_RX, + ixlv_msix_que, que, que->rxr.me, buf); + if (err) { + device_printf(iflib_get_dev(ctx), "Failed to allocate q int %d err: %d", i, err); + vsi->num_rx_queues = i + 1; + goto fail; + } + que->msix = vector; + } + for (int i = 0; i < vsi->num_tx_queues; i++, tx_que++) { + snprintf(buf, sizeof(buf), "txq%d", i); + iflib_softirq_alloc_generic(ctx, i + 1, IFLIB_INTR_TX, tx_que, tx_que->txr.me, buf); + } + + return (0); +fail: + iflib_irq_free(ctx, &vsi->irq); + que = vsi->rx_queues; + for (int i = 0; i < vsi->num_rx_queues; i++, que++) + iflib_irq_free(ctx, &que->que_irq); + return (err); +} + +/********************************************************************* + * + * Setup networking device structure and register an interface. + * + **********************************************************************/ +static int +ixlv_setup_interface(if_ctx_t ctx) +{ + struct ifnet *ifp; + struct ixlv_sc *sc = iflib_get_softc(ctx); + struct ixl_vsi *vsi = &sc->vsi; + + INIT_DBG_DEV(dev, "begin"); + + ifp = vsi->ifp = iflib_get_ifp(ctx); + + if_setbaudrate(ifp, 4000000000); + ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; + vsi->max_frame_size = + ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN + + ETHER_VLAN_ENCAP_LEN; + + /* + * Tell the upper layer(s) we support long frames. + */ + ifp->if_hdrlen = sizeof(struct ether_vlan_header); + + ifp->if_capabilities |= IFCAP_HWCSUM; + ifp->if_capabilities |= IFCAP_HWCSUM_IPV6; + ifp->if_capabilities |= IFCAP_TSO; + ifp->if_capabilities |= IFCAP_JUMBO_MTU; + + ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING + | IFCAP_VLAN_HWTSO + | IFCAP_VLAN_MTU + | IFCAP_VLAN_HWCSUM + | IFCAP_LRO; + ifp->if_capenable = ifp->if_capabilities; + + /* + ** Don't turn this on by default, if vlans are + ** created on another pseudo device (eg. lagg) + ** then vlan events are not passed thru, breaking + ** operation, but with HW FILTER off it works. If + ** using vlans directly on the ixl driver you can + ** enable this and get full hardware tag filtering. + */ + ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; + + // JFV Add media types later? + + ifmedia_add(sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); + ifmedia_set(sc->media, IFM_ETHER | IFM_AUTO); + + INIT_DBG_DEV(dev, "end"); + return (0); +} + +/* +** This routine is run via an vlan config EVENT, +** it enables us to use the HW Filter table since +** we can get the vlan id. This just creates the +** entry in the soft version of the VFTA, init will +** repopulate the real table. +*/ +static void +ixlv_if_vlan_register(if_ctx_t ctx, u16 vtag) +{ + struct ixlv_sc *sc = iflib_get_softc(ctx); + struct ixl_vsi *vsi = &sc->vsi; + struct ixlv_vlan_filter *v; + + + if ((vtag == 0) || (vtag > 4095)) /* Invalid */ + return; + + /* Sanity check - make sure it doesn't already exist */ + SLIST_FOREACH(v, sc->vlan_filters, next) { + if (v->vlan == vtag) + return; + } + + ++vsi->num_vlans; + /* should either fail or be M_WAITOK XXX */ + v = malloc(sizeof(struct ixlv_vlan_filter), M_IXL, M_NOWAIT | M_ZERO); + + SLIST_INSERT_HEAD(sc->vlan_filters, v, next); + v->vlan = vtag; + v->flags = IXL_FILTER_ADD; + ixl_vc_enqueue(&sc->vc_mgr, &sc->add_vlan_cmd, + IXLV_FLAG_AQ_ADD_VLAN_FILTER, ixl_init_cmd_complete, sc); +} + +/* +** This routine is run via an vlan +** unconfig EVENT, remove our entry +** in the soft vfta. +*/ +static void +ixlv_if_vlan_unregister(if_ctx_t ctx, u16 vtag) +{ + struct ixlv_sc *sc = iflib_get_softc(ctx); + struct ixl_vsi *vsi = &sc->vsi; + struct ixlv_vlan_filter *v; + int i = 0; + if ((vtag == 0) || (vtag > 4095)) /* Invalid */ + return; + + SLIST_FOREACH(v, sc->vlan_filters, next) { + if (v->vlan == vtag) { + v->flags = IXL_FILTER_DEL; + ++i; + --vsi->num_vlans; + } + } + if (i) + ixl_vc_enqueue(&sc->vc_mgr, &sc->del_vlan_cmd, + IXLV_FLAG_AQ_DEL_VLAN_FILTER, ixl_init_cmd_complete, sc); +} + +/* +** Get a new filter and add it to the mac filter list. +*/ +static struct ixlv_mac_filter * +ixlv_get_mac_filter(struct ixlv_sc *sc) +{ + struct ixlv_mac_filter *f; + + f = malloc(sizeof(struct ixlv_mac_filter), + M_IXL, M_NOWAIT | M_ZERO); + if (f) + SLIST_INSERT_HEAD(sc->mac_filters, f, next); + + return (f); +} + +/* +** Find the filter with matching MAC address +*/ +static struct ixlv_mac_filter * +ixlv_find_mac_filter(struct ixlv_sc *sc, u8 *macaddr) +{ + struct ixlv_mac_filter *f; + bool match = FALSE; + + SLIST_FOREACH(f, sc->mac_filters, next) { + if (cmp_etheraddr(f->macaddr, macaddr)) { + match = TRUE; + break; + } + } + + if (!match) + f = NULL; + return (f); +} + +/* +** Admin Queue interrupt handler +*/ +static int +ixlv_msix_adminq(void *arg) +{ + struct ixlv_sc *sc = arg; + struct i40e_hw *hw = &sc->hw; + u32 reg, mask; + + reg = rd32(hw, I40E_VFINT_ICR01); + mask = rd32(hw, I40E_VFINT_ICR0_ENA1); + + reg = rd32(hw, I40E_VFINT_DYN_CTL01); + reg |= I40E_VFINT_DYN_CTL01_CLEARPBA_MASK; + wr32(hw, I40E_VFINT_DYN_CTL01, reg); + iflib_admin_intr_deferred(sc->vsi.ctx); + return (FILTER_HANDLED); +} + +void +ixlv_if_intr_enable(if_ctx_t ctx) +{ + struct ixlv_sc *sc = iflib_get_softc(ctx); + struct ixl_vsi *vsi = &sc->vsi; + struct i40e_hw *hw = vsi->hw; + struct ixl_rx_queue *que = vsi->rx_queues; + + ixlv_enable_adminq_irq(hw); + for (int i = 0; i < vsi->num_rx_queues; i++, que++) + ixlv_if_queue_intr_enable(ctx, que->rxr.me); +} + +void +ixlv_enable_intr(struct ixl_vsi *vsi) +{ + ixlv_if_intr_enable(vsi->ctx); +} + +void +ixlv_if_intr_disable(if_ctx_t ctx) +{ + struct ixlv_sc *sc = iflib_get_softc(ctx); + struct ixl_vsi *vsi = &sc->vsi; + struct i40e_hw *hw = vsi->hw; + struct ixl_rx_queue *que = vsi->rx_queues; + + ixlv_disable_adminq_irq(hw); + for (int i = 0; i < vsi->num_rx_queues; i++, que++) + ixlv_if_queue_intr_disable(ctx, que->rxr.me); +} + +void +ixlv_disable_intr(struct ixl_vsi *vsi) +{ + ixlv_if_intr_disable(vsi->ctx); +} + +static void +ixlv_disable_adminq_irq(struct i40e_hw *hw) +{ + wr32(hw, I40E_VFINT_DYN_CTL01, 0); + wr32(hw, I40E_VFINT_ICR0_ENA1, 0); + /* flush */ + rd32(hw, I40E_VFGEN_RSTAT); + return; +} + +static void +ixlv_enable_adminq_irq(struct i40e_hw *hw) +{ + wr32(hw, I40E_VFINT_DYN_CTL01, + I40E_VFINT_DYN_CTL01_INTENA_MASK | + I40E_VFINT_DYN_CTL01_ITR_INDX_MASK); + wr32(hw, I40E_VFINT_ICR0_ENA1, I40E_VFINT_ICR0_ENA1_ADMINQ_MASK); + /* flush */ + rd32(hw, I40E_VFGEN_RSTAT); + return; +} + +static void +ixlv_if_queue_intr_enable(if_ctx_t ctx, uint16_t id) +{ + struct ixlv_sc *sc = iflib_get_softc(ctx); + struct ixl_vsi *vsi = &sc->vsi; + struct i40e_hw *hw = vsi->hw; + u32 reg; + + reg = I40E_VFINT_DYN_CTLN1_INTENA_MASK | + I40E_VFINT_DYN_CTLN1_CLEARPBA_MASK; + wr32(hw, I40E_VFINT_DYN_CTLN1(id), reg); +} + +static void +ixlv_if_queue_intr_disable(if_ctx_t ctx, uint16_t qid) +{ + struct ixlv_sc *sc = iflib_get_softc(ctx); + struct ixl_vsi *vsi = &sc->vsi; + struct i40e_hw *hw = vsi->hw; + + wr32(hw, I40E_VFINT_DYN_CTLN1(qid), 0); + rd32(hw, I40E_VFGEN_RSTAT); +} + + +/* +** Provide a update to the queue RX +** interrupt moderation value. +*/ +static void +ixlv_set_queue_rx_itr(struct ixl_rx_queue *que) +{ + struct ixl_vsi *vsi = que->vsi; + struct i40e_hw *hw = vsi->hw; + struct rx_ring *rxr = &que->rxr; + u16 rx_itr; + u16 rx_latency = 0; + int rx_bytes; + + + /* Idle, do nothing */ + if (rxr->bytes == 0) + return; + + if (ixlv_dynamic_rx_itr) { + rx_bytes = rxr->bytes/rxr->itr; + rx_itr = rxr->itr; + + /* Adjust latency range */ + switch (rxr->latency) { + case IXL_LOW_LATENCY: + if (rx_bytes > 10) { + rx_latency = IXL_AVE_LATENCY; + rx_itr = IXL_ITR_20K; + } + break; + case IXL_AVE_LATENCY: + if (rx_bytes > 20) { + rx_latency = IXL_BULK_LATENCY; + rx_itr = IXL_ITR_8K; + } else if (rx_bytes <= 10) { + rx_latency = IXL_LOW_LATENCY; + rx_itr = IXL_ITR_100K; + } + break; + case IXL_BULK_LATENCY: + if (rx_bytes <= 20) { + rx_latency = IXL_AVE_LATENCY; + rx_itr = IXL_ITR_20K; + } + break; + } + + rxr->latency = rx_latency; + + if (rx_itr != rxr->itr) { + /* do an exponential smoothing */ + rx_itr = (10 * rx_itr * rxr->itr) / + ((9 * rx_itr) + rxr->itr); + rxr->itr = rx_itr & IXL_MAX_ITR; + wr32(hw, I40E_VFINT_ITRN1(IXL_RX_ITR, + que->rxr.me), rxr->itr); + } + } else { /* We may have have toggled to non-dynamic */ + if (vsi->rx_itr_setting & IXL_ITR_DYNAMIC) + vsi->rx_itr_setting = ixlv_rx_itr; + /* Update the hardware if needed */ + if (rxr->itr != vsi->rx_itr_setting) { + rxr->itr = vsi->rx_itr_setting; + wr32(hw, I40E_VFINT_ITRN1(IXL_RX_ITR, + que->rxr.me), rxr->itr); + } + } + rxr->bytes = 0; + rxr->packets = 0; + return; +} + + +#ifdef notyet +/* +** Provide a update to the queue TX +** interrupt moderation value. +*/ +static void +ixlv_set_queue_tx_itr(struct ixl_tx_queue *que) +{ + struct ixl_vsi *vsi = que->vsi; + struct i40e_hw *hw = vsi->hw; + struct tx_ring *txr = &que->txr; + u16 tx_itr; + u16 tx_latency = 0; + int tx_bytes; + + + /* Idle, do nothing */ + if (txr->bytes == 0) + return; + + if (ixlv_dynamic_tx_itr) { + tx_bytes = txr->bytes/txr->itr; + tx_itr = txr->itr; + + switch (txr->latency) { + case IXL_LOW_LATENCY: + if (tx_bytes > 10) { + tx_latency = IXL_AVE_LATENCY; + tx_itr = IXL_ITR_20K; + } + break; + case IXL_AVE_LATENCY: + if (tx_bytes > 20) { + tx_latency = IXL_BULK_LATENCY; + tx_itr = IXL_ITR_8K; + } else if (tx_bytes <= 10) { + tx_latency = IXL_LOW_LATENCY; + tx_itr = IXL_ITR_100K; + } + break; + case IXL_BULK_LATENCY: + if (tx_bytes <= 20) { + tx_latency = IXL_AVE_LATENCY; + tx_itr = IXL_ITR_20K; + } + break; + } + + txr->latency = tx_latency; + + if (tx_itr != txr->itr) { + /* do an exponential smoothing */ + tx_itr = (10 * tx_itr * txr->itr) / + ((9 * tx_itr) + txr->itr); + txr->itr = tx_itr & IXL_MAX_ITR; + wr32(hw, I40E_VFINT_ITRN1(IXL_TX_ITR, + que->txr.me), txr->itr); + } + + } else { /* We may have have toggled to non-dynamic */ + if (vsi->tx_itr_setting & IXL_ITR_DYNAMIC) + vsi->tx_itr_setting = ixlv_tx_itr; + /* Update the hardware if needed */ + if (txr->itr != vsi->tx_itr_setting) { + txr->itr = vsi->tx_itr_setting; + wr32(hw, I40E_VFINT_ITRN1(IXL_TX_ITR, + que->txr.me), txr->itr); + } + } + txr->bytes = 0; + txr->packets = 0; + return; +} +#endif + +/********************************************************************* + * + * MSIX Queue Interrupt Service routine + * + **********************************************************************/ +static int +ixlv_msix_que(void *arg) +{ + struct ixl_rx_queue *que = arg; + + ixlv_set_queue_rx_itr(que); +#ifdef notyet + ixlv_set_queue_tx_itr(que); +#endif + return (FILTER_SCHEDULE_THREAD); +} + + +/* XXX */ +void +ixlv_update_link_status(struct ixlv_sc *sc) +{ + /* satisfy linker for the moment */ + ; +} + + +/********************************************************************* + * + * Media Ioctl callback + * + * This routine is called whenever the user queries the status of + * the interface using ifconfig. + * + **********************************************************************/ +static void +ixlv_if_media_status(if_ctx_t ctx, struct ifmediareq * ifmr) +{ + struct ixlv_sc *sc = iflib_get_softc(ctx); + INIT_DBG_IF(ifp, "begin"); + + + ixlv_update_link_status(sc); + + ifmr->ifm_status = IFM_AVALID; + ifmr->ifm_active = IFM_ETHER; + + if (!sc->link_up) { + INIT_DBG_IF(ifp, "end: link not up"); + return; + } + + ifmr->ifm_status |= IFM_ACTIVE; + /* Hardware is always full-duplex */ + ifmr->ifm_active |= IFM_FDX; + INIT_DBG_IF(ifp, "end"); +} + +/********************************************************************* + * Multicast Initialization + * + * This routine is called by init to reset a fresh state. + * + **********************************************************************/ + +static void +ixlv_init_multi(struct ixl_vsi *vsi) +{ + struct ixlv_mac_filter *f; + struct ixlv_sc *sc = vsi->back; + int mcnt = 0; + + IOCTL_DBG_IF(vsi->ifp, "begin"); + + /* First clear any multicast filters */ + SLIST_FOREACH(f, sc->mac_filters, next) { + if ((f->flags & IXL_FILTER_USED) + && (f->flags & IXL_FILTER_MC)) { + f->flags |= IXL_FILTER_DEL; + mcnt++; + } + } + if (mcnt > 0) + ixl_vc_enqueue(&sc->vc_mgr, &sc->del_multi_cmd, + IXLV_FLAG_AQ_DEL_MAC_FILTER, ixl_init_cmd_complete, + sc); + + IOCTL_DBG_IF(vsi->ifp, "end"); +} + +static int +ixlv_mc_filter_apply(void *arg, struct ifmultiaddr *ifma, int count __unused) +{ + struct ixlv_sc *sc = arg; + + if (ifma->ifma_addr->sa_family != AF_LINK) + return (0); + if (!ixlv_add_mac_filter(sc, + (u8*)LLADDR((struct sockaddr_dl *) ifma->ifma_addr), + IXL_FILTER_MC)) + return (1); + + return (0); +} +static void +ixlv_if_multi_set(if_ctx_t ctx) +{ + struct ixlv_sc *sc = iflib_get_softc(ctx); + int mcnt = 0; + + IOCTL_DBG_IF(ifp, "begin"); + + mcnt = if_multiaddr_count(iflib_get_ifp(ctx), MAX_MULTICAST_ADDR); + /* XXX */ + ixlv_del_multi(&sc->vsi); + + if (__predict_false(mcnt >= MAX_MULTICAST_ADDR)) { + IOCTL_DEBUGOUT("%s: end: too many filters - no promiscuous mode in VF", __func__); + return; + } + mcnt = if_multi_apply(iflib_get_ifp(ctx), ixlv_mc_filter_apply, sc); + /* + ** Notify AQ task that sw filters need to be + ** added to hw list + */ + if (mcnt > 0) + ixl_vc_enqueue(&sc->vc_mgr, &sc->add_multi_cmd, + IXLV_FLAG_AQ_ADD_MAC_FILTER, ixl_init_cmd_complete, + sc); + + IOCTL_DBG_IF(ifp, "end"); +} + +static int +ixlv_if_promisc_set(if_ctx_t ctx, int flags) +{ + if (flags & (IFF_ALLMULTI|IFF_PROMISC)) + return (EINVAL); + return (0); +} + +static int +ixlv_if_mtu_set(if_ctx_t ctx, uint32_t mtu) +{ + struct ixlv_sc *sc; + + sc = iflib_get_softc(ctx); + if (mtu > IXL_MAX_FRAME - ETHER_HDR_LEN - ETHER_CRC_LEN - ETHER_VLAN_ENCAP_LEN) + return (EINVAL); + else + sc->vsi.max_frame_size = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN + + ETHER_VLAN_ENCAP_LEN; + return (0); +} + + +static void +ixlv_del_multi(struct ixl_vsi *vsi) +{ + struct ixlv_mac_filter *f; + struct ifmultiaddr *ifma; + struct ifnet *ifp = vsi->ifp; + struct ixlv_sc *sc = vsi->back; + int mcnt = 0; + bool match = FALSE; + + IOCTL_DBG_IF(ifp, "begin"); + + /* Search for removed multicast addresses */ + if_maddr_rlock(ifp); + SLIST_FOREACH(f, sc->mac_filters, next) { + if ((f->flags & IXL_FILTER_USED) + && (f->flags & IXL_FILTER_MC)) { + /* check if mac address in filter is in sc's list */ + match = FALSE; + TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + if (ifma->ifma_addr->sa_family != AF_LINK) + continue; + u8 *mc_addr = + (u8 *)LLADDR((struct sockaddr_dl *)ifma->ifma_addr); + if (cmp_etheraddr(f->macaddr, mc_addr)) { + match = TRUE; + break; + } + } + /* if this filter is not in the sc's list, remove it */ + if (match == FALSE && !(f->flags & IXL_FILTER_DEL)) { + f->flags |= IXL_FILTER_DEL; + mcnt++; + IOCTL_DBG_IF(ifp, "marked: " MAC_FORMAT, + MAC_FORMAT_ARGS(f->macaddr)); + } + else if (match == FALSE) + IOCTL_DBG_IF(ifp, "exists: " MAC_FORMAT, + MAC_FORMAT_ARGS(f->macaddr)); + } + } + if_maddr_runlock(ifp); + + if (mcnt > 0) + ixl_vc_enqueue(&sc->vc_mgr, &sc->del_multi_cmd, + IXLV_FLAG_AQ_DEL_MAC_FILTER, ixl_init_cmd_complete, + sc); + + IOCTL_DBG_IF(ifp, "end"); +} + +/********************************************************************* + * Timer routine + * + * This routine checks for link status,updates statistics, + * and runs the watchdog check. + * + **********************************************************************/ + +static void +ixlv_if_timer(if_ctx_t ctx, uint16_t qid) +{ + struct ixlv_sc *sc = iflib_get_softc(ctx); + struct i40e_hw *hw = &sc->hw; + struct ixl_vsi *vsi = &sc->vsi; + struct ixl_tx_queue *que = &vsi->tx_queues[qid]; + u32 mask, val; + + /* If Reset is in progress just bail */ + if (sc->init_state == IXLV_RESET_PENDING) + return; + mask = (I40E_VFINT_DYN_CTLN1_INTENA_MASK | + I40E_VFINT_DYN_CTLN1_SWINT_TRIG_MASK); + + /* Any queues with outstanding work get a sw irq */ + /* should be set by encap */ + if (que->busy) + wr32(hw, I40E_VFINT_DYN_CTLN1(que->txr.me), mask); + + if (qid != 0) + return; + + /* Check for when PF triggers a VF reset */ + val = rd32(hw, I40E_VFGEN_RSTAT) & + I40E_VFGEN_RSTAT_VFR_STATE_MASK; + + if (val != I40E_VFR_VFACTIVE + && val != I40E_VFR_COMPLETED) { + DDPRINTF(dev, "reset in progress! (%d)", val); + return; + } + + ixlv_request_stats(sc); +} + +/* +** Note: this routine updates the OS on the link state +** the real check of the hardware only happens with +** a link interrupt. +*/ +static void +ixlv_if_update_admin_status(if_ctx_t ctx) +{ + struct ixlv_sc *sc = iflib_get_softc(ctx); + struct ifnet *ifp = iflib_get_ifp(ctx); + struct ixl_vsi *vsi = &sc->vsi; + + if (sc->link_up){ + if (vsi->link_active == FALSE) { + if (bootverbose) + if_printf(ifp,"Link is Up, %d Gbps\n", + (sc->link_speed == I40E_LINK_SPEED_40GB) ? 40:10); + vsi->link_active = TRUE; + if_link_state_change(ifp, LINK_STATE_UP); + } + } else { /* Link down */ + if (vsi->link_active == TRUE) { + if (bootverbose) + if_printf(ifp,"Link is Down\n"); + if_link_state_change(ifp, LINK_STATE_DOWN); + vsi->link_active = FALSE; + } + } + + return; +} + +/********************************************************************* + * + * This routine disables all traffic on the adapter by issuing a + * global reset on the MAC and deallocates TX/RX buffers. + * + **********************************************************************/ + +static void +ixlv_if_stop(if_ctx_t ctx) +{ + struct ifnet *ifp; + struct ixlv_sc *sc; + int start; + + ifp = iflib_get_ifp(ctx); + sc = iflib_get_softc(ctx); + INIT_DBG_IF(ifp, "begin"); + + ixl_vc_flush(&sc->vc_mgr); + + start = ticks; + while ((ifp->if_drv_flags & IFF_DRV_RUNNING) && + ((ticks - start) < hz/10)) + ixlv_do_adminq_locked(sc); + + INIT_DBG_IF(ifp, "end"); +} + +/* +** ixlv_config_rss - setup RSS +** +** RSS keys and table are cleared on VF reset. +*/ +static void +ixlv_config_rss(struct ixlv_sc *sc) +{ + struct i40e_hw *hw = &sc->hw; + struct ixl_vsi *vsi = &sc->vsi; + u32 lut = 0; + u64 set_hena = 0, hena; + int i, j, que_id; +#ifdef RSS + u32 rss_hash_config; + u32 rss_seed[IXL_KEYSZ]; +#else + u32 rss_seed[IXL_KEYSZ] = {0x41b01687, + 0x183cfd8c, 0xce880440, 0x580cbc3c, + 0x35897377, 0x328b25e1, 0x4fa98922, + 0xb7d90c14, 0xd5bad70d, 0xcd15a2c1}; +#endif + + /* Don't set up RSS if using a single queue */ + if (vsi->num_rx_queues == 1) { + wr32(hw, I40E_VFQF_HENA(0), 0); + wr32(hw, I40E_VFQF_HENA(1), 0); + ixl_flush(hw); + return; + } + +#ifdef RSS + /* Fetch the configured RSS key */ + rss_getkey((uint8_t *) &rss_seed); +#endif + /* Fill out hash function seed */ + for (i = 0; i <= IXL_KEYSZ; i++) + wr32(hw, I40E_VFQF_HKEY(i), rss_seed[i]); + + /* Enable PCTYPES for RSS: */ +#ifdef RSS + rss_hash_config = rss_gethashconfig(); + if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4) + set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER); + if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4) + set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP); + if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4) + set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP); + if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6) + set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER); + if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX) + set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6); + if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6) + set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP); + if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6) + set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP); +#else + set_hena = + ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP) | + ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP) | + ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) | + ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) | + ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV4) | + ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP) | + ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP) | + ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) | + ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) | + ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6) | + ((u64)1 << I40E_FILTER_PCTYPE_L2_PAYLOAD); +#endif + hena = (u64)rd32(hw, I40E_VFQF_HENA(0)) | + ((u64)rd32(hw, I40E_VFQF_HENA(1)) << 32); + hena |= set_hena; + wr32(hw, I40E_VFQF_HENA(0), (u32)hena); + wr32(hw, I40E_VFQF_HENA(1), (u32)(hena >> 32)); + + /* Populate the LUT with max no. of queues in round robin fashion */ + for (i = 0, j = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++, j++) { + if (j == vsi->num_rx_queues) + j = 0; +#ifdef RSS + /* + * Fetch the RSS bucket id for the given indirection entry. + * Cap it at the number of configured buckets (which is + * num_rx_queues.) + */ + que_id = rss_get_indirection_to_bucket(i); + que_id = que_id % vsi->num_rx_queues; +#else + que_id = j; +#endif + /* lut = 4-byte sliding window of 4 lut entries */ + lut = (lut << 8) | (que_id & 0xF); + /* On i = 3, we have 4 entries in lut; write to the register */ + if ((i & 3) == 3) { + wr32(hw, I40E_VFQF_HLUT(i), lut); + DDPRINTF(sc->dev, "HLUT(%2d): %#010x", i, lut); + } + } + ixl_flush(hw); +} + + +/* +** This routine refreshes vlan filters, called by init +** it scans the filter table and then updates the AQ +*/ +static void +ixlv_setup_vlan_filters(struct ixlv_sc *sc) +{ + struct ixl_vsi *vsi = &sc->vsi; + struct ixlv_vlan_filter *f; + int cnt = 0; + + if (vsi->num_vlans == 0) + return; + /* + ** Scan the filter table for vlan entries, + ** and if found call for the AQ update. + */ + SLIST_FOREACH(f, sc->vlan_filters, next) + if (f->flags & IXL_FILTER_ADD) + cnt++; + if (cnt > 0) + ixl_vc_enqueue(&sc->vc_mgr, &sc->add_vlan_cmd, + IXLV_FLAG_AQ_ADD_VLAN_FILTER, ixl_init_cmd_complete, sc); +} + + +/* +** This routine adds new MAC filters to the sc's list; +** these are later added in hardware by sending a virtual +** channel message. +*/ +static int +ixlv_add_mac_filter(struct ixlv_sc *sc, u8 *macaddr, u16 flags) +{ + struct ixlv_mac_filter *f; + + /* Does one already exist? */ + f = ixlv_find_mac_filter(sc, macaddr); + if (f != NULL) { + IDPRINTF(sc->vsi.ifp, "exists: " MAC_FORMAT, + MAC_FORMAT_ARGS(macaddr)); + return (EEXIST); + } + + /* If not, get a new empty filter */ + f = ixlv_get_mac_filter(sc); + if (f == NULL) { + if_printf(sc->vsi.ifp, "%s: no filters available!!\n", + __func__); + return (ENOMEM); + } + + IDPRINTF(sc->vsi.ifp, "marked: " MAC_FORMAT, + MAC_FORMAT_ARGS(macaddr)); + + bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN); + f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED); + f->flags |= flags; + return (0); +} + +/* +** Marks a MAC filter for deletion. +*/ +static int +ixlv_del_mac_filter(struct ixlv_sc *sc, u8 *macaddr) +{ + struct ixlv_mac_filter *f; + + f = ixlv_find_mac_filter(sc, macaddr); + if (f == NULL) + return (ENOENT); + + f->flags |= IXL_FILTER_DEL; + return (0); +} + +static void +ixlv_do_adminq_locked(struct ixlv_sc *sc) +{ + struct i40e_hw *hw = &sc->hw; + struct i40e_arq_event_info event; + struct i40e_virtchnl_msg *v_msg; + device_t dev = sc->dev; + u16 result = 0; + u32 reg, oldreg; + i40e_status ret; + + event.buf_len = IXL_AQ_BUF_SZ; + event.msg_buf = sc->aq_buffer; + v_msg = (struct i40e_virtchnl_msg *)&event.desc; + + do { + ret = i40e_clean_arq_element(hw, &event, &result); + if (ret) + break; + ixlv_vc_completion(sc, v_msg->v_opcode, + v_msg->v_retval, event.msg_buf, event.msg_len); + if (result != 0) + bzero(event.msg_buf, IXL_AQ_BUF_SZ); + } while (result); + + /* check for Admin queue errors */ + oldreg = reg = rd32(hw, hw->aq.arq.len); + if (reg & I40E_VF_ARQLEN1_ARQVFE_MASK) { + device_printf(dev, "ARQ VF Error detected\n"); + reg &= ~I40E_VF_ARQLEN1_ARQVFE_MASK; + } + if (reg & I40E_VF_ARQLEN1_ARQOVFL_MASK) { + device_printf(dev, "ARQ Overflow Error detected\n"); + reg &= ~I40E_VF_ARQLEN1_ARQOVFL_MASK; + } + if (reg & I40E_VF_ARQLEN1_ARQCRIT_MASK) { + device_printf(dev, "ARQ Critical Error detected\n"); + reg &= ~I40E_VF_ARQLEN1_ARQCRIT_MASK; + } + if (oldreg != reg) + wr32(hw, hw->aq.arq.len, reg); + + oldreg = reg = rd32(hw, hw->aq.asq.len); + if (reg & I40E_VF_ATQLEN1_ATQVFE_MASK) { + device_printf(dev, "ASQ VF Error detected\n"); + reg &= ~I40E_VF_ATQLEN1_ATQVFE_MASK; + } + if (reg & I40E_VF_ATQLEN1_ATQOVFL_MASK) { + device_printf(dev, "ASQ Overflow Error detected\n"); + reg &= ~I40E_VF_ATQLEN1_ATQOVFL_MASK; + } + if (reg & I40E_VF_ATQLEN1_ATQCRIT_MASK) { + device_printf(dev, "ASQ Critical Error detected\n"); + reg &= ~I40E_VF_ATQLEN1_ATQCRIT_MASK; + } + if (oldreg != reg) + wr32(hw, hw->aq.asq.len, reg); + + ixlv_enable_adminq_irq(hw); +} + +static void +ixlv_add_sysctls(struct ixlv_sc *sc) +{ + device_t dev = sc->dev; + struct ixl_vsi *vsi = &sc->vsi; + struct i40e_eth_stats *es = &vsi->eth_stats; + + struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); + struct sysctl_oid *tree = device_get_sysctl_tree(dev); + struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree); + + struct sysctl_oid *vsi_node, *queue_node; + struct sysctl_oid_list *vsi_list, *queue_list; + +#define QUEUE_NAME_LEN 32 + char queue_namebuf[QUEUE_NAME_LEN]; + + struct ixl_rx_queue *rx_queues = vsi->rx_queues; + struct ixl_tx_queue *tx_queues = vsi->tx_queues; + struct tx_ring *txr; + struct rx_ring *rxr; + + /* Driver statistics sysctls */ + SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events", + CTLFLAG_RD, &sc->watchdog_events, + "Watchdog timeouts"); + SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "admin_irq", + CTLFLAG_RD, &sc->admin_irq, + "Admin Queue IRQ Handled"); + + /* VSI statistics sysctls */ + vsi_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "vsi", + CTLFLAG_RD, NULL, "VSI-specific statistics"); + vsi_list = SYSCTL_CHILDREN(vsi_node); + + struct ixl_sysctl_info ctls[] = + { + {&es->rx_bytes, "good_octets_rcvd", "Good Octets Received"}, + {&es->rx_unicast, "ucast_pkts_rcvd", + "Unicast Packets Received"}, + {&es->rx_multicast, "mcast_pkts_rcvd", + "Multicast Packets Received"}, + {&es->rx_broadcast, "bcast_pkts_rcvd", + "Broadcast Packets Received"}, + {&es->rx_discards, "rx_discards", "Discarded RX packets"}, + {&es->rx_unknown_protocol, "rx_unknown_proto", "RX unknown protocol packets"}, + {&es->tx_bytes, "good_octets_txd", "Good Octets Transmitted"}, + {&es->tx_unicast, "ucast_pkts_txd", "Unicast Packets Transmitted"}, + {&es->tx_multicast, "mcast_pkts_txd", + "Multicast Packets Transmitted"}, + {&es->tx_broadcast, "bcast_pkts_txd", + "Broadcast Packets Transmitted"}, + {&es->tx_errors, "tx_errors", "TX packet errors"}, + // end + {0,0,0} + }; + struct ixl_sysctl_info *entry = ctls; + while (entry->stat != NULL) + { + SYSCTL_ADD_QUAD(ctx, child, OID_AUTO, entry->name, + CTLFLAG_RD, entry->stat, + entry->description); + entry++; + } + + /* Queue sysctls */ + for (int q = 0; q < vsi->num_tx_queues; q++) { + snprintf(queue_namebuf, QUEUE_NAME_LEN, "que%d", q); + queue_node = SYSCTL_ADD_NODE(ctx, vsi_list, OID_AUTO, queue_namebuf, + CTLFLAG_RD, NULL, "Queue Name"); + queue_list = SYSCTL_CHILDREN(queue_node); + + txr = &(tx_queues[q].txr); + /* Examine queue state */ + SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "qtx_head", + CTLTYPE_UINT | CTLFLAG_RD, &tx_queues[q], + sizeof(struct ixl_tx_queue), + ixlv_sysctl_qtx_tail_handler, "IU", + "Queue Transmit Descriptor Tail"); + } + for (int q = 0; q < vsi->num_rx_queues; q++) { + snprintf(queue_namebuf, QUEUE_NAME_LEN, "que%d", q); + queue_node = SYSCTL_ADD_NODE(ctx, vsi_list, OID_AUTO, queue_namebuf, + CTLFLAG_RD, NULL, "Queue Name"); + queue_list = SYSCTL_CHILDREN(queue_node); + + rxr = &(rx_queues[q].rxr); + SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "qrx_head", + CTLTYPE_UINT | CTLFLAG_RD, &rx_queues[q], + sizeof(struct ixl_rx_queue), + ixlv_sysctl_qrx_tail_handler, "IU", + "Queue Receive Descriptor Tail"); + } +} + +static void +ixlv_init_filters(struct ixlv_sc *sc) +{ + sc->mac_filters = malloc(sizeof(struct ixlv_mac_filter), + M_IXL, M_NOWAIT | M_ZERO); + SLIST_INIT(sc->mac_filters); + sc->vlan_filters = malloc(sizeof(struct ixlv_vlan_filter), + M_IXL, M_NOWAIT | M_ZERO); + SLIST_INIT(sc->vlan_filters); + return; +} + +static void +ixlv_free_filters(struct ixlv_sc *sc) +{ + struct ixlv_mac_filter *f; + struct ixlv_vlan_filter *v; + + while (!SLIST_EMPTY(sc->mac_filters)) { + f = SLIST_FIRST(sc->mac_filters); + SLIST_REMOVE_HEAD(sc->mac_filters, next); + free(f, M_IXL); + } + while (!SLIST_EMPTY(sc->vlan_filters)) { + v = SLIST_FIRST(sc->vlan_filters); + SLIST_REMOVE_HEAD(sc->vlan_filters, next); + free(v, M_IXL); + } + return; +} + +/** + * ixlv_sysctl_qtx_tail_handler + * Retrieves I40E_QTX_TAIL1 value from hardware + * for a sysctl. + */ +static int +ixlv_sysctl_qtx_tail_handler(SYSCTL_HANDLER_ARGS) +{ + struct ixl_tx_queue *que; + int error; + u32 val; + + que = ((struct ixl_tx_queue *)oidp->oid_arg1); + if (!que) return 0; + + val = rd32(que->vsi->hw, que->txr.tail); + error = sysctl_handle_int(oidp, &val, 0, req); + if (error || !req->newptr) + return error; + return (0); +} + +/** + * ixlv_sysctl_qrx_tail_handler + * Retrieves I40E_QRX_TAIL1 value from hardware + * for a sysctl. + */ +static int +ixlv_sysctl_qrx_tail_handler(SYSCTL_HANDLER_ARGS) +{ + struct ixl_rx_queue *que; + int error; + u32 val; + + que = ((struct ixl_rx_queue *)oidp->oid_arg1); + if (!que) return 0; + + val = rd32(que->vsi->hw, que->rxr.tail); + error = sysctl_handle_int(oidp, &val, 0, req); + if (error || !req->newptr) + return error; + return (0); +} Index: sys/dev/ixl/iflib_ixl.h =================================================================== --- /dev/null +++ sys/dev/ixl/iflib_ixl.h @@ -0,0 +1,589 @@ +/****************************************************************************** + + Copyright (c) 2013-2015, Intel Corporation + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of the Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +******************************************************************************/ +/*$FreeBSD$*/ + + +#ifndef _IFLIB_IXL_H_ +#define _IFLIB_IXL_H_ + + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef PCI_IOV +#include +#include +#include +#endif + +#include "ifdi_if.h" +#include "i40e_type.h" +#include "i40e_prototype.h" + + +#if defined(IXL_DEBUG) || defined(IXL_DEBUG_SYSCTL) +#include + + +#define MAC_FORMAT "%02x:%02x:%02x:%02x:%02x:%02x" +#define MAC_FORMAT_ARGS(mac_addr) \ + (mac_addr)[0], (mac_addr)[1], (mac_addr)[2], (mac_addr)[3], \ + (mac_addr)[4], (mac_addr)[5] +#define ON_OFF_STR(is_set) ((is_set) ? "On" : "Off") +#endif /* IXL_DEBUG || IXL_DEBUG_SYSCTL */ + +#ifdef IXL_DEBUG +/* Enable debug sysctls */ +#ifndef IXL_DEBUG_SYSCTL +#define IXL_DEBUG_SYSCTL 1 +#endif + +#define _DBG_PRINTF(S, ...) printf("%s: " S "\n", __func__, ##__VA_ARGS__) +#define _DEV_DBG_PRINTF(dev, S, ...) device_printf(dev, "%s: " S "\n", __func__, ##__VA_ARGS__) +#define _IF_DBG_PRINTF(ifp, S, ...) if_printf(ifp, "%s: " S "\n", __func__, ##__VA_ARGS__) + +/* Defines for printing generic debug information */ +#define DPRINTF(...) _DBG_PRINTF(__VA_ARGS__) +#define DDPRINTF(...) _DEV_DBG_PRINTF(__VA_ARGS__) +#define IDPRINTF(...) _IF_DBG_PRINTF(__VA_ARGS__) + +/* Defines for printing specific debug information */ +#define DEBUG_INIT 1 +#define DEBUG_IOCTL 1 +#define DEBUG_HW 1 + +#define INIT_DEBUGOUT(...) if (DEBUG_INIT) _DBG_PRINTF(__VA_ARGS__) +#define INIT_DBG_DEV(...) if (DEBUG_INIT) _DEV_DBG_PRINTF(__VA_ARGS__) +#define INIT_DBG_IF(...) if (DEBUG_INIT) _IF_DBG_PRINTF(__VA_ARGS__) + +#define IOCTL_DEBUGOUT(...) if (DEBUG_IOCTL) _DBG_PRINTF(__VA_ARGS__) +#define IOCTL_DBG_IF2(ifp, S, ...) if (DEBUG_IOCTL) \ + if_printf(ifp, S "\n", ##__VA_ARGS__) +#define IOCTL_DBG_IF(...) if (DEBUG_IOCTL) _IF_DBG_PRINTF(__VA_ARGS__) + +#define HW_DEBUGOUT(...) if (DEBUG_HW) _DBG_PRINTF(__VA_ARGS__) + +#else /* no IXL_DEBUG */ +#define DEBUG_INIT 0 +#define DEBUG_IOCTL 0 +#define DEBUG_HW 0 + +#define DPRINTF(...) +#define DDPRINTF(...) +#define IDPRINTF(...) + +#define INIT_DEBUGOUT(...) +#define INIT_DBG_DEV(...) +#define INIT_DBG_IF(...) +#define IOCTL_DEBUGOUT(...) +#define IOCTL_DBG_IF2(...) +#define IOCTL_DBG_IF(...) +#define HW_DEBUGOUT(...) +#endif /* IXL_DEBUG */ + +/* Tunables */ + +/* + * Ring Descriptors Valid Range: 32-4096 Default Value: 1024 This value is the + * number of tx/rx descriptors allocated by the driver. Increasing this + * value allows the driver to queue more operations. Each descriptor is 16 + * or 32 bytes (configurable in FVL) + */ +#define DEFAULT_RING 1024 +#define PERFORM_RING 2048 +#define MAX_RING 4096 +#define MIN_RING 32 + +/* +** Default number of entries in Tx queue buf_ring. +*/ +#define SMALL_TXBRSZ 4096 +/* This may require mbuf cluster tuning */ +#define DEFAULT_TXBRSZ (SMALL_TXBRSZ * SMALL_TXBRSZ) + +/* Alignment for rings */ +#define DBA_ALIGN 128 + +/* + * This parameter controls the maximum no of times the driver will loop in + * the isr. Minimum Value = 1 + */ +#define MAX_LOOP 10 + +/* + * This is the max watchdog interval, ie. the time that can + * pass between any two TX clean operations, such only happening + * when the TX hardware is functioning. + */ +#define IXL_WATCHDOG (10 * hz) + +/* + * This parameters control when the driver calls the routine to reclaim + * transmit descriptors. + */ +#define IXL_TX_CLEANUP_THRESHOLD (que->num_desc / 8) +#define IXL_TX_OP_THRESHOLD (que->num_desc / 32) + +/* Flow control constants */ +#define IXL_FC_PAUSE 0xFFFF +#define IXL_FC_HI 0x20000 +#define IXL_FC_LO 0x10000 + +#define MAX_MULTICAST_ADDR 128 + +#define IXL_BAR 3 +#define IXL_ADM_LIMIT 2 +#define IXL_TSO_SIZE ((255*1024)-1) +#define IXL_TX_BUF_SZ ((u32) 1514) +#define IXL_AQ_BUF_SZ ((u32) 4096) +#define IXL_RX_HDR 128 +/* Controls the length of the Admin Queue */ +#define IXL_AQ_LEN 256 +#define IXL_AQ_LEN_MAX 1024 +#define IXL_AQ_BUFSZ 4096 +#define IXL_RX_LIMIT 512 +#define IXL_RX_ITR 0 +#define IXL_TX_ITR 1 +#define IXL_ITR_NONE 3 +#define IXL_QUEUE_EOL 0x7FF +#define IXL_MAX_FRAME 0x2600 +#define IXL_MAX_TX_SEGS 8 +#define IXL_MAX_TSO_SEGS 66 +#define IXL_QUEUE_HUNG 0x80000000 +#define IXL_KEYSZ 10 + +#define IXL_VF_MAX_BUFFER 0x3F80 +#define IXL_VF_MAX_HDR_BUFFER 0x840 +#define IXL_VF_MAX_FRAME 0x3FFF + +/* ERJ: hardware can support ~1.5k filters between all functions */ +#define IXL_MAX_FILTERS 256 +#define IXL_MAX_TX_BUSY 10 + +#define IXL_NVM_VERSION_LO_SHIFT 0 +#define IXL_NVM_VERSION_LO_MASK (0xff << IXL_NVM_VERSION_LO_SHIFT) +#define IXL_NVM_VERSION_HI_SHIFT 12 +#define IXL_NVM_VERSION_HI_MASK (0xf << IXL_NVM_VERSION_HI_SHIFT) + + +/* + * Interrupt Moderation parameters + */ +#define IXL_MAX_ITR 0x07FF +#define IXL_ITR_100K 0x0005 +#define IXL_ITR_20K 0x0019 +#define IXL_ITR_8K 0x003E +#define IXL_ITR_4K 0x007A +#define IXL_ITR_DYNAMIC 0x8000 +#define IXL_LOW_LATENCY 0 +#define IXL_AVE_LATENCY 1 +#define IXL_BULK_LATENCY 2 + +/* MacVlan Flags */ +#define IXL_FILTER_USED (u16)(1 << 0) +#define IXL_FILTER_VLAN (u16)(1 << 1) +#define IXL_FILTER_ADD (u16)(1 << 2) +#define IXL_FILTER_DEL (u16)(1 << 3) +#define IXL_FILTER_MC (u16)(1 << 4) + +/* used in the vlan field of the filter when not a vlan */ +#define IXL_VLAN_ANY -1 + +#define CSUM_OFFLOAD_IPV4 (CSUM_IP|CSUM_TCP|CSUM_UDP|CSUM_SCTP) +#define CSUM_OFFLOAD_IPV6 (CSUM_TCP_IPV6|CSUM_UDP_IPV6|CSUM_SCTP_IPV6) +#define CSUM_OFFLOAD (CSUM_OFFLOAD_IPV4|CSUM_OFFLOAD_IPV6|CSUM_TSO) + +/* Misc flags for ixl_vsi.flags */ +#define IXL_FLAGS_KEEP_TSO4 (1 << 0) +#define IXL_FLAGS_KEEP_TSO6 (1 << 1) + +#define IXL_VF_RESET_TIMEOUT 100 + +#define IXL_VSI_DATA_PORT 0x01 + +#define IXLV_MAX_QUEUES 16 +#define IXL_MAX_VSI_QUEUES (2 * (I40E_VSILAN_QTABLE_MAX_INDEX + 1)) + +#define IXL_RX_CTX_BASE_UNITS 128 +#define IXL_TX_CTX_BASE_UNITS 128 + +#define IXL_VPINT_LNKLSTN_REG(hw, vector, vf_num) \ + I40E_VPINT_LNKLSTN(((vector) - 1) + \ + (((hw)->func_caps.num_msix_vectors_vf - 1) * (vf_num))) + +#define IXL_VFINT_DYN_CTLN_REG(hw, vector, vf_num) \ + I40E_VFINT_DYN_CTLN(((vector) - 1) + \ + (((hw)->func_caps.num_msix_vectors_vf - 1) * (vf_num))) + +#define IXL_PF_PCI_CIAA_VF_DEVICE_STATUS 0xAA + +#define IXL_PF_PCI_CIAD_VF_TRANS_PENDING_MASK 0x20 + +#define IXL_GLGEN_VFLRSTAT_INDEX(glb_vf) ((glb_vf) / 32) +#define IXL_GLGEN_VFLRSTAT_MASK(glb_vf) (1 << ((glb_vf) % 32)) + +#define IXL_MAX_ITR_IDX 3 + +#define IXL_END_OF_INTR_LNKLST 0x7FF + +#if __FreeBSD_version >= 1100036 +#define IXL_SET_IPACKETS(vsi, count) (vsi)->ipackets = (count) +#define IXL_SET_IERRORS(vsi, count) (vsi)->ierrors = (count) +#define IXL_SET_OPACKETS(vsi, count) (vsi)->opackets = (count) +#define IXL_SET_OERRORS(vsi, count) (vsi)->oerrors = (count) +#define IXL_SET_COLLISIONS(vsi, count) /* Do nothing; collisions is always 0. */ +#define IXL_SET_IBYTES(vsi, count) (vsi)->ibytes = (count) +#define IXL_SET_OBYTES(vsi, count) (vsi)->obytes = (count) +#define IXL_SET_IMCASTS(vsi, count) (vsi)->imcasts = (count) +#define IXL_SET_OMCASTS(vsi, count) (vsi)->omcasts = (count) +#define IXL_SET_IQDROPS(vsi, count) (vsi)->iqdrops = (count) +#define IXL_SET_OQDROPS(vsi, count) (vsi)->oqdrops = (count) +#define IXL_SET_NOPROTO(vsi, count) (vsi)->noproto = (count) +#else +#define IXL_SET_IPACKETS(vsi, count) (vsi)->ifp->if_ipackets = (count) +#define IXL_SET_IERRORS(vsi, count) (vsi)->ifp->if_ierrors = (count) +#define IXL_SET_OPACKETS(vsi, count) (vsi)->ifp->if_opackets = (count) +#define IXL_SET_OERRORS(vsi, count) (vsi)->ifp->if_oerrors = (count) +#define IXL_SET_COLLISIONS(vsi, count) (vsi)->ifp->if_collisions = (count) +#define IXL_SET_IBYTES(vsi, count) (vsi)->ifp->if_ibytes = (count) +#define IXL_SET_OBYTES(vsi, count) (vsi)->ifp->if_obytes = (count) +#define IXL_SET_IMCASTS(vsi, count) (vsi)->ifp->if_imcasts = (count) +#define IXL_SET_OMCASTS(vsi, count) (vsi)->ifp->if_omcasts = (count) +#define IXL_SET_IQDROPS(vsi, count) (vsi)->ifp->if_iqdrops = (count) +#define IXL_SET_OQDROPS(vsi, odrops) (vsi)->ifp->if_snd.ifq_drops = (odrops) +#define IXL_SET_NOPROTO(vsi, count) (vsi)->noproto = (count) +#endif + +/* + ***************************************************************************** + * vendor_info_array + * + * This array contains the list of Subvendor/Subdevice IDs on which the driver + * should load. + * + ***************************************************************************** + */ +typedef struct _ixl_vendor_info_t { + unsigned int vendor_id; + unsigned int device_id; + unsigned int subvendor_id; + unsigned int subdevice_id; + unsigned int index; +} ixl_vendor_info_t; + + +/* +** This struct has multiple uses, multicast +** addresses, vlans, and mac filters all use it. +*/ +struct ixl_mac_filter { + SLIST_ENTRY(ixl_mac_filter) next; + u8 macaddr[ETHER_ADDR_LEN]; + s16 vlan; + u16 flags; +}; + + +/* + * The Transmit ring control struct + */ +struct tx_ring { + struct ixl_tx_queue *que; + u32 tail; + struct i40e_tx_desc *tx_base; + uint64_t tx_paddr; + u16 atr_rate; + u16 atr_count; + u16 itr; + u16 latency; + u32 me; + + /* Used for Dynamic ITR calculation */ + u32 packets; + u32 bytes; + + /* Soft Stats */ + u64 tx_bytes; + u64 total_packets; +}; + + +/* + * The Receive ring control struct + */ +struct rx_ring { + struct ixl_rx_queue *que; + union i40e_rx_desc *rx_base; + uint64_t rx_paddr; + bool discard; + u16 itr; + u16 latency; + + u32 mbuf_sz; + u32 tail; + u32 me; + + /* Used for Dynamic ITR calculation */ + u32 packets; + u32 bytes; + + /* Soft stats */ + u64 split; + u64 rx_packets; + u64 rx_bytes; + u64 discarded; +}; + +/* +** Driver queue struct: this is the interrupt container +** for the associated tx and rx ring pair. +*/ +struct ixl_tx_queue { + struct ixl_vsi *vsi; + int busy; + struct tx_ring txr; + /* Queue stats */ + u64 irqs; + u64 tso; +}; + + +struct ixl_rx_queue { + struct ixl_vsi *vsi; + u32 msix; /* This queue's MSIX vector */ + u32 eims; /* This queue's EIMS bit */ + struct rx_ring rxr; + + struct if_irq que_irq; + + /* Queue stats */ + u64 irqs; +}; + +#define DOWNCAST(sctx) ((struct ixl_vsi *)(sctx)) +/* +** Virtual Station interface: +** there would be one of these per traffic class/type +** for now just one, and its embedded in the pf +*/ +SLIST_HEAD(ixl_ftl_head, ixl_mac_filter); + +struct ixl_vsi { + if_ctx_t ctx; + if_softc_ctx_t shared; + + struct ifnet *ifp; + struct ifmedia *media; + +#define num_rx_queues shared->isc_nrxqsets +#define num_tx_queues shared->isc_ntxqsets +#define max_frame_size shared->isc_max_frame_size + + void *back; + struct i40e_hw *hw; + u64 que_mask; + int id; + u16 vsi_num; + u16 msix_base; /* station base MSIX vector */ + u16 first_queue; + u16 rx_itr_setting; + u16 tx_itr_setting; + struct ixl_tx_queue *tx_queues; /* head of queues */ + struct ixl_rx_queue *rx_queues; /* head of queues */ + bool link_active; + u16 seid; + u32 link_speed; + struct if_irq irq; + u16 uplink_seid; + u16 downlink_seid; + + /* MAC/VLAN Filter list */ + struct ixl_ftl_head ftl; + u16 num_macs; + + struct i40e_aqc_vsi_properties_data info; + + u16 num_vlans; + + /* Per-VSI stats from hardware */ + struct i40e_eth_stats eth_stats; + struct i40e_eth_stats eth_stats_offsets; + bool stat_offsets_loaded; + /* VSI stat counters */ + u64 ipackets; + u64 ierrors; + u64 opackets; + u64 oerrors; + u64 ibytes; + u64 obytes; + u64 imcasts; + u64 omcasts; + u64 iqdrops; + u64 oqdrops; + u64 noproto; + + /* Driver statistics */ + u64 hw_filters_del; + u64 hw_filters_add; + + /* Misc. */ + u64 active_queues; + u64 flags; + struct sysctl_oid *vsi_node; +}; + +/* +** Find the next available unused filter +*/ +static inline struct ixl_mac_filter * +ixl_get_filter(struct ixl_vsi *vsi) +{ + struct ixl_mac_filter *f; + + /* create a new empty filter */ + f = malloc(sizeof(struct ixl_mac_filter), + M_IXL, M_NOWAIT | M_ZERO); + if (f) + SLIST_INSERT_HEAD(&vsi->ftl, f, next); + + return (f); +} + +/* +** Compare two ethernet addresses +*/ +static inline bool +cmp_etheraddr(const u8 *ea1, const u8 *ea2) +{ + + return (bcmp(ea1, ea2, 6) == 0); +} + +/* + * Info for stats sysctls + */ +struct ixl_sysctl_info { + u64 *stat; + char *name; + char *description; +}; + +extern int ixl_atr_rate; + +/* +** ixl_fw_version_str - format the FW and NVM version strings +*/ +static inline char * +ixl_fw_version_str(struct i40e_hw *hw) +{ + static char buf[32]; + + snprintf(buf, sizeof(buf), + "f%d.%d a%d.%d n%02x.%02x e%08x", + hw->aq.fw_maj_ver, hw->aq.fw_min_ver, + hw->aq.api_maj_ver, hw->aq.api_min_ver, + (hw->nvm.version & IXL_NVM_VERSION_HI_MASK) >> + IXL_NVM_VERSION_HI_SHIFT, + (hw->nvm.version & IXL_NVM_VERSION_LO_MASK) >> + IXL_NVM_VERSION_LO_SHIFT, + hw->nvm.eetrack); + return buf; +} + +/********************************************************************* + * TXRX Function prototypes + *********************************************************************/ +void ixl_init_tx_ring(struct ixl_vsi *, struct ixl_tx_queue *); + +#ifdef IXL_FDIR +void ixl_atr(struct ixl_rx_queue *, struct tcphdr *, int); +#endif + +/********************************************************************* + * common Function prototypes + *********************************************************************/ + +int ixl_if_media_change(if_ctx_t); +int ixl_if_tx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, int); +int ixl_if_rx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, int); +void ixl_if_queues_free(if_ctx_t ctx); + +#endif /* _IXL_H_ */ Index: sys/dev/ixl/iflib_ixl_pf.h =================================================================== --- /dev/null +++ sys/dev/ixl/iflib_ixl_pf.h @@ -0,0 +1,125 @@ +/****************************************************************************** + + Copyright (c) 2013-2015, Intel Corporation + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of the Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +******************************************************************************/ +/*$FreeBSD$*/ + + +#ifndef _IFLIB_IXL_PF_H_ +#define _IFLIB_IXL_PF_H_ + +#define VF_FLAG_ENABLED 0x01 +#define VF_FLAG_SET_MAC_CAP 0x02 +#define VF_FLAG_VLAN_CAP 0x04 +#define VF_FLAG_PROMISC_CAP 0x08 +#define VF_FLAG_MAC_ANTI_SPOOF 0x10 + +struct ixl_vf { + struct ixl_vsi vsi; + uint32_t vf_flags; + + uint8_t mac[ETHER_ADDR_LEN]; + uint16_t vf_num; + + struct sysctl_ctx_list ctx; +}; + +/* Physical controller structure */ +struct ixl_pf { + /* + ** VSI - Stations: + ** These are the traffic class holders, and + ** will have a stack interface and queues + ** associated with them. + ** NOTE: for now using just one, so embed it. + ** also, to make it interchangeable place it _first_ + */ + struct ixl_vsi vsi; + + + struct i40e_hw hw; + struct i40e_osdep osdep; + device_t dev; + + struct resource *pci_mem; + + /* + * Interrupt resources: this set is + * either used for legacy, or for Link + * when doing MSIX + */ + + int msix; + int if_flags; + + u32 qbase; + u32 admvec; + + bool link_up; + u32 link_speed; + int advertised_speed; + int fc; /* local flow ctrl setting */ + + /* Misc stats maintained by the driver */ + u64 watchdog_events; + u64 admin_irq; + + /* Statistics from hw */ + struct i40e_hw_port_stats stats; + struct i40e_hw_port_stats stats_offsets; + bool stat_offsets_loaded; + + struct ixl_vf *vfs; + int num_vfs; + uint16_t veb_seid; + struct task vflr_task; + int vc_debug_lvl; +}; + +#define IXL_SET_ADVERTISE_HELP \ +"Control link advertise speed:\n" \ +"\tFlags:\n" \ +"\t\t0x1 - advertise 100 Mb\n" \ +"\t\t0x2 - advertise 1G\n" \ +"\t\t0x4 - advertise 10G\n" \ +"\t\t0x8 - advertise 20G\n\n" \ +"\tDoes not work on 40G devices." + +#define I40E_VC_DEBUG(pf, level, ...) \ + do { \ + if ((pf)->vc_debug_lvl >= (level)) \ + device_printf((pf)->dev, __VA_ARGS__); \ + } while (0) + +#define i40e_send_vf_nack(pf, vf, op, st) \ + ixl_send_vf_nack_msg((pf), (vf), (op), (st), __FILE__, __LINE__) + +#endif /* _IXL_PF_H_ */ Index: sys/dev/ixl/iflib_ixl_txrx.c =================================================================== --- /dev/null +++ sys/dev/ixl/iflib_ixl_txrx.c @@ -0,0 +1,582 @@ +/****************************************************************************** + + Copyright (c) 2013-2015, Intel Corporation + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of the Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +******************************************************************************/ +/*$FreeBSD$*/ + +/* +** IXL driver TX/RX Routines: +** This was seperated to allow usage by +** both the BASE and the VF drivers. +*/ + +#ifndef IXL_STANDALONE_BUILD +#include "opt_inet.h" +#include "opt_inet6.h" +#include "opt_rss.h" +#endif + +#include "ixl.h" + +#ifdef RSS +#include +#endif + +/* Local Prototypes */ +static void ixl_rx_checksum(if_rxd_info_t ri, u32 status, u32 error, u8 ptype); + +static int ixl_isc_txd_encap(void *arg, if_pkt_info_t pi); +static void ixl_isc_txd_flush(void *arg, uint16_t txqid, uint32_t pidx); +static int ixl_isc_txd_credits_update(void *arg, uint16_t qid, uint32_t cidx, bool clear); + +static void ixl_isc_rxd_refill(void *arg, uint16_t rxqid, uint8_t flid __unused, + uint32_t pidx, uint64_t *paddrs, caddr_t *vaddrs __unused, uint16_t count); +static void ixl_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, uint32_t pidx); +static int ixl_isc_rxd_available(void *arg, uint16_t rxqid, uint32_t idx); +static int ixl_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri); + +extern int ixl_intr(void *arg); + +struct if_txrx ixl_txrx = { + ixl_isc_txd_encap, + ixl_isc_txd_flush, + ixl_isc_txd_credits_update, + ixl_isc_rxd_available, + ixl_isc_rxd_pkt_get, + ixl_isc_rxd_refill, + ixl_isc_rxd_flush, + ixl_intr +}; + +extern if_shared_ctx_t ixl_sctx; + +/* +** Find mbuf chains passed to the driver +** that are 'sparse', using more than 8 +** segments to deliver an mss-size chunk of data +*/ +static int +ixl_tso_detect_sparse(bus_dma_segment_t *segs, int nsegs, int segsz) +{ + int i, count, curseg; + + if (nsegs <= IXL_MAX_TX_SEGS-2) + return (0); + for (curseg = count = i = 0; i < nsegs; i++) { + curseg += segs[i].ds_len; + count++; + if (__predict_false(count == IXL_MAX_TX_SEGS-2)) + return (1); + if (curseg > segsz) { + curseg -= segsz; + count = 1; + } + if (curseg == segsz) + curseg = count = 0; + } + return (0); +} + +/********************************************************************* + * + * Setup descriptor for hw offloads + * + **********************************************************************/ + +static void +ixl_tx_setup_offload(struct ixl_tx_queue *que, + if_pkt_info_t pi, u32 *cmd, u32 *off) +{ + + + switch (pi->ipi_etype) { +#ifdef INET + case ETHERTYPE_IP: + *cmd |= I40E_TX_DESC_CMD_IIPT_IPV4_CSUM; + break; +#endif +#ifdef INET6 + case ETHERTYPE_IPV6: + *cmd |= I40E_TX_DESC_CMD_IIPT_IPV6; + break; +#endif + default: + break; + } + + *off |= (pi->ipi_ehdrlen >> 1) << I40E_TX_DESC_LENGTH_MACLEN_SHIFT; + *off |= (pi->ipi_ip_hlen >> 2) << I40E_TX_DESC_LENGTH_IPLEN_SHIFT; + + switch (pi->ipi_ipproto) { + case IPPROTO_TCP: + if (pi->ipi_csum_flags & (CSUM_TCP|CSUM_TCP_IPV6)) { + *cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP; + *off |= (pi->ipi_tcp_hlen >> 2) << + I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; + } +#ifdef IXL_FDIR + ixl_atr(que, pi->ipi_tcp_hflags, pi->ipi_etype); +#endif + break; + case IPPROTO_UDP: + if (pi->ipi_csum_flags & (CSUM_UDP|CSUM_UDP_IPV6)) { + *cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP; + *off |= (sizeof(struct udphdr) >> 2) << + I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; + } + break; + + case IPPROTO_SCTP: + if (pi->ipi_csum_flags & (CSUM_SCTP|CSUM_SCTP_IPV6)) { + *cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP; + *off |= (sizeof(struct sctphdr) >> 2) << + I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; + } + /* Fall Thru */ + default: + break; + } + +} + +/********************************************************************** + * + * Setup context for hardware segmentation offload (TSO) + * + **********************************************************************/ +static int +ixl_tso_setup(struct tx_ring *txr, if_pkt_info_t pi) +{ + struct i40e_tx_context_desc *TXD; + u32 cmd, mss, type, tsolen; + int idx; + u64 type_cmd_tso_mss; + + idx = pi->ipi_pidx; + TXD = (struct i40e_tx_context_desc *) &txr->tx_base[idx]; + tsolen = pi->ipi_len - (pi->ipi_ehdrlen + pi->ipi_ip_hlen + pi->ipi_tcp_hlen); + + type = I40E_TX_DESC_DTYPE_CONTEXT; + cmd = I40E_TX_CTX_DESC_TSO; + mss = pi->ipi_tso_segsz; + + type_cmd_tso_mss = ((u64)type << I40E_TXD_CTX_QW1_DTYPE_SHIFT) | + ((u64)cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) | + ((u64)tsolen << I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) | + ((u64)mss << I40E_TXD_CTX_QW1_MSS_SHIFT); + TXD->type_cmd_tso_mss = htole64(type_cmd_tso_mss); + + TXD->tunneling_params = htole32(0); + + return ((idx + 1) & (ixl_sctx->isc_ntxd-1)); +} + +/********************************************************************* + * + * This routine maps the mbufs to tx descriptors, allowing the + * TX engine to transmit the packets. + * - return 0 on success, positive on failure + * + **********************************************************************/ +#define IXL_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS) + +static int +ixl_isc_txd_encap(void *arg, if_pkt_info_t pi) +{ + struct ixl_vsi *vsi = arg; + struct ixl_tx_queue *que = &vsi->tx_queues[pi->ipi_qsidx]; + struct tx_ring *txr = &que->txr; + int nsegs = pi->ipi_nsegs; + bus_dma_segment_t *segs = pi->ipi_segs; + struct i40e_tx_desc *txd = NULL; + int i, j, mask; + u32 cmd, off; + + cmd = off = 0; + i = pi->ipi_pidx; + + if (pi->ipi_flags & IPI_TX_INTR) + cmd |= (I40E_TX_DESC_CMD_RS << I40E_TXD_QW1_CMD_SHIFT); + + /* Set up the TSO/CSUM offload */ + if (pi->ipi_csum_flags & CSUM_OFFLOAD) { + /* Set up the TSO context descriptor if required */ + if (pi->ipi_csum_flags & CSUM_TSO) { + if (ixl_tso_detect_sparse(segs, nsegs, pi->ipi_tso_segsz)) + return (EFBIG); + + i = ixl_tso_setup(txr, pi); + } + ixl_tx_setup_offload(que, pi, &cmd, &off); + } + + if (pi->ipi_mflags & M_VLANTAG) + cmd |= I40E_TX_DESC_CMD_IL2TAG1; + + cmd |= I40E_TX_DESC_CMD_ICRC; + mask = ixl_sctx->isc_ntxd-1; + for (j = 0; j < nsegs; j++) { + bus_size_t seglen; + + txd = &txr->tx_base[i]; + seglen = segs[j].ds_len; + + txd->buffer_addr = htole64(segs[j].ds_addr); + txd->cmd_type_offset_bsz = + htole64(I40E_TX_DESC_DTYPE_DATA + | ((u64)cmd << I40E_TXD_QW1_CMD_SHIFT) + | ((u64)off << I40E_TXD_QW1_OFFSET_SHIFT) + | ((u64)seglen << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) + | ((u64)htole16(pi->ipi_vtag) << I40E_TXD_QW1_L2TAG1_SHIFT)); + + i = (i+1) & mask; + } + /* Set the last descriptor for report */ + txd->cmd_type_offset_bsz |= + htole64(((u64)IXL_TXD_CMD << I40E_TXD_QW1_CMD_SHIFT)); + pi->ipi_new_pidx = i; + + ++txr->total_packets; + return (0); +} + +static void +ixl_isc_txd_flush(void *arg, uint16_t txqid, uint32_t pidx) +{ + struct ixl_vsi *vsi = arg; + struct tx_ring *txr = &vsi->tx_queues[txqid].txr; + /* + * Advance the Transmit Descriptor Tail (Tdt), this tells the + * hardware that this frame is available to transmit. + */ + wr32(vsi->hw, txr->tail, pidx); +} + +/********************************************************************* + * + * (Re)Initialize a queue transmit ring. + * - called by init, it clears the descriptor ring, + * and frees any stale mbufs + * + **********************************************************************/ +void +ixl_init_tx_ring(struct ixl_vsi *vsi, struct ixl_tx_queue *que) +{ + struct tx_ring *txr = &que->txr; + + /* Clear the old ring contents */ + bzero((void *)txr->tx_base, + (sizeof(struct i40e_tx_desc)) * ixl_sctx->isc_ntxd); + +#ifdef IXL_FDIR + /* Initialize flow director */ + txr->atr_rate = ixl_atr_rate; + txr->atr_count = 0; +#endif + wr32(vsi->hw, I40E_QTX_TAIL(que->txr.me), 0); + wr32(vsi->hw, I40E_QTX_HEAD(que->txr.me), 0); +} + +/* +** ixl_get_tx_head - Retrieve the value from the +** location the HW records its HEAD index +*/ +static inline u32 +ixl_get_tx_head(struct ixl_tx_queue *que) +{ + struct tx_ring *txr = &que->txr; + void *head = &txr->tx_base[ixl_sctx->isc_ntxd]; + + return LE32_TO_CPU(*(volatile __le32 *)head); +} + +/********************************************************************** + * + * Examine each tx_buffer in the used queue. If the hardware is done + * processing the packet then free associated resources. The + * tx_buffer is put back on the free queue. + * + **********************************************************************/ +static int +ixl_isc_txd_credits_update(void *arg, uint16_t qid, uint32_t cidx, bool clear) +{ + struct ixl_vsi *vsi = arg; + struct ixl_tx_queue *que = &vsi->tx_queues[qid]; + + int head, credits; + + /* Get the Head WB value */ + head = ixl_get_tx_head(que); + + credits = head - cidx; + if (credits < 0) + credits += ixl_sctx->isc_ntxd; + return (credits); +} + +/********************************************************************* + * + * Refresh mbuf buffers for RX descriptor rings + * - now keeps its own state so discards due to resource + * exhaustion are unnecessary, if an mbuf cannot be obtained + * it just returns, keeping its placeholder, thus it can simply + * be recalled to try again. + * + **********************************************************************/ +static void +ixl_isc_rxd_refill(void *arg, uint16_t rxqid, uint8_t flid __unused, + uint32_t pidx, uint64_t *paddrs, caddr_t *vaddrs __unused, uint16_t count) + +{ + struct ixl_vsi *vsi = arg; + struct rx_ring *rxr = &vsi->rx_queues[rxqid].rxr; + int i, mask; + uint32_t next_pidx; + + mask = ixl_sctx->isc_nrxd-1; + for (i = 0, next_pidx = pidx; i < count; i++) { + rxr->rx_base[next_pidx].read.pkt_addr = htole64(paddrs[i]); + next_pidx = (next_pidx + 1) & mask; + } +} + +static void +ixl_isc_rxd_flush(void * arg, uint16_t rxqid, uint8_t flid __unused, uint32_t pidx) +{ + struct ixl_vsi *vsi = arg; + struct rx_ring *rxr = &vsi->rx_queues[rxqid].rxr; + + wr32(vsi->hw, rxr->tail, pidx); +} + +static int +ixl_isc_rxd_available(void *arg, uint16_t rxqid, uint32_t idx) +{ + struct ixl_vsi *vsi = arg; + struct rx_ring *rxr = &vsi->rx_queues[rxqid].rxr; + union i40e_rx_desc *cur; + u64 qword; + uint32_t status; + int cnt, i, mask; + + mask = ixl_sctx->isc_nrxd-1; + for (cnt = 0, i = idx; cnt < ixl_sctx->isc_nrxd;) { + cur = &rxr->rx_base[i]; + qword = le64toh(cur->wb.qword1.status_error_len); + status = (qword & I40E_RXD_QW1_STATUS_MASK) + >> I40E_RXD_QW1_STATUS_SHIFT; + if ((status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) == 0) + break; + if (status & (1 << I40E_RX_DESC_STATUS_EOF_SHIFT)) + cnt++; + i = (i + 1) & mask; + } + return (cnt); +} + +/* +** i40e_ptype_to_hash: parse the packet type +** to determine the appropriate hash. +*/ +static inline int +ixl_ptype_to_hash(u8 ptype) +{ + struct i40e_rx_ptype_decoded decoded; + u8 ex = 0; + + decoded = decode_rx_desc_ptype(ptype); + ex = decoded.outer_frag; + + if (!decoded.known) + return M_HASHTYPE_OPAQUE; + + if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_L2) + return M_HASHTYPE_OPAQUE; + + /* Note: anything that gets to this point is IP */ + if (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6) { + switch (decoded.inner_prot) { + case I40E_RX_PTYPE_INNER_PROT_TCP: + if (ex) + return M_HASHTYPE_RSS_TCP_IPV6_EX; + else + return M_HASHTYPE_RSS_TCP_IPV6; + case I40E_RX_PTYPE_INNER_PROT_UDP: + if (ex) + return M_HASHTYPE_RSS_UDP_IPV6_EX; + else + return M_HASHTYPE_RSS_UDP_IPV6; + default: + if (ex) + return M_HASHTYPE_RSS_IPV6_EX; + else + return M_HASHTYPE_RSS_IPV6; + } + } + if (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV4) { + switch (decoded.inner_prot) { + case I40E_RX_PTYPE_INNER_PROT_TCP: + return M_HASHTYPE_RSS_TCP_IPV4; + case I40E_RX_PTYPE_INNER_PROT_UDP: + if (ex) + return M_HASHTYPE_RSS_UDP_IPV4_EX; + else + return M_HASHTYPE_RSS_UDP_IPV4; + default: + return M_HASHTYPE_RSS_IPV4; + } + } + /* We should never get here!! */ + return M_HASHTYPE_OPAQUE; +} + +/********************************************************************* + * + * This routine executes in ithread context. It sends data which has been + * dma'ed into host memory to upper layer. + * + * Returns 0 upon success, errno on failure + *********************************************************************/ + +static int +ixl_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) +{ + struct ixl_vsi *vsi = arg; + struct ixl_rx_queue *que = &vsi->rx_queues[ri->iri_qsidx]; + struct rx_ring *rxr = &que->rxr; + union i40e_rx_desc *cur; + u32 status, error; + u16 hlen, plen, vtag; + u64 qword; + u8 ptype; + bool eop; + int i, cidx; + + cidx = ri->iri_cidx; + i = 0; + do { + cur = &rxr->rx_base[cidx]; + qword = le64toh(cur->wb.qword1.status_error_len); + status = (qword & I40E_RXD_QW1_STATUS_MASK) + >> I40E_RXD_QW1_STATUS_SHIFT; + error = (qword & I40E_RXD_QW1_ERROR_MASK) + >> I40E_RXD_QW1_ERROR_SHIFT; + plen = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) + >> I40E_RXD_QW1_LENGTH_PBUF_SHIFT; + hlen = (qword & I40E_RXD_QW1_LENGTH_HBUF_MASK) + >> I40E_RXD_QW1_LENGTH_HBUF_SHIFT; + ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) + >> I40E_RXD_QW1_PTYPE_SHIFT; + + /* we should never be called without a valid descriptor */ + MPASS((status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) != 0); + + ri->iri_len += plen; + rxr->rx_bytes += plen; + + cur->wb.qword1.status_error_len = 0; + eop = (status & (1 << I40E_RX_DESC_STATUS_EOF_SHIFT)); + if (status & (1 << I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)) + vtag = le16toh(cur->wb.qword0.lo_dword.l2tag1); + else + vtag = 0; + + /* + ** Make sure bad packets are discarded, + ** note that only EOP descriptor has valid + ** error results. + */ + if (eop && (error & (1 << I40E_RX_DESC_ERROR_RXE_SHIFT))) { + rxr->discarded++; + return (EBADMSG); + } + ri->iri_frags[i].irf_flid = 0; + ri->iri_frags[i].irf_idx = cidx; + if (++cidx == ixl_sctx->isc_ntxd) + cidx = 0; + i++; + /* even a 16K packet shouldn't consume more than 8 clusters */ + MPASS(i < 9); + } while (!eop); + + rxr->rx_packets++; + /* capture data for dynamic ITR adjustment */ + rxr->packets++; + if ((vsi->ifp->if_capenable & IFCAP_RXCSUM) != 0) + ixl_rx_checksum(ri, status, error, ptype); + ri->iri_flowid = le32toh(cur->wb.qword0.hi_dword.rss); + ri->iri_rsstype = ixl_ptype_to_hash(ptype); + ri->iri_vtag = vtag; + ri->iri_nfrags = i; + if (vtag) + ri->iri_flags |= M_VLANTAG; + return (0); +} + +/********************************************************************* + * + * Verify that the hardware indicated that the checksum is valid. + * Inform the stack about the status of checksum so that stack + * doesn't spend time verifying the checksum. + * + *********************************************************************/ +static void +ixl_rx_checksum(if_rxd_info_t ri, u32 status, u32 error, u8 ptype) +{ + struct i40e_rx_ptype_decoded decoded; + + decoded = decode_rx_desc_ptype(ptype); + /* Errors? */ + if (error & ((1 << I40E_RX_DESC_ERROR_IPE_SHIFT) | + (1 << I40E_RX_DESC_ERROR_L4E_SHIFT))) { + ri->iri_csum_flags = 0; + return; + } + + /* IPv6 with extension headers likely have bad csum */ + if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP && + decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6) + if (status & + (1 << I40E_RX_DESC_STATUS_IPV6EXADD_SHIFT)) { + ri->iri_csum_flags = 0; + return; + } + + /* IP Checksum Good */ + ri->iri_csum_flags = CSUM_IP_CHECKED; + ri->iri_csum_flags |= CSUM_IP_VALID; + + if (status & (1 << I40E_RX_DESC_STATUS_L3L4P_SHIFT)) { + ri->iri_csum_flags |= + (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); + ri->iri_csum_data |= htons(0xffff); + } +} Index: sys/dev/ixl/iflib_ixlv.h =================================================================== --- /dev/null +++ sys/dev/ixl/iflib_ixlv.h @@ -0,0 +1,207 @@ +/****************************************************************************** + + Copyright (c) 2013-2015, Intel Corporation + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of the Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +******************************************************************************/ +/*$FreeBSD$*/ + + +#ifndef _IXLV_H_ +#define _IXLV_H_ + +#include "ixlv_vc_mgr.h" + +#define IXLV_AQ_MAX_ERR 1000 +#define IXLV_MAX_FILTERS 128 +#define IXLV_MAX_QUEUES 16 +#define IXLV_AQ_TIMEOUT (1 * hz) +#define IXLV_CALLOUT_TIMO (hz / 50) /* 20 msec */ + +#define IXLV_FLAG_AQ_ENABLE_QUEUES (u32)(1) +#define IXLV_FLAG_AQ_DISABLE_QUEUES (u32)(1 << 1) +#define IXLV_FLAG_AQ_ADD_MAC_FILTER (u32)(1 << 2) +#define IXLV_FLAG_AQ_ADD_VLAN_FILTER (u32)(1 << 3) +#define IXLV_FLAG_AQ_DEL_MAC_FILTER (u32)(1 << 4) +#define IXLV_FLAG_AQ_DEL_VLAN_FILTER (u32)(1 << 5) +#define IXLV_FLAG_AQ_CONFIGURE_QUEUES (u32)(1 << 6) +#define IXLV_FLAG_AQ_MAP_VECTORS (u32)(1 << 7) +#define IXLV_FLAG_AQ_HANDLE_RESET (u32)(1 << 8) +#define IXLV_FLAG_AQ_CONFIGURE_PROMISC (u32)(1 << 9) +#define IXLV_FLAG_AQ_GET_STATS (u32)(1 << 10) + +/* printf %b arg */ +#define IXLV_FLAGS \ + "\20\1ENABLE_QUEUES\2DISABLE_QUEUES\3ADD_MAC_FILTER" \ + "\4ADD_VLAN_FILTER\5DEL_MAC_FILTER\6DEL_VLAN_FILTER" \ + "\7CONFIGURE_QUEUES\10MAP_VECTORS\11HANDLE_RESET" \ + "\12CONFIGURE_PROMISC\13GET_STATS" + +/* Hack for compatibility with 1.0.x linux pf driver */ +#define I40E_VIRTCHNL_OP_EVENT 17 + +/* Driver state */ +enum ixlv_state_t { + IXLV_START, + IXLV_FAILED, + IXLV_RESET_REQUIRED, + IXLV_RESET_PENDING, + IXLV_VERSION_CHECK, + IXLV_GET_RESOURCES, + IXLV_INIT_READY, + IXLV_INIT_START, + IXLV_INIT_CONFIG, + IXLV_INIT_MAPPING, + IXLV_INIT_ENABLE, + IXLV_INIT_COMPLETE, + IXLV_RUNNING, +}; + +struct ixlv_mac_filter { + SLIST_ENTRY(ixlv_mac_filter) next; + u8 macaddr[ETHER_ADDR_LEN]; + u16 flags; +}; +SLIST_HEAD(mac_list, ixlv_mac_filter); + +struct ixlv_vlan_filter { + SLIST_ENTRY(ixlv_vlan_filter) next; + u16 vlan; + u16 flags; +}; +SLIST_HEAD(vlan_list, ixlv_vlan_filter); + +/* Software controller structure */ +struct ixlv_sc { + struct i40e_hw hw; + struct i40e_osdep osdep; + struct device *dev; + + struct resource *pci_mem; + + enum ixlv_state_t init_state; + + /* + * Interrupt resources + */ + void *tag; + struct resource *res; /* For the AQ */ + + struct ifmedia *media; + int msix; + int pf_version; + int if_flags; + + bool link_up; + u32 link_speed; + + u32 qbase; + u32 admvec; +#ifdef notyet + struct task aq_irq; + struct task aq_sched; +#endif + struct ixl_vsi vsi; + + /* Filter lists */ + struct mac_list *mac_filters; + struct vlan_list *vlan_filters; + + /* Promiscuous mode */ + u32 promiscuous_flags; + + /* Admin queue task flags */ + u32 aq_wait_count; + + struct ixl_vc_mgr vc_mgr; + struct ixl_vc_cmd add_mac_cmd; + struct ixl_vc_cmd del_mac_cmd; + struct ixl_vc_cmd config_queues_cmd; + struct ixl_vc_cmd map_vectors_cmd; + struct ixl_vc_cmd enable_queues_cmd; + struct ixl_vc_cmd add_vlan_cmd; + struct ixl_vc_cmd del_vlan_cmd; + struct ixl_vc_cmd add_multi_cmd; + struct ixl_vc_cmd del_multi_cmd; + + /* Virtual comm channel */ + struct i40e_virtchnl_vf_resource *vf_res; + struct i40e_virtchnl_vsi_resource *vsi_res; + + /* Misc stats maintained by the driver */ + u64 watchdog_events; + u64 admin_irq; + + u8 aq_buffer[IXL_AQ_BUF_SZ]; +}; + +/* +** This checks for a zero mac addr, something that will be likely +** unless the Admin on the Host has created one. +*/ +static inline bool +ixlv_check_ether_addr(u8 *addr) +{ + bool status = TRUE; + + if ((addr[0] == 0 && addr[1]== 0 && addr[2] == 0 && + addr[3] == 0 && addr[4]== 0 && addr[5] == 0)) + status = FALSE; + return (status); +} + +/* +** VF Common function prototypes +*/ +int ixlv_send_api_ver(struct ixlv_sc *); +int ixlv_verify_api_ver(struct ixlv_sc *); +int ixlv_send_vf_config_msg(struct ixlv_sc *); +int ixlv_get_vf_config(struct ixlv_sc *); +void ixlv_init(void *); +int ixlv_reinit_locked(struct ixlv_sc *); +void ixlv_configure_queues(struct ixlv_sc *); +void ixlv_enable_queues(struct ixlv_sc *); +void ixlv_disable_queues(struct ixlv_sc *); +void ixlv_map_queues(struct ixlv_sc *); +void ixlv_enable_intr(struct ixl_vsi *); +void ixlv_disable_intr(struct ixl_vsi *); +void ixlv_add_ether_filters(struct ixlv_sc *); +void ixlv_del_ether_filters(struct ixlv_sc *); +void ixlv_request_stats(struct ixlv_sc *); +void ixlv_request_reset(struct ixlv_sc *); +void ixlv_vc_completion(struct ixlv_sc *, + enum i40e_virtchnl_ops, i40e_status, u8 *, u16); +void ixlv_add_ether_filter(struct ixlv_sc *); +void ixlv_add_vlans(struct ixlv_sc *); +void ixlv_del_vlans(struct ixlv_sc *); +void ixlv_update_stats_counters(struct ixlv_sc *, + struct i40e_eth_stats *); +void ixlv_update_link_status(struct ixlv_sc *); + +#endif /* _IXLV_H_ */ Index: sys/dev/ixl/iflib_ixlvc.c =================================================================== --- /dev/null +++ sys/dev/ixl/iflib_ixlvc.c @@ -0,0 +1,1131 @@ +/****************************************************************************** + + Copyright (c) 2013-2015, Intel Corporation + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of the Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +******************************************************************************/ +/*$FreeBSD$*/ + +/* +** Virtual Channel support +** These are support functions to communication +** between the VF and PF drivers. +*/ + +#include "ixl.h" +#include "ixlv.h" +#include "i40e_prototype.h" + + +/* busy wait delay in msec */ +#define IXLV_BUSY_WAIT_DELAY 10 +#define IXLV_BUSY_WAIT_COUNT 50 + +static void ixl_vc_process_resp(struct ixl_vc_mgr *, uint32_t, + enum i40e_status_code); +static void ixl_vc_process_next(struct ixl_vc_mgr *mgr); +static void ixl_vc_schedule_retry(struct ixl_vc_mgr *mgr); +static void ixl_vc_send_current(struct ixl_vc_mgr *mgr); + +#ifdef IXL_DEBUG +/* +** Validate VF messages +*/ +static int ixl_vc_validate_vf_msg(struct ixlv_sc *sc, u32 v_opcode, + u8 *msg, u16 msglen) +{ + bool err_msg_format = false; + int valid_len; + + /* Validate message length. */ + switch (v_opcode) { + case I40E_VIRTCHNL_OP_VERSION: + valid_len = sizeof(struct i40e_virtchnl_version_info); + break; + case I40E_VIRTCHNL_OP_RESET_VF: + case I40E_VIRTCHNL_OP_GET_VF_RESOURCES: + valid_len = 0; + break; + case I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE: + valid_len = sizeof(struct i40e_virtchnl_txq_info); + break; + case I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE: + valid_len = sizeof(struct i40e_virtchnl_rxq_info); + break; + case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES: + valid_len = sizeof(struct i40e_virtchnl_vsi_queue_config_info); + if (msglen >= valid_len) { + struct i40e_virtchnl_vsi_queue_config_info *vqc = + (struct i40e_virtchnl_vsi_queue_config_info *)msg; + valid_len += (vqc->num_queue_pairs * + sizeof(struct + i40e_virtchnl_queue_pair_info)); + if (vqc->num_queue_pairs == 0) + err_msg_format = true; + } + break; + case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP: + valid_len = sizeof(struct i40e_virtchnl_irq_map_info); + if (msglen >= valid_len) { + struct i40e_virtchnl_irq_map_info *vimi = + (struct i40e_virtchnl_irq_map_info *)msg; + valid_len += (vimi->num_vectors * + sizeof(struct i40e_virtchnl_vector_map)); + if (vimi->num_vectors == 0) + err_msg_format = true; + } + break; + case I40E_VIRTCHNL_OP_ENABLE_QUEUES: + case I40E_VIRTCHNL_OP_DISABLE_QUEUES: + valid_len = sizeof(struct i40e_virtchnl_queue_select); + break; + case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS: + case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS: + valid_len = sizeof(struct i40e_virtchnl_ether_addr_list); + if (msglen >= valid_len) { + struct i40e_virtchnl_ether_addr_list *veal = + (struct i40e_virtchnl_ether_addr_list *)msg; + valid_len += veal->num_elements * + sizeof(struct i40e_virtchnl_ether_addr); + if (veal->num_elements == 0) + err_msg_format = true; + } + break; + case I40E_VIRTCHNL_OP_ADD_VLAN: + case I40E_VIRTCHNL_OP_DEL_VLAN: + valid_len = sizeof(struct i40e_virtchnl_vlan_filter_list); + if (msglen >= valid_len) { + struct i40e_virtchnl_vlan_filter_list *vfl = + (struct i40e_virtchnl_vlan_filter_list *)msg; + valid_len += vfl->num_elements * sizeof(u16); + if (vfl->num_elements == 0) + err_msg_format = true; + } + break; + case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE: + valid_len = sizeof(struct i40e_virtchnl_promisc_info); + break; + case I40E_VIRTCHNL_OP_GET_STATS: + valid_len = sizeof(struct i40e_virtchnl_queue_select); + break; + /* These are always errors coming from the VF. */ + case I40E_VIRTCHNL_OP_EVENT: + case I40E_VIRTCHNL_OP_UNKNOWN: + default: + return EPERM; + break; + } + /* few more checks */ + if ((valid_len != msglen) || (err_msg_format)) + return EINVAL; + else + return 0; +} +#endif + +/* +** ixlv_send_pf_msg +** +** Send message to PF and print status if failure. +*/ +static int +ixlv_send_pf_msg(struct ixlv_sc *sc, + enum i40e_virtchnl_ops op, u8 *msg, u16 len) +{ + struct i40e_hw *hw = &sc->hw; + device_t dev = sc->dev; + i40e_status err; + +#ifdef IXL_DEBUG + /* + ** Pre-validating messages to the PF + */ + int val_err; + val_err = ixl_vc_validate_vf_msg(sc, op, msg, len); + if (val_err) + device_printf(dev, "Error validating msg to PF for op %d," + " msglen %d: error %d\n", op, len, val_err); +#endif + + err = i40e_aq_send_msg_to_pf(hw, op, I40E_SUCCESS, msg, len, NULL); + if (err) + device_printf(dev, "Unable to send opcode %d to PF, " + "error %d, aq status %d\n", op, err, hw->aq.asq_last_status); + return err; +} + + +/* +** ixlv_send_api_ver +** +** Send API version admin queue message to the PF. The reply is not checked +** in this function. Returns 0 if the message was successfully +** sent, or one of the I40E_ADMIN_QUEUE_ERROR_ statuses if not. +*/ +int +ixlv_send_api_ver(struct ixlv_sc *sc) +{ + struct i40e_virtchnl_version_info vvi; + + vvi.major = I40E_VIRTCHNL_VERSION_MAJOR; + vvi.minor = I40E_VIRTCHNL_VERSION_MINOR; + + return ixlv_send_pf_msg(sc, I40E_VIRTCHNL_OP_VERSION, + (u8 *)&vvi, sizeof(vvi)); +} + +/* +** ixlv_verify_api_ver +** +** Compare API versions with the PF. Must be called after admin queue is +** initialized. Returns 0 if API versions match, EIO if +** they do not, or I40E_ERR_ADMIN_QUEUE_NO_WORK if the admin queue is empty. +*/ +int +ixlv_verify_api_ver(struct ixlv_sc *sc) +{ + struct i40e_virtchnl_version_info *pf_vvi; + struct i40e_hw *hw = &sc->hw; + struct i40e_arq_event_info event; + i40e_status err; + int retries = 0; + + event.buf_len = IXL_AQ_BUFSZ; + event.msg_buf = malloc(event.buf_len, M_IXL, M_NOWAIT); + if (!event.msg_buf) { + err = ENOMEM; + goto out; + } + + do { + if (++retries > IXLV_AQ_MAX_ERR) + goto out_alloc; + + /* NOTE: initial delay is necessary */ + i40e_msec_delay(100); + err = i40e_clean_arq_element(hw, &event, NULL); + } while (err == I40E_ERR_ADMIN_QUEUE_NO_WORK); + if (err) + goto out_alloc; + + err = (i40e_status)le32toh(event.desc.cookie_low); + if (err) { + err = EIO; + goto out_alloc; + } + + if ((enum i40e_virtchnl_ops)le32toh(event.desc.cookie_high) != + I40E_VIRTCHNL_OP_VERSION) { + DDPRINTF(sc->dev, "Received unexpected op response: %d\n", + le32toh(event.desc.cookie_high)); + err = EIO; + goto out_alloc; + } + + pf_vvi = (struct i40e_virtchnl_version_info *)event.msg_buf; + if ((pf_vvi->major > I40E_VIRTCHNL_VERSION_MAJOR) || + ((pf_vvi->major == I40E_VIRTCHNL_VERSION_MAJOR) && + (pf_vvi->minor > I40E_VIRTCHNL_VERSION_MINOR))) + err = EIO; + else + sc->pf_version = pf_vvi->minor; + +out_alloc: + free(event.msg_buf, M_IXL); +out: + return err; +} + +/* +** ixlv_send_vf_config_msg +** +** Send VF configuration request admin queue message to the PF. The reply +** is not checked in this function. Returns 0 if the message was +** successfully sent, or one of the I40E_ADMIN_QUEUE_ERROR_ statuses if not. +*/ +int +ixlv_send_vf_config_msg(struct ixlv_sc *sc) +{ + u32 caps; + + caps = I40E_VIRTCHNL_VF_OFFLOAD_L2 | + I40E_VIRTCHNL_VF_OFFLOAD_RSS_AQ | + I40E_VIRTCHNL_VF_OFFLOAD_RSS_REG | + I40E_VIRTCHNL_VF_OFFLOAD_VLAN; + + if (sc->pf_version) + return ixlv_send_pf_msg(sc, I40E_VIRTCHNL_OP_GET_VF_RESOURCES, + (u8 *)&caps, sizeof(caps)); + else + return ixlv_send_pf_msg(sc, I40E_VIRTCHNL_OP_GET_VF_RESOURCES, + NULL, 0); +} + +/* +** ixlv_get_vf_config +** +** Get VF configuration from PF and populate hw structure. Must be called after +** admin queue is initialized. Busy waits until response is received from PF, +** with maximum timeout. Response from PF is returned in the buffer for further +** processing by the caller. +*/ +int +ixlv_get_vf_config(struct ixlv_sc *sc) +{ + struct i40e_hw *hw = &sc->hw; + device_t dev = sc->dev; + struct i40e_arq_event_info event; + u16 len; + i40e_status err = 0; + u32 retries = 0; + + /* Note this assumes a single VSI */ + len = sizeof(struct i40e_virtchnl_vf_resource) + + sizeof(struct i40e_virtchnl_vsi_resource); + event.buf_len = len; + event.msg_buf = malloc(event.buf_len, M_IXL, M_NOWAIT); + if (!event.msg_buf) { + err = ENOMEM; + goto out; + } + + for (;;) { + err = i40e_clean_arq_element(hw, &event, NULL); + if (err == I40E_ERR_ADMIN_QUEUE_NO_WORK) { + if (++retries <= IXLV_AQ_MAX_ERR) + i40e_msec_delay(10); + } else if ((enum i40e_virtchnl_ops)le32toh(event.desc.cookie_high) != + I40E_VIRTCHNL_OP_GET_VF_RESOURCES) { + DDPRINTF(dev, "Received a response from PF," + " opcode %d, error %d", + le32toh(event.desc.cookie_high), + le32toh(event.desc.cookie_low)); + retries++; + continue; + } else { + err = (i40e_status)le32toh(event.desc.cookie_low); + if (err) { + device_printf(dev, "%s: Error returned from PF," + " opcode %d, error %d\n", __func__, + le32toh(event.desc.cookie_high), + le32toh(event.desc.cookie_low)); + err = EIO; + goto out_alloc; + } + /* We retrieved the config message, with no errors */ + break; + } + + if (retries > IXLV_AQ_MAX_ERR) { + INIT_DBG_DEV(dev, "Did not receive response after %d tries.", + retries); + err = ETIMEDOUT; + goto out_alloc; + } + } + + memcpy(sc->vf_res, event.msg_buf, min(event.msg_len, len)); + i40e_vf_parse_hw_config(hw, sc->vf_res); + +out_alloc: + free(event.msg_buf, M_IXL); +out: + return err; +} + +/* +** ixlv_configure_queues +** +** Request that the PF set up our queues. +*/ +void +ixlv_configure_queues(struct ixlv_sc *sc) +{ + device_t dev = sc->dev; + struct ixl_vsi *vsi = &sc->vsi; + struct ixl_tx_queue *tx_que = vsi->tx_queues; + struct ixl_rx_queue *rx_que = vsi->rx_queues; + struct tx_ring *txr; + struct rx_ring *rxr; + if_shared_ctx_t sctx; + int len, pairs; + + struct i40e_virtchnl_vsi_queue_config_info *vqci; + struct i40e_virtchnl_queue_pair_info *vqpi; + + MPASS(vsi->num_tx_queues == vsi->num_rx_queues); + pairs = vsi->num_tx_queues; + len = sizeof(struct i40e_virtchnl_vsi_queue_config_info) + + (sizeof(struct i40e_virtchnl_queue_pair_info) * pairs); + vqci = malloc(len, M_IXL, M_NOWAIT | M_ZERO); + if (!vqci) { + device_printf(dev, "%s: unable to allocate memory\n", __func__); + ixl_vc_schedule_retry(&sc->vc_mgr); + return; + } + sctx = iflib_get_sctx(sc->vsi.ctx); + vqci->vsi_id = sc->vsi_res->vsi_id; + vqci->num_queue_pairs = pairs; + vqpi = vqci->qpair; + /* Size check is not needed here - HW max is 16 queue pairs, and we + * can fit info for 31 of them into the AQ buffer before it overflows. + */ + for (int i = 0; i < pairs; i++, tx_que++, rx_que++, vqpi++) { + txr = &tx_que->txr; + rxr = &rx_que->rxr; + vqpi->txq.vsi_id = vqci->vsi_id; + vqpi->txq.queue_id = i; + vqpi->txq.ring_len = sctx->isc_ntxd; + vqpi->txq.dma_ring_addr = txr->tx_paddr; + /* Enable Head writeback */ + vqpi->txq.headwb_enabled = 1; + vqpi->txq.dma_headwb_addr = txr->tx_paddr + + (sctx->isc_ntxd * sizeof(struct i40e_tx_desc)); + + vqpi->rxq.vsi_id = vqci->vsi_id; + vqpi->rxq.queue_id = i; + vqpi->rxq.ring_len = sctx->isc_ntxd; + vqpi->rxq.dma_ring_addr = rxr->rx_paddr; + vqpi->rxq.max_pkt_size = vsi->max_frame_size; + vqpi->rxq.databuffer_size = rxr->mbuf_sz; + vqpi->rxq.splithdr_enabled = 0; + } + + ixlv_send_pf_msg(sc, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, + (u8 *)vqci, len); + free(vqci, M_IXL); +} + +/* +** ixlv_enable_queues +** +** Request that the PF enable all of our queues. +*/ +void +ixlv_enable_queues(struct ixlv_sc *sc) +{ + struct i40e_virtchnl_queue_select vqs; + + vqs.vsi_id = sc->vsi_res->vsi_id; + vqs.tx_queues = (1 << sc->vsi_res->num_queue_pairs) - 1; + vqs.rx_queues = vqs.tx_queues; + ixlv_send_pf_msg(sc, I40E_VIRTCHNL_OP_ENABLE_QUEUES, + (u8 *)&vqs, sizeof(vqs)); +} + +/* +** ixlv_disable_queues +** +** Request that the PF disable all of our queues. +*/ +void +ixlv_disable_queues(struct ixlv_sc *sc) +{ + struct i40e_virtchnl_queue_select vqs; + + vqs.vsi_id = sc->vsi_res->vsi_id; + vqs.tx_queues = (1 << sc->vsi_res->num_queue_pairs) - 1; + vqs.rx_queues = vqs.tx_queues; + ixlv_send_pf_msg(sc, I40E_VIRTCHNL_OP_DISABLE_QUEUES, + (u8 *)&vqs, sizeof(vqs)); +} + +/* +** ixlv_map_queues +** +** Request that the PF map queues to interrupt vectors. Misc causes, including +** admin queue, are always mapped to vector 0. +*/ +void +ixlv_map_queues(struct ixlv_sc *sc) +{ + struct i40e_virtchnl_irq_map_info *vm; + int i, q, len; + struct ixl_vsi *vsi = &sc->vsi; + struct ixl_tx_queue *tx_que = vsi->tx_queues; + struct ixl_rx_queue *rx_que = vsi->rx_queues; + + /* How many queue vectors, adminq uses one */ + q = sc->msix - 1; + + len = sizeof(struct i40e_virtchnl_irq_map_info) + + (sc->msix * sizeof(struct i40e_virtchnl_vector_map)); + vm = malloc(len, M_IXL, M_NOWAIT); + if (!vm) { + printf("%s: unable to allocate memory\n", __func__); + ixl_vc_schedule_retry(&sc->vc_mgr); + return; + } + + vm->num_vectors = sc->msix; + /* Queue vectors first */ + for (i = 0; i < q; i++, tx_que++, rx_que++) { + vm->vecmap[i].vsi_id = sc->vsi_res->vsi_id; + vm->vecmap[i].vector_id = i + 1; /* first is adminq */ + vm->vecmap[i].txq_map = (1 << tx_que->txr.me); + vm->vecmap[i].rxq_map = (1 << rx_que->rxr.me); + vm->vecmap[i].rxitr_idx = 0; + vm->vecmap[i].txitr_idx = 0; + } + + /* Misc vector last - this is only for AdminQ messages */ + vm->vecmap[i].vsi_id = sc->vsi_res->vsi_id; + vm->vecmap[i].vector_id = 0; + vm->vecmap[i].txq_map = 0; + vm->vecmap[i].rxq_map = 0; + vm->vecmap[i].rxitr_idx = 0; + vm->vecmap[i].txitr_idx = 0; + + ixlv_send_pf_msg(sc, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, + (u8 *)vm, len); + free(vm, M_IXL); +} + +/* +** Scan the Filter List looking for vlans that need +** to be added, then create the data to hand to the AQ +** for handling. +*/ +void +ixlv_add_vlans(struct ixlv_sc *sc) +{ + struct i40e_virtchnl_vlan_filter_list *v; + struct ixlv_vlan_filter *f, *ftmp; + device_t dev = sc->dev; + int len, i = 0, cnt = 0; + + /* Get count of VLAN filters to add */ + SLIST_FOREACH(f, sc->vlan_filters, next) { + if (f->flags & IXL_FILTER_ADD) + cnt++; + } + + if (!cnt) { /* no work... */ + ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_ADD_VLAN_FILTER, + I40E_SUCCESS); + return; + } + + len = sizeof(struct i40e_virtchnl_vlan_filter_list) + + (cnt * sizeof(u16)); + + if (len > IXL_AQ_BUF_SZ) { + device_printf(dev, "%s: Exceeded Max AQ Buf size\n", + __func__); + ixl_vc_schedule_retry(&sc->vc_mgr); + return; + } + + v = malloc(len, M_IXL, M_NOWAIT); + if (!v) { + device_printf(dev, "%s: unable to allocate memory\n", + __func__); + ixl_vc_schedule_retry(&sc->vc_mgr); + return; + } + + v->vsi_id = sc->vsi_res->vsi_id; + v->num_elements = cnt; + + /* Scan the filter array */ + SLIST_FOREACH_SAFE(f, sc->vlan_filters, next, ftmp) { + if (f->flags & IXL_FILTER_ADD) { + bcopy(&f->vlan, &v->vlan_id[i], sizeof(u16)); + f->flags = IXL_FILTER_USED; + i++; + } + if (i == cnt) + break; + } + // ERJ: Should this be taken out? + if (i == 0) { /* Should not happen... */ + device_printf(dev, "%s: i == 0?\n", __func__); + ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_ADD_VLAN_FILTER, + I40E_SUCCESS); + return; + } + + ixlv_send_pf_msg(sc, I40E_VIRTCHNL_OP_ADD_VLAN, (u8 *)v, len); + free(v, M_IXL); + /* add stats? */ +} + +/* +** Scan the Filter Table looking for vlans that need +** to be removed, then create the data to hand to the AQ +** for handling. +*/ +void +ixlv_del_vlans(struct ixlv_sc *sc) +{ + device_t dev = sc->dev; + struct i40e_virtchnl_vlan_filter_list *v; + struct ixlv_vlan_filter *f, *ftmp; + int len, i = 0, cnt = 0; + + /* Get count of VLAN filters to delete */ + SLIST_FOREACH(f, sc->vlan_filters, next) { + if (f->flags & IXL_FILTER_DEL) + cnt++; + } + + if (!cnt) { /* no work... */ + ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_DEL_VLAN_FILTER, + I40E_SUCCESS); + return; + } + + len = sizeof(struct i40e_virtchnl_vlan_filter_list) + + (cnt * sizeof(u16)); + + if (len > IXL_AQ_BUF_SZ) { + device_printf(dev, "%s: Exceeded Max AQ Buf size\n", + __func__); + ixl_vc_schedule_retry(&sc->vc_mgr); + return; + } + + v = malloc(len, M_IXL, M_NOWAIT | M_ZERO); + if (!v) { + device_printf(dev, "%s: unable to allocate memory\n", + __func__); + ixl_vc_schedule_retry(&sc->vc_mgr); + return; + } + + v->vsi_id = sc->vsi_res->vsi_id; + v->num_elements = cnt; + + /* Scan the filter array */ + SLIST_FOREACH_SAFE(f, sc->vlan_filters, next, ftmp) { + if (f->flags & IXL_FILTER_DEL) { + bcopy(&f->vlan, &v->vlan_id[i], sizeof(u16)); + i++; + SLIST_REMOVE(sc->vlan_filters, f, ixlv_vlan_filter, next); + free(f, M_IXL); + } + if (i == cnt) + break; + } + // ERJ: Take this out? + if (i == 0) { /* Should not happen... */ + device_printf(dev, "%s: i == 0?\n", __func__); + ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_DEL_VLAN_FILTER, + I40E_SUCCESS); + return; + } + + ixlv_send_pf_msg(sc, I40E_VIRTCHNL_OP_DEL_VLAN, (u8 *)v, len); + free(v, M_IXL); + /* add stats? */ +} + + +/* +** This routine takes additions to the vsi filter +** table and creates an Admin Queue call to create +** the filters in the hardware. +*/ +void +ixlv_add_ether_filters(struct ixlv_sc *sc) +{ + struct i40e_virtchnl_ether_addr_list *a; + struct ixlv_mac_filter *f; + device_t dev = sc->dev; + int len, j = 0, cnt = 0; + + /* Get count of MAC addresses to add */ + SLIST_FOREACH(f, sc->mac_filters, next) { + if (f->flags & IXL_FILTER_ADD) + cnt++; + } + if (cnt == 0) { /* Should not happen... */ + DDPRINTF(dev, "cnt == 0, exiting..."); + ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_ADD_MAC_FILTER, + I40E_SUCCESS); + return; + } + + len = sizeof(struct i40e_virtchnl_ether_addr_list) + + (cnt * sizeof(struct i40e_virtchnl_ether_addr)); + + a = malloc(len, M_IXL, M_NOWAIT | M_ZERO); + if (a == NULL) { + device_printf(dev, "%s: Failed to get memory for " + "virtchnl_ether_addr_list\n", __func__); + ixl_vc_schedule_retry(&sc->vc_mgr); + return; + } + a->vsi_id = sc->vsi.id; + a->num_elements = cnt; + + /* Scan the filter array */ + SLIST_FOREACH(f, sc->mac_filters, next) { + if (f->flags & IXL_FILTER_ADD) { + bcopy(f->macaddr, a->list[j].addr, ETHER_ADDR_LEN); + f->flags &= ~IXL_FILTER_ADD; + j++; + + DDPRINTF(dev, "ADD: " MAC_FORMAT, + MAC_FORMAT_ARGS(f->macaddr)); + } + if (j == cnt) + break; + } + DDPRINTF(dev, "len %d, j %d, cnt %d", + len, j, cnt); + ixlv_send_pf_msg(sc, + I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, (u8 *)a, len); + /* add stats? */ + free(a, M_IXL); + return; +} + +/* +** This routine takes filters flagged for deletion in the +** sc MAC filter list and creates an Admin Queue call +** to delete those filters in the hardware. +*/ +void +ixlv_del_ether_filters(struct ixlv_sc *sc) +{ + struct i40e_virtchnl_ether_addr_list *d; + device_t dev = sc->dev; + struct ixlv_mac_filter *f, *f_temp; + int len, j = 0, cnt = 0; + + /* Get count of MAC addresses to delete */ + SLIST_FOREACH(f, sc->mac_filters, next) { + if (f->flags & IXL_FILTER_DEL) + cnt++; + } + if (cnt == 0) { + DDPRINTF(dev, "cnt == 0, exiting..."); + ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_DEL_MAC_FILTER, + I40E_SUCCESS); + return; + } + + len = sizeof(struct i40e_virtchnl_ether_addr_list) + + (cnt * sizeof(struct i40e_virtchnl_ether_addr)); + + d = malloc(len, M_IXL, M_NOWAIT | M_ZERO); + if (d == NULL) { + device_printf(dev, "%s: Failed to get memory for " + "virtchnl_ether_addr_list\n", __func__); + ixl_vc_schedule_retry(&sc->vc_mgr); + return; + } + d->vsi_id = sc->vsi.id; + d->num_elements = cnt; + + /* Scan the filter array */ + SLIST_FOREACH_SAFE(f, sc->mac_filters, next, f_temp) { + if (f->flags & IXL_FILTER_DEL) { + bcopy(f->macaddr, d->list[j].addr, ETHER_ADDR_LEN); + DDPRINTF(dev, "DEL: " MAC_FORMAT, + MAC_FORMAT_ARGS(f->macaddr)); + j++; + SLIST_REMOVE(sc->mac_filters, f, ixlv_mac_filter, next); + free(f, M_IXL); + } + if (j == cnt) + break; + } + ixlv_send_pf_msg(sc, + I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS, (u8 *)d, len); + /* add stats? */ + free(d, M_IXL); + return; +} + +/* +** ixlv_request_reset +** Request that the PF reset this VF. No response is expected. +*/ +void +ixlv_request_reset(struct ixlv_sc *sc) +{ + /* + ** Set the reset status to "in progress" before + ** the request, this avoids any possibility of + ** a mistaken early detection of completion. + */ + wr32(&sc->hw, I40E_VFGEN_RSTAT, I40E_VFR_INPROGRESS); + ixlv_send_pf_msg(sc, I40E_VIRTCHNL_OP_RESET_VF, NULL, 0); +} + +/* +** ixlv_request_stats +** Request the statistics for this VF's VSI from PF. +*/ +void +ixlv_request_stats(struct ixlv_sc *sc) +{ + struct i40e_virtchnl_queue_select vqs; + int error = 0; + + vqs.vsi_id = sc->vsi_res->vsi_id; + /* Low priority, we don't need to error check */ + error = ixlv_send_pf_msg(sc, I40E_VIRTCHNL_OP_GET_STATS, + (u8 *)&vqs, sizeof(vqs)); +#ifdef IXL_DEBUG + if (error) + device_printf(sc->dev, "Error sending stats request to PF: %d\n", error); +#endif +} + +/* +** Updates driver's stats counters with VSI stats returned from PF. +*/ +void +ixlv_update_stats_counters(struct ixlv_sc *sc, struct i40e_eth_stats *es) +{ + struct ixl_vsi *vsi = &sc->vsi; + uint64_t tx_discards; + + tx_discards = es->tx_discards; + + /* Update ifnet stats */ + IXL_SET_IPACKETS(vsi, es->rx_unicast + + es->rx_multicast + + es->rx_broadcast); + IXL_SET_OPACKETS(vsi, es->tx_unicast + + es->tx_multicast + + es->tx_broadcast); + IXL_SET_IBYTES(vsi, es->rx_bytes); + IXL_SET_OBYTES(vsi, es->tx_bytes); + IXL_SET_IMCASTS(vsi, es->rx_multicast); + IXL_SET_OMCASTS(vsi, es->tx_multicast); + + IXL_SET_OERRORS(vsi, es->tx_errors); + IXL_SET_IQDROPS(vsi, es->rx_discards); + IXL_SET_OQDROPS(vsi, tx_discards); + IXL_SET_NOPROTO(vsi, es->rx_unknown_protocol); + IXL_SET_COLLISIONS(vsi, 0); + + vsi->eth_stats = *es; +} + +/* +** ixlv_vc_completion +** +** Asynchronous completion function for admin queue messages. Rather than busy +** wait, we fire off our requests and assume that no errors will be returned. +** This function handles the reply messages. +*/ +void +ixlv_vc_completion(struct ixlv_sc *sc, + enum i40e_virtchnl_ops v_opcode, + i40e_status v_retval, u8 *msg, u16 msglen) +{ + device_t dev = sc->dev; + struct ixl_vsi *vsi = &sc->vsi; + + if (v_opcode == I40E_VIRTCHNL_OP_EVENT) { + struct i40e_virtchnl_pf_event *vpe = + (struct i40e_virtchnl_pf_event *)msg; + + switch (vpe->event) { + case I40E_VIRTCHNL_EVENT_LINK_CHANGE: +#ifdef IXL_DEBUG + device_printf(dev, "Link change: status %d, speed %d\n", + vpe->event_data.link_event.link_status, + vpe->event_data.link_event.link_speed); +#endif + sc->link_up = + vpe->event_data.link_event.link_status; + sc->link_speed = + vpe->event_data.link_event.link_speed; + ixlv_update_link_status(sc); + break; + case I40E_VIRTCHNL_EVENT_RESET_IMPENDING: + device_printf(dev, "PF initiated reset!\n"); + sc->init_state = IXLV_RESET_PENDING; + ixlv_init(sc); + break; + default: + device_printf(dev, "%s: Unknown event %d from AQ\n", + __func__, vpe->event); + break; + } + + return; + } + + /* Catch-all error response */ + if (v_retval) { + device_printf(dev, + "%s: AQ returned error %d to our request %d!\n", + __func__, v_retval, v_opcode); + } + +#ifdef IXL_DEBUG + if (v_opcode != I40E_VIRTCHNL_OP_GET_STATS) + DDPRINTF(dev, "opcode %d", v_opcode); +#endif + + switch (v_opcode) { + case I40E_VIRTCHNL_OP_GET_STATS: + ixlv_update_stats_counters(sc, (struct i40e_eth_stats *)msg); + break; + case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS: + ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_ADD_MAC_FILTER, + v_retval); + if (v_retval) { + device_printf(dev, "WARNING: Error adding VF mac filter!\n"); + device_printf(dev, "WARNING: Device may not receive traffic!\n"); + } + break; + case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS: + ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_DEL_MAC_FILTER, + v_retval); + break; + case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE: + ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_CONFIGURE_PROMISC, + v_retval); + break; + case I40E_VIRTCHNL_OP_ADD_VLAN: + ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_ADD_VLAN_FILTER, + v_retval); + break; + case I40E_VIRTCHNL_OP_DEL_VLAN: + ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_DEL_VLAN_FILTER, + v_retval); + break; + case I40E_VIRTCHNL_OP_ENABLE_QUEUES: + ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_ENABLE_QUEUES, + v_retval); + if (v_retval == 0) { + /* Update link status */ + ixlv_update_link_status(sc); + /* Turn on all interrupts */ + ixlv_enable_intr(vsi); + /* And inform the stack we're ready */ + vsi->ifp->if_drv_flags |= IFF_DRV_RUNNING; + vsi->ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; + } + break; + case I40E_VIRTCHNL_OP_DISABLE_QUEUES: + ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_DISABLE_QUEUES, + v_retval); + if (v_retval == 0) { + /* Turn off all interrupts */ + ixlv_disable_intr(vsi); + /* Tell the stack that the interface is no longer active */ + vsi->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); + } + break; + case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES: + ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_CONFIGURE_QUEUES, + v_retval); + break; + case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP: + ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_MAP_VECTORS, + v_retval); + break; + default: + device_printf(dev, + "%s: Received unexpected message %d from PF.\n", + __func__, v_opcode); + break; + } + return; +} + +static void +ixl_vc_send_cmd(struct ixlv_sc *sc, uint32_t request) +{ + + switch (request) { + case IXLV_FLAG_AQ_MAP_VECTORS: + ixlv_map_queues(sc); + break; + + case IXLV_FLAG_AQ_ADD_MAC_FILTER: + ixlv_add_ether_filters(sc); + break; + + case IXLV_FLAG_AQ_ADD_VLAN_FILTER: + ixlv_add_vlans(sc); + break; + + case IXLV_FLAG_AQ_DEL_MAC_FILTER: + ixlv_del_ether_filters(sc); + break; + + case IXLV_FLAG_AQ_DEL_VLAN_FILTER: + ixlv_del_vlans(sc); + break; + + case IXLV_FLAG_AQ_CONFIGURE_QUEUES: + ixlv_configure_queues(sc); + break; + + case IXLV_FLAG_AQ_DISABLE_QUEUES: + ixlv_disable_queues(sc); + break; + + case IXLV_FLAG_AQ_ENABLE_QUEUES: + ixlv_enable_queues(sc); + break; + } +} + +void +ixl_vc_init_mgr(struct ixlv_sc *sc, struct ixl_vc_mgr *mgr) +{ + mgr->sc = sc; + mgr->current = NULL; + TAILQ_INIT(&mgr->pending); +} + +static void +ixl_vc_process_completion(struct ixl_vc_mgr *mgr, enum i40e_status_code err) +{ + struct ixl_vc_cmd *cmd; + + cmd = mgr->current; + mgr->current = NULL; + cmd->flags &= ~IXLV_VC_CMD_FLAG_BUSY; + + cmd->callback(cmd, cmd->arg, err); + ixl_vc_process_next(mgr); +} + +static void +ixl_vc_process_resp(struct ixl_vc_mgr *mgr, uint32_t request, + enum i40e_status_code err) +{ + struct ixl_vc_cmd *cmd; + + cmd = mgr->current; + if (cmd == NULL || cmd->request != request) + return; + + callout_stop(&mgr->callout); + ixl_vc_process_completion(mgr, err); +} + +static void +ixl_vc_cmd_timeout(void *arg) +{ + struct ixl_vc_mgr *mgr = (struct ixl_vc_mgr *)arg; + + ixl_vc_process_completion(mgr, I40E_ERR_TIMEOUT); +} + +static void +ixl_vc_cmd_retry(void *arg) +{ + struct ixl_vc_mgr *mgr = (struct ixl_vc_mgr *)arg; + + ixl_vc_send_current(mgr); +} + +static void +ixl_vc_send_current(struct ixl_vc_mgr *mgr) +{ + struct ixl_vc_cmd *cmd; + + cmd = mgr->current; + ixl_vc_send_cmd(mgr->sc, cmd->request); + callout_reset(&mgr->callout, IXLV_VC_TIMEOUT, ixl_vc_cmd_timeout, mgr); +} + +static void +ixl_vc_process_next(struct ixl_vc_mgr *mgr) +{ + struct ixl_vc_cmd *cmd; + + if (mgr->current != NULL) + return; + + if (TAILQ_EMPTY(&mgr->pending)) + return; + + cmd = TAILQ_FIRST(&mgr->pending); + TAILQ_REMOVE(&mgr->pending, cmd, next); + + mgr->current = cmd; + ixl_vc_send_current(mgr); +} + +static void +ixl_vc_schedule_retry(struct ixl_vc_mgr *mgr) +{ + + callout_reset(&mgr->callout, howmany(hz, 100), ixl_vc_cmd_retry, mgr); +} + +void +ixl_vc_enqueue(struct ixl_vc_mgr *mgr, struct ixl_vc_cmd *cmd, + uint32_t req, ixl_vc_callback_t *callback, void *arg) +{ + + if (cmd->flags & IXLV_VC_CMD_FLAG_BUSY) { + if (mgr->current == cmd) + mgr->current = NULL; + else + TAILQ_REMOVE(&mgr->pending, cmd, next); + } + + cmd->request = req; + cmd->callback = callback; + cmd->arg = arg; + cmd->flags |= IXLV_VC_CMD_FLAG_BUSY; + TAILQ_INSERT_TAIL(&mgr->pending, cmd, next); + + ixl_vc_process_next(mgr); +} + +void +ixl_vc_flush(struct ixl_vc_mgr *mgr) +{ + struct ixl_vc_cmd *cmd; + + KASSERT(TAILQ_EMPTY(&mgr->pending) || mgr->current != NULL, + ("ixlv: pending commands waiting but no command in progress")); + + cmd = mgr->current; + if (cmd != NULL) { + mgr->current = NULL; + cmd->flags &= ~IXLV_VC_CMD_FLAG_BUSY; + cmd->callback(cmd, cmd->arg, I40E_ERR_ADAPTER_STOPPED); + } + + while ((cmd = TAILQ_FIRST(&mgr->pending)) != NULL) { + TAILQ_REMOVE(&mgr->pending, cmd, next); + cmd->flags &= ~IXLV_VC_CMD_FLAG_BUSY; + cmd->callback(cmd, cmd->arg, I40E_ERR_ADAPTER_STOPPED); + } + + callout_stop(&mgr->callout); +} + Index: sys/dev/ixl/ixl.h =================================================================== --- sys/dev/ixl/ixl.h +++ sys/dev/ixl/ixl.h @@ -1,655 +1,19 @@ -/****************************************************************************** - - Copyright (c) 2013-2015, Intel Corporation - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - 3. Neither the name of the Intel Corporation nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - -******************************************************************************/ -/*$FreeBSD$*/ - +#include "opt_iflib.h" #ifndef _IXL_H_ #define _IXL_H_ - #include #include -#include -#include #include #include #include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef PCI_IOV -#include -#include -#include -#endif - -#include "i40e_type.h" -#include "i40e_prototype.h" - -#if defined(IXL_DEBUG) || defined(IXL_DEBUG_SYSCTL) -#include - -#define MAC_FORMAT "%02x:%02x:%02x:%02x:%02x:%02x" -#define MAC_FORMAT_ARGS(mac_addr) \ - (mac_addr)[0], (mac_addr)[1], (mac_addr)[2], (mac_addr)[3], \ - (mac_addr)[4], (mac_addr)[5] -#define ON_OFF_STR(is_set) ((is_set) ? "On" : "Off") -#endif /* IXL_DEBUG || IXL_DEBUG_SYSCTL */ - -#ifdef IXL_DEBUG -/* Enable debug sysctls */ -#ifndef IXL_DEBUG_SYSCTL -#define IXL_DEBUG_SYSCTL 1 -#endif - -#define _DBG_PRINTF(S, ...) printf("%s: " S "\n", __func__, ##__VA_ARGS__) -#define _DEV_DBG_PRINTF(dev, S, ...) device_printf(dev, "%s: " S "\n", __func__, ##__VA_ARGS__) -#define _IF_DBG_PRINTF(ifp, S, ...) if_printf(ifp, "%s: " S "\n", __func__, ##__VA_ARGS__) - -/* Defines for printing generic debug information */ -#define DPRINTF(...) _DBG_PRINTF(__VA_ARGS__) -#define DDPRINTF(...) _DEV_DBG_PRINTF(__VA_ARGS__) -#define IDPRINTF(...) _IF_DBG_PRINTF(__VA_ARGS__) - -/* Defines for printing specific debug information */ -#define DEBUG_INIT 1 -#define DEBUG_IOCTL 1 -#define DEBUG_HW 1 - -#define INIT_DEBUGOUT(...) if (DEBUG_INIT) _DBG_PRINTF(__VA_ARGS__) -#define INIT_DBG_DEV(...) if (DEBUG_INIT) _DEV_DBG_PRINTF(__VA_ARGS__) -#define INIT_DBG_IF(...) if (DEBUG_INIT) _IF_DBG_PRINTF(__VA_ARGS__) - -#define IOCTL_DEBUGOUT(...) if (DEBUG_IOCTL) _DBG_PRINTF(__VA_ARGS__) -#define IOCTL_DBG_IF2(ifp, S, ...) if (DEBUG_IOCTL) \ - if_printf(ifp, S "\n", ##__VA_ARGS__) -#define IOCTL_DBG_IF(...) if (DEBUG_IOCTL) _IF_DBG_PRINTF(__VA_ARGS__) - -#define HW_DEBUGOUT(...) if (DEBUG_HW) _DBG_PRINTF(__VA_ARGS__) - -#else /* no IXL_DEBUG */ -#define DEBUG_INIT 0 -#define DEBUG_IOCTL 0 -#define DEBUG_HW 0 - -#define DPRINTF(...) -#define DDPRINTF(...) -#define IDPRINTF(...) - -#define INIT_DEBUGOUT(...) -#define INIT_DBG_DEV(...) -#define INIT_DBG_IF(...) -#define IOCTL_DEBUGOUT(...) -#define IOCTL_DBG_IF2(...) -#define IOCTL_DBG_IF(...) -#define HW_DEBUGOUT(...) -#endif /* IXL_DEBUG */ - -/* Tunables */ - -/* - * Ring Descriptors Valid Range: 32-4096 Default Value: 1024 This value is the - * number of tx/rx descriptors allocated by the driver. Increasing this - * value allows the driver to queue more operations. Each descriptor is 16 - * or 32 bytes (configurable in FVL) - */ -#define DEFAULT_RING 1024 -#define PERFORM_RING 2048 -#define MAX_RING 4096 -#define MIN_RING 32 - -/* -** Default number of entries in Tx queue buf_ring. -*/ -#define SMALL_TXBRSZ 4096 -/* This may require mbuf cluster tuning */ -#define DEFAULT_TXBRSZ (SMALL_TXBRSZ * SMALL_TXBRSZ) - -/* Alignment for rings */ -#define DBA_ALIGN 128 - -/* - * This parameter controls the maximum no of times the driver will loop in - * the isr. Minimum Value = 1 - */ -#define MAX_LOOP 10 - -/* - * This is the max watchdog interval, ie. the time that can - * pass between any two TX clean operations, such only happening - * when the TX hardware is functioning. - */ -#define IXL_WATCHDOG (10 * hz) - -/* - * This parameters control when the driver calls the routine to reclaim - * transmit descriptors. - */ -#define IXL_TX_CLEANUP_THRESHOLD (que->num_desc / 8) -#define IXL_TX_OP_THRESHOLD (que->num_desc / 32) - -/* Flow control constants */ -#define IXL_FC_PAUSE 0xFFFF -#define IXL_FC_HI 0x20000 -#define IXL_FC_LO 0x10000 - -#define MAX_MULTICAST_ADDR 128 - -#define IXL_BAR 3 -#define IXL_ADM_LIMIT 2 -#define IXL_TSO_SIZE 65535 -#define IXL_TX_BUF_SZ ((u32) 1514) -#define IXL_AQ_BUF_SZ ((u32) 4096) -#define IXL_RX_HDR 128 -/* Controls the length of the Admin Queue */ -#define IXL_AQ_LEN 256 -#define IXL_AQ_LEN_MAX 1024 -#define IXL_AQ_BUFSZ 4096 -#define IXL_RX_LIMIT 512 -#define IXL_RX_ITR 0 -#define IXL_TX_ITR 1 -#define IXL_ITR_NONE 3 -#define IXL_QUEUE_EOL 0x7FF -#define IXL_MAX_FRAME 0x2600 -#define IXL_MAX_TX_SEGS 8 -#define IXL_MAX_TSO_SEGS 66 -#define IXL_SPARSE_CHAIN 6 -#define IXL_QUEUE_HUNG 0x80000000 -#define IXL_KEYSZ 10 - -#define IXL_VF_MAX_BUFFER 0x3F80 -#define IXL_VF_MAX_HDR_BUFFER 0x840 -#define IXL_VF_MAX_FRAME 0x3FFF - -/* ERJ: hardware can support ~1.5k filters between all functions */ -#define IXL_MAX_FILTERS 256 -#define IXL_MAX_TX_BUSY 10 - -#define IXL_NVM_VERSION_LO_SHIFT 0 -#define IXL_NVM_VERSION_LO_MASK (0xff << IXL_NVM_VERSION_LO_SHIFT) -#define IXL_NVM_VERSION_HI_SHIFT 12 -#define IXL_NVM_VERSION_HI_MASK (0xf << IXL_NVM_VERSION_HI_SHIFT) - - -/* - * Interrupt Moderation parameters - */ -#define IXL_MAX_ITR 0x07FF -#define IXL_ITR_100K 0x0005 -#define IXL_ITR_20K 0x0019 -#define IXL_ITR_8K 0x003E -#define IXL_ITR_4K 0x007A -#define IXL_ITR_DYNAMIC 0x8000 -#define IXL_LOW_LATENCY 0 -#define IXL_AVE_LATENCY 1 -#define IXL_BULK_LATENCY 2 - -/* MacVlan Flags */ -#define IXL_FILTER_USED (u16)(1 << 0) -#define IXL_FILTER_VLAN (u16)(1 << 1) -#define IXL_FILTER_ADD (u16)(1 << 2) -#define IXL_FILTER_DEL (u16)(1 << 3) -#define IXL_FILTER_MC (u16)(1 << 4) - -/* used in the vlan field of the filter when not a vlan */ -#define IXL_VLAN_ANY -1 - -#define CSUM_OFFLOAD_IPV4 (CSUM_IP|CSUM_TCP|CSUM_UDP|CSUM_SCTP) -#define CSUM_OFFLOAD_IPV6 (CSUM_TCP_IPV6|CSUM_UDP_IPV6|CSUM_SCTP_IPV6) -#define CSUM_OFFLOAD (CSUM_OFFLOAD_IPV4|CSUM_OFFLOAD_IPV6|CSUM_TSO) - -/* Misc flags for ixl_vsi.flags */ -#define IXL_FLAGS_KEEP_TSO4 (1 << 0) -#define IXL_FLAGS_KEEP_TSO6 (1 << 1) - -#define IXL_VF_RESET_TIMEOUT 100 - -#define IXL_VSI_DATA_PORT 0x01 +MALLOC_DECLARE(M_IXL); -#define IXLV_MAX_QUEUES 16 -#define IXL_MAX_VSI_QUEUES (2 * (I40E_VSILAN_QTABLE_MAX_INDEX + 1)) - -#define IXL_RX_CTX_BASE_UNITS 128 -#define IXL_TX_CTX_BASE_UNITS 128 - -#define IXL_VPINT_LNKLSTN_REG(hw, vector, vf_num) \ - I40E_VPINT_LNKLSTN(((vector) - 1) + \ - (((hw)->func_caps.num_msix_vectors_vf - 1) * (vf_num))) - -#define IXL_VFINT_DYN_CTLN_REG(hw, vector, vf_num) \ - I40E_VFINT_DYN_CTLN(((vector) - 1) + \ - (((hw)->func_caps.num_msix_vectors_vf - 1) * (vf_num))) - -#define IXL_PF_PCI_CIAA_VF_DEVICE_STATUS 0xAA - -#define IXL_PF_PCI_CIAD_VF_TRANS_PENDING_MASK 0x20 - -#define IXL_GLGEN_VFLRSTAT_INDEX(glb_vf) ((glb_vf) / 32) -#define IXL_GLGEN_VFLRSTAT_MASK(glb_vf) (1 << ((glb_vf) % 32)) - -#define IXL_MAX_ITR_IDX 3 - -#define IXL_END_OF_INTR_LNKLST 0x7FF - -#define IXL_TX_LOCK(_sc) mtx_lock(&(_sc)->mtx) -#define IXL_TX_UNLOCK(_sc) mtx_unlock(&(_sc)->mtx) -#define IXL_TX_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->mtx) -#define IXL_TX_TRYLOCK(_sc) mtx_trylock(&(_sc)->mtx) -#define IXL_TX_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->mtx, MA_OWNED) - -#define IXL_RX_LOCK(_sc) mtx_lock(&(_sc)->mtx) -#define IXL_RX_UNLOCK(_sc) mtx_unlock(&(_sc)->mtx) -#define IXL_RX_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->mtx) - -#if __FreeBSD_version >= 1100036 -#define IXL_SET_IPACKETS(vsi, count) (vsi)->ipackets = (count) -#define IXL_SET_IERRORS(vsi, count) (vsi)->ierrors = (count) -#define IXL_SET_OPACKETS(vsi, count) (vsi)->opackets = (count) -#define IXL_SET_OERRORS(vsi, count) (vsi)->oerrors = (count) -#define IXL_SET_COLLISIONS(vsi, count) /* Do nothing; collisions is always 0. */ -#define IXL_SET_IBYTES(vsi, count) (vsi)->ibytes = (count) -#define IXL_SET_OBYTES(vsi, count) (vsi)->obytes = (count) -#define IXL_SET_IMCASTS(vsi, count) (vsi)->imcasts = (count) -#define IXL_SET_OMCASTS(vsi, count) (vsi)->omcasts = (count) -#define IXL_SET_IQDROPS(vsi, count) (vsi)->iqdrops = (count) -#define IXL_SET_OQDROPS(vsi, count) (vsi)->oqdrops = (count) -#define IXL_SET_NOPROTO(vsi, count) (vsi)->noproto = (count) +#ifdef IFLIB +#include "iflib_ixl.h" #else -#define IXL_SET_IPACKETS(vsi, count) (vsi)->ifp->if_ipackets = (count) -#define IXL_SET_IERRORS(vsi, count) (vsi)->ifp->if_ierrors = (count) -#define IXL_SET_OPACKETS(vsi, count) (vsi)->ifp->if_opackets = (count) -#define IXL_SET_OERRORS(vsi, count) (vsi)->ifp->if_oerrors = (count) -#define IXL_SET_COLLISIONS(vsi, count) (vsi)->ifp->if_collisions = (count) -#define IXL_SET_IBYTES(vsi, count) (vsi)->ifp->if_ibytes = (count) -#define IXL_SET_OBYTES(vsi, count) (vsi)->ifp->if_obytes = (count) -#define IXL_SET_IMCASTS(vsi, count) (vsi)->ifp->if_imcasts = (count) -#define IXL_SET_OMCASTS(vsi, count) (vsi)->ifp->if_omcasts = (count) -#define IXL_SET_IQDROPS(vsi, count) (vsi)->ifp->if_iqdrops = (count) -#define IXL_SET_OQDROPS(vsi, odrops) (vsi)->ifp->if_snd.ifq_drops = (odrops) -#define IXL_SET_NOPROTO(vsi, count) (vsi)->noproto = (count) +#include "legacy_ixl.h" #endif -/* - ***************************************************************************** - * vendor_info_array - * - * This array contains the list of Subvendor/Subdevice IDs on which the driver - * should load. - * - ***************************************************************************** - */ -typedef struct _ixl_vendor_info_t { - unsigned int vendor_id; - unsigned int device_id; - unsigned int subvendor_id; - unsigned int subdevice_id; - unsigned int index; -} ixl_vendor_info_t; - - -struct ixl_tx_buf { - u32 eop_index; - struct mbuf *m_head; - bus_dmamap_t map; - bus_dma_tag_t tag; -}; - -struct ixl_rx_buf { - struct mbuf *m_head; - struct mbuf *m_pack; - struct mbuf *fmp; - bus_dmamap_t hmap; - bus_dmamap_t pmap; -}; - -/* -** This struct has multiple uses, multicast -** addresses, vlans, and mac filters all use it. -*/ -struct ixl_mac_filter { - SLIST_ENTRY(ixl_mac_filter) next; - u8 macaddr[ETHER_ADDR_LEN]; - s16 vlan; - u16 flags; -}; - - -/* - * The Transmit ring control struct - */ -struct tx_ring { - struct ixl_queue *que; - struct mtx mtx; - u32 tail; - struct i40e_tx_desc *base; - struct i40e_dma_mem dma; - u16 next_avail; - u16 next_to_clean; - u16 atr_rate; - u16 atr_count; - u16 itr; - u16 latency; - struct ixl_tx_buf *buffers; - volatile u16 avail; - u32 cmd; - bus_dma_tag_t tx_tag; - bus_dma_tag_t tso_tag; - char mtx_name[16]; - struct buf_ring *br; - - /* Used for Dynamic ITR calculation */ - u32 packets; - u32 bytes; - - /* Soft Stats */ - u64 tx_bytes; - u64 no_desc; - u64 total_packets; -}; - - -/* - * The Receive ring control struct - */ -struct rx_ring { - struct ixl_queue *que; - struct mtx mtx; - union i40e_rx_desc *base; - struct i40e_dma_mem dma; - struct lro_ctrl lro; - bool lro_enabled; - bool hdr_split; - bool discard; - u16 next_refresh; - u16 next_check; - u16 itr; - u16 latency; - char mtx_name[16]; - struct ixl_rx_buf *buffers; - u32 mbuf_sz; - u32 tail; - bus_dma_tag_t htag; - bus_dma_tag_t ptag; - - /* Used for Dynamic ITR calculation */ - u32 packets; - u32 bytes; - - /* Soft stats */ - u64 split; - u64 rx_packets; - u64 rx_bytes; - u64 discarded; - u64 not_done; -}; - -/* -** Driver queue struct: this is the interrupt container -** for the associated tx and rx ring pair. -*/ -struct ixl_queue { - struct ixl_vsi *vsi; - u32 me; - u32 msix; /* This queue's MSIX vector */ - u32 eims; /* This queue's EIMS bit */ - struct resource *res; - void *tag; - int num_desc; /* both tx and rx */ - int busy; - struct tx_ring txr; - struct rx_ring rxr; - struct task task; - struct task tx_task; - struct taskqueue *tq; - - /* Queue stats */ - u64 irqs; - u64 tso; - u64 mbuf_defrag_failed; - u64 mbuf_hdr_failed; - u64 mbuf_pkt_failed; - u64 tx_map_avail; - u64 tx_dma_setup; - u64 dropped_pkts; -}; - -/* -** Virtual Station interface: -** there would be one of these per traffic class/type -** for now just one, and its embedded in the pf -*/ -SLIST_HEAD(ixl_ftl_head, ixl_mac_filter); -struct ixl_vsi { - void *back; - struct ifnet *ifp; - struct device *dev; - struct i40e_hw *hw; - struct ifmedia media; - u64 que_mask; - int id; - u16 vsi_num; - u16 msix_base; /* station base MSIX vector */ - u16 first_queue; - u16 num_queues; - u16 rx_itr_setting; - u16 tx_itr_setting; - struct ixl_queue *queues; /* head of queues */ - bool link_active; - u16 seid; - u16 uplink_seid; - u16 downlink_seid; - u16 max_frame_size; - - /* MAC/VLAN Filter list */ - struct ixl_ftl_head ftl; - u16 num_macs; - - struct i40e_aqc_vsi_properties_data info; - - eventhandler_tag vlan_attach; - eventhandler_tag vlan_detach; - u16 num_vlans; - - /* Per-VSI stats from hardware */ - struct i40e_eth_stats eth_stats; - struct i40e_eth_stats eth_stats_offsets; - bool stat_offsets_loaded; - /* VSI stat counters */ - u64 ipackets; - u64 ierrors; - u64 opackets; - u64 oerrors; - u64 ibytes; - u64 obytes; - u64 imcasts; - u64 omcasts; - u64 iqdrops; - u64 oqdrops; - u64 noproto; - - /* Driver statistics */ - u64 hw_filters_del; - u64 hw_filters_add; - - /* Misc. */ - u64 active_queues; - u64 flags; - struct sysctl_oid *vsi_node; -}; - -/* -** Find the number of unrefreshed RX descriptors -*/ -static inline u16 -ixl_rx_unrefreshed(struct ixl_queue *que) -{ - struct rx_ring *rxr = &que->rxr; - - if (rxr->next_check > rxr->next_refresh) - return (rxr->next_check - rxr->next_refresh - 1); - else - return ((que->num_desc + rxr->next_check) - - rxr->next_refresh - 1); -} - -/* -** Find the next available unused filter -*/ -static inline struct ixl_mac_filter * -ixl_get_filter(struct ixl_vsi *vsi) -{ - struct ixl_mac_filter *f; - - /* create a new empty filter */ - f = malloc(sizeof(struct ixl_mac_filter), - M_DEVBUF, M_NOWAIT | M_ZERO); - if (f) - SLIST_INSERT_HEAD(&vsi->ftl, f, next); - - return (f); -} - -/* -** Compare two ethernet addresses -*/ -static inline bool -cmp_etheraddr(const u8 *ea1, const u8 *ea2) -{ - bool cmp = FALSE; - - if ((ea1[0] == ea2[0]) && (ea1[1] == ea2[1]) && - (ea1[2] == ea2[2]) && (ea1[3] == ea2[3]) && - (ea1[4] == ea2[4]) && (ea1[5] == ea2[5])) - cmp = TRUE; - - return (cmp); -} - -/* - * Info for stats sysctls - */ -struct ixl_sysctl_info { - u64 *stat; - char *name; - char *description; -}; - -extern int ixl_atr_rate; - -/* -** ixl_fw_version_str - format the FW and NVM version strings -*/ -static inline char * -ixl_fw_version_str(struct i40e_hw *hw) -{ - static char buf[32]; - - snprintf(buf, sizeof(buf), - "f%d.%d a%d.%d n%02x.%02x e%08x", - hw->aq.fw_maj_ver, hw->aq.fw_min_ver, - hw->aq.api_maj_ver, hw->aq.api_min_ver, - (hw->nvm.version & IXL_NVM_VERSION_HI_MASK) >> - IXL_NVM_VERSION_HI_SHIFT, - (hw->nvm.version & IXL_NVM_VERSION_LO_MASK) >> - IXL_NVM_VERSION_LO_SHIFT, - hw->nvm.eetrack); - return buf; -} - -/********************************************************************* - * TXRX Function prototypes - *********************************************************************/ -int ixl_allocate_tx_data(struct ixl_queue *); -int ixl_allocate_rx_data(struct ixl_queue *); -void ixl_init_tx_ring(struct ixl_queue *); -int ixl_init_rx_ring(struct ixl_queue *); -bool ixl_rxeof(struct ixl_queue *, int); -bool ixl_txeof(struct ixl_queue *); -int ixl_mq_start(struct ifnet *, struct mbuf *); -int ixl_mq_start_locked(struct ifnet *, struct tx_ring *); -void ixl_deferred_mq_start(void *, int); -void ixl_qflush(struct ifnet *); -void ixl_free_vsi(struct ixl_vsi *); -void ixl_free_que_tx(struct ixl_queue *); -void ixl_free_que_rx(struct ixl_queue *); -#ifdef IXL_FDIR -void ixl_atr(struct ixl_queue *, struct tcphdr *, int); #endif -#if __FreeBSD_version >= 1100000 -uint64_t ixl_get_counter(if_t ifp, ift_counter cnt); -#endif - -#endif /* _IXL_H_ */ Index: sys/dev/ixl/ixl_pf.h =================================================================== --- sys/dev/ixl/ixl_pf.h +++ sys/dev/ixl/ixl_pf.h @@ -1,139 +1,6 @@ -/****************************************************************************** - - Copyright (c) 2013-2015, Intel Corporation - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - 3. Neither the name of the Intel Corporation nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - -******************************************************************************/ -/*$FreeBSD$*/ - - -#ifndef _IXL_PF_H_ -#define _IXL_PF_H_ - -#define VF_FLAG_ENABLED 0x01 -#define VF_FLAG_SET_MAC_CAP 0x02 -#define VF_FLAG_VLAN_CAP 0x04 -#define VF_FLAG_PROMISC_CAP 0x08 -#define VF_FLAG_MAC_ANTI_SPOOF 0x10 - -struct ixl_vf { - struct ixl_vsi vsi; - uint32_t vf_flags; - - uint8_t mac[ETHER_ADDR_LEN]; - uint16_t vf_num; - - struct sysctl_ctx_list ctx; -}; - -/* Physical controller structure */ -struct ixl_pf { - struct i40e_hw hw; - struct i40e_osdep osdep; - struct device *dev; - - struct resource *pci_mem; - struct resource *msix_mem; - - /* - * Interrupt resources: this set is - * either used for legacy, or for Link - * when doing MSIX - */ - void *tag; - struct resource *res; - - struct callout timer; - int msix; - int if_flags; - - struct mtx pf_mtx; - - u32 qbase; - u32 admvec; - struct task adminq; - struct taskqueue *tq; - - bool link_up; - u32 link_speed; - int advertised_speed; - int fc; /* local flow ctrl setting */ - - /* - ** Network interfaces - ** These are the traffic class holders, and - ** will have a stack interface and queues - ** associated with them. - ** NOTE: The PF has only a single interface, - ** so it is embedded in the PF struct. - */ - struct ixl_vsi vsi; - - /* Misc stats maintained by the driver */ - u64 watchdog_events; - u64 admin_irq; - - /* Statistics from hw */ - struct i40e_hw_port_stats stats; - struct i40e_hw_port_stats stats_offsets; - bool stat_offsets_loaded; - - struct ixl_vf *vfs; - int num_vfs; - uint16_t veb_seid; - struct task vflr_task; - int vc_debug_lvl; -}; - -#define IXL_SET_ADVERTISE_HELP \ -"Control link advertise speed:\n" \ -"\tFlags:\n" \ -"\t\t0x1 - advertise 100 Mb\n" \ -"\t\t0x2 - advertise 1G\n" \ -"\t\t0x4 - advertise 10G\n" \ -"\t\t0x8 - advertise 20G\n\n" \ -"\tDoes not work on 40G devices." - -#define I40E_VC_DEBUG(pf, level, ...) \ - do { \ - if ((pf)->vc_debug_lvl >= (level)) \ - device_printf((pf)->dev, __VA_ARGS__); \ - } while (0) - -#define i40e_send_vf_nack(pf, vf, op, st) \ - ixl_send_vf_nack_msg((pf), (vf), (op), (st), __FILE__, __LINE__) - -#define IXL_PF_LOCK_INIT(_sc, _name) \ - mtx_init(&(_sc)->pf_mtx, _name, "IXL PF Lock", MTX_DEF) -#define IXL_PF_LOCK(_sc) mtx_lock(&(_sc)->pf_mtx) -#define IXL_PF_UNLOCK(_sc) mtx_unlock(&(_sc)->pf_mtx) -#define IXL_PF_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->pf_mtx) -#define IXL_PF_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->pf_mtx, MA_OWNED) - -#endif /* _IXL_PF_H_ */ +#include "opt_iflib.h" +#ifdef IFLIB +#include "iflib_ixl_pf.h" +#else +#include "legacy_ixl_pf.h" +#endif Index: sys/dev/ixl/ixl_txrx.c =================================================================== --- sys/dev/ixl/ixl_txrx.c +++ sys/dev/ixl/ixl_txrx.c @@ -1,1827 +1,6 @@ -/****************************************************************************** - - Copyright (c) 2013-2015, Intel Corporation - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - 3. Neither the name of the Intel Corporation nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - -******************************************************************************/ -/*$FreeBSD$*/ - -/* -** IXL driver TX/RX Routines: -** This was seperated to allow usage by -** both the BASE and the VF drivers. -*/ - -#ifndef IXL_STANDALONE_BUILD -#include "opt_inet.h" -#include "opt_inet6.h" -#include "opt_rss.h" -#endif - -#include "ixl.h" - -#ifdef RSS -#include -#endif - -/* Local Prototypes */ -static void ixl_rx_checksum(struct mbuf *, u32, u32, u8); -static void ixl_refresh_mbufs(struct ixl_queue *, int); -static int ixl_xmit(struct ixl_queue *, struct mbuf **); -static int ixl_tx_setup_offload(struct ixl_queue *, - struct mbuf *, u32 *, u32 *); -static bool ixl_tso_setup(struct ixl_queue *, struct mbuf *); - -static __inline void ixl_rx_discard(struct rx_ring *, int); -static __inline void ixl_rx_input(struct rx_ring *, struct ifnet *, - struct mbuf *, u8); - -#ifdef DEV_NETMAP -#include -#endif /* DEV_NETMAP */ - -/* -** Multiqueue Transmit driver -*/ -int -ixl_mq_start(struct ifnet *ifp, struct mbuf *m) -{ - struct ixl_vsi *vsi = ifp->if_softc; - struct ixl_queue *que; - struct tx_ring *txr; - int err, i; -#ifdef RSS - u32 bucket_id; -#endif - - /* - ** Which queue to use: - ** - ** When doing RSS, map it to the same outbound - ** queue as the incoming flow would be mapped to. - ** If everything is setup correctly, it should be - ** the same bucket that the current CPU we're on is. - */ - if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) { -#ifdef RSS - if (rss_hash2bucket(m->m_pkthdr.flowid, - M_HASHTYPE_GET(m), &bucket_id) == 0) { - i = bucket_id % vsi->num_queues; - } else -#endif - i = m->m_pkthdr.flowid % vsi->num_queues; - } else - i = curcpu % vsi->num_queues; - /* - ** This may not be perfect, but until something - ** better comes along it will keep from scheduling - ** on stalled queues. - */ - if (((1 << i) & vsi->active_queues) == 0) - i = ffsl(vsi->active_queues); - - que = &vsi->queues[i]; - txr = &que->txr; - - err = drbr_enqueue(ifp, txr->br, m); - if (err) - return (err); - if (IXL_TX_TRYLOCK(txr)) { - ixl_mq_start_locked(ifp, txr); - IXL_TX_UNLOCK(txr); - } else - taskqueue_enqueue(que->tq, &que->tx_task); - - return (0); -} - -int -ixl_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr) -{ - struct ixl_queue *que = txr->que; - struct ixl_vsi *vsi = que->vsi; - struct mbuf *next; - int err = 0; - - - if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) || - vsi->link_active == 0) - return (ENETDOWN); - - /* Process the transmit queue */ - while ((next = drbr_peek(ifp, txr->br)) != NULL) { - if ((err = ixl_xmit(que, &next)) != 0) { - if (next == NULL) - drbr_advance(ifp, txr->br); - else - drbr_putback(ifp, txr->br, next); - break; - } - drbr_advance(ifp, txr->br); - /* Send a copy of the frame to the BPF listener */ - ETHER_BPF_MTAP(ifp, next); - if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) - break; - } - - if (txr->avail < IXL_TX_CLEANUP_THRESHOLD) - ixl_txeof(que); - - return (err); -} - -/* - * Called from a taskqueue to drain queued transmit packets. - */ -void -ixl_deferred_mq_start(void *arg, int pending) -{ - struct ixl_queue *que = arg; - struct tx_ring *txr = &que->txr; - struct ixl_vsi *vsi = que->vsi; - struct ifnet *ifp = vsi->ifp; - - IXL_TX_LOCK(txr); - if (!drbr_empty(ifp, txr->br)) - ixl_mq_start_locked(ifp, txr); - IXL_TX_UNLOCK(txr); -} - -/* -** Flush all queue ring buffers -*/ -void -ixl_qflush(struct ifnet *ifp) -{ - struct ixl_vsi *vsi = ifp->if_softc; - - for (int i = 0; i < vsi->num_queues; i++) { - struct ixl_queue *que = &vsi->queues[i]; - struct tx_ring *txr = &que->txr; - struct mbuf *m; - IXL_TX_LOCK(txr); - while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) - m_freem(m); - IXL_TX_UNLOCK(txr); - } - if_qflush(ifp); -} - -/* -** Find mbuf chains passed to the driver -** that are 'sparse', using more than 8 -** mbufs to deliver an mss-size chunk of data -*/ -static inline bool -ixl_tso_detect_sparse(struct mbuf *mp) -{ - struct mbuf *m; - int num = 0, mss; - bool ret = FALSE; - - mss = mp->m_pkthdr.tso_segsz; - for (m = mp->m_next; m != NULL; m = m->m_next) { - num++; - mss -= m->m_len; - if (mss < 1) - break; - if (m->m_next == NULL) - break; - } - if (num > IXL_SPARSE_CHAIN) - ret = TRUE; - - return (ret); -} - - -/********************************************************************* - * - * This routine maps the mbufs to tx descriptors, allowing the - * TX engine to transmit the packets. - * - return 0 on success, positive on failure - * - **********************************************************************/ -#define IXL_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS) - -static int -ixl_xmit(struct ixl_queue *que, struct mbuf **m_headp) -{ - struct ixl_vsi *vsi = que->vsi; - struct i40e_hw *hw = vsi->hw; - struct tx_ring *txr = &que->txr; - struct ixl_tx_buf *buf; - struct i40e_tx_desc *txd = NULL; - struct mbuf *m_head, *m; - int i, j, error, nsegs, maxsegs; - int first, last = 0; - u16 vtag = 0; - u32 cmd, off; - bus_dmamap_t map; - bus_dma_tag_t tag; - bus_dma_segment_t segs[IXL_MAX_TSO_SEGS]; - - - cmd = off = 0; - m_head = *m_headp; - - /* - * Important to capture the first descriptor - * used because it will contain the index of - * the one we tell the hardware to report back - */ - first = txr->next_avail; - buf = &txr->buffers[first]; - map = buf->map; - tag = txr->tx_tag; - maxsegs = IXL_MAX_TX_SEGS; - - if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { - /* Use larger mapping for TSO */ - tag = txr->tso_tag; - maxsegs = IXL_MAX_TSO_SEGS; - if (ixl_tso_detect_sparse(m_head)) { - m = m_defrag(m_head, M_NOWAIT); - if (m == NULL) { - m_freem(*m_headp); - *m_headp = NULL; - return (ENOBUFS); - } - *m_headp = m; - } - } - - /* - * Map the packet for DMA. - */ - error = bus_dmamap_load_mbuf_sg(tag, map, - *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); - - if (error == EFBIG) { - struct mbuf *m; - - m = m_defrag(*m_headp, M_NOWAIT); - if (m == NULL) { - que->mbuf_defrag_failed++; - m_freem(*m_headp); - *m_headp = NULL; - return (ENOBUFS); - } - *m_headp = m; - - /* Try it again */ - error = bus_dmamap_load_mbuf_sg(tag, map, - *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); - - if (error == ENOMEM) { - que->tx_dma_setup++; - return (error); - } else if (error != 0) { - que->tx_dma_setup++; - m_freem(*m_headp); - *m_headp = NULL; - return (error); - } - } else if (error == ENOMEM) { - que->tx_dma_setup++; - return (error); - } else if (error != 0) { - que->tx_dma_setup++; - m_freem(*m_headp); - *m_headp = NULL; - return (error); - } - - /* Make certain there are enough descriptors */ - if (nsegs > txr->avail - 2) { - txr->no_desc++; - error = ENOBUFS; - goto xmit_fail; - } - m_head = *m_headp; - - /* Set up the TSO/CSUM offload */ - if (m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD) { - error = ixl_tx_setup_offload(que, m_head, &cmd, &off); - if (error) - goto xmit_fail; - } - - cmd |= I40E_TX_DESC_CMD_ICRC; - /* Grab the VLAN tag */ - if (m_head->m_flags & M_VLANTAG) { - cmd |= I40E_TX_DESC_CMD_IL2TAG1; - vtag = htole16(m_head->m_pkthdr.ether_vtag); - } - - i = txr->next_avail; - for (j = 0; j < nsegs; j++) { - bus_size_t seglen; - - buf = &txr->buffers[i]; - buf->tag = tag; /* Keep track of the type tag */ - txd = &txr->base[i]; - seglen = segs[j].ds_len; - - txd->buffer_addr = htole64(segs[j].ds_addr); - txd->cmd_type_offset_bsz = - htole64(I40E_TX_DESC_DTYPE_DATA - | ((u64)cmd << I40E_TXD_QW1_CMD_SHIFT) - | ((u64)off << I40E_TXD_QW1_OFFSET_SHIFT) - | ((u64)seglen << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) - | ((u64)vtag << I40E_TXD_QW1_L2TAG1_SHIFT)); - - last = i; /* descriptor that will get completion IRQ */ - - if (++i == que->num_desc) - i = 0; - - buf->m_head = NULL; - buf->eop_index = -1; - } - /* Set the last descriptor for report */ - txd->cmd_type_offset_bsz |= - htole64(((u64)IXL_TXD_CMD << I40E_TXD_QW1_CMD_SHIFT)); - txr->avail -= nsegs; - txr->next_avail = i; - - buf->m_head = m_head; - /* Swap the dma map between the first and last descriptor */ - txr->buffers[first].map = buf->map; - buf->map = map; - bus_dmamap_sync(tag, map, BUS_DMASYNC_PREWRITE); - - /* Set the index of the descriptor that will be marked done */ - buf = &txr->buffers[first]; - buf->eop_index = last; - - bus_dmamap_sync(txr->dma.tag, txr->dma.map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - /* - * Advance the Transmit Descriptor Tail (Tdt), this tells the - * hardware that this frame is available to transmit. - */ - ++txr->total_packets; - wr32(hw, txr->tail, i); - - /* Mark outstanding work */ - if (que->busy == 0) - que->busy = 1; - return (0); - -xmit_fail: - bus_dmamap_unload(tag, buf->map); - return (error); -} - - -/********************************************************************* - * - * Allocate memory for tx_buffer structures. The tx_buffer stores all - * the information needed to transmit a packet on the wire. This is - * called only once at attach, setup is done every reset. - * - **********************************************************************/ -int -ixl_allocate_tx_data(struct ixl_queue *que) -{ - struct tx_ring *txr = &que->txr; - struct ixl_vsi *vsi = que->vsi; - device_t dev = vsi->dev; - struct ixl_tx_buf *buf; - int error = 0; - - /* - * Setup DMA descriptor areas. - */ - if ((error = bus_dma_tag_create(NULL, /* parent */ - 1, 0, /* alignment, bounds */ - BUS_SPACE_MAXADDR, /* lowaddr */ - BUS_SPACE_MAXADDR, /* highaddr */ - NULL, NULL, /* filter, filterarg */ - IXL_TSO_SIZE, /* maxsize */ - IXL_MAX_TX_SEGS, /* nsegments */ - PAGE_SIZE, /* maxsegsize */ - 0, /* flags */ - NULL, /* lockfunc */ - NULL, /* lockfuncarg */ - &txr->tx_tag))) { - device_printf(dev,"Unable to allocate TX DMA tag\n"); - goto fail; - } - - /* Make a special tag for TSO */ - if ((error = bus_dma_tag_create(NULL, /* parent */ - 1, 0, /* alignment, bounds */ - BUS_SPACE_MAXADDR, /* lowaddr */ - BUS_SPACE_MAXADDR, /* highaddr */ - NULL, NULL, /* filter, filterarg */ - IXL_TSO_SIZE, /* maxsize */ - IXL_MAX_TSO_SEGS, /* nsegments */ - PAGE_SIZE, /* maxsegsize */ - 0, /* flags */ - NULL, /* lockfunc */ - NULL, /* lockfuncarg */ - &txr->tso_tag))) { - device_printf(dev,"Unable to allocate TX TSO DMA tag\n"); - goto fail; - } - - if (!(txr->buffers = - (struct ixl_tx_buf *) malloc(sizeof(struct ixl_tx_buf) * - que->num_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { - device_printf(dev, "Unable to allocate tx_buffer memory\n"); - error = ENOMEM; - goto fail; - } - - /* Create the descriptor buffer default dma maps */ - buf = txr->buffers; - for (int i = 0; i < que->num_desc; i++, buf++) { - buf->tag = txr->tx_tag; - error = bus_dmamap_create(buf->tag, 0, &buf->map); - if (error != 0) { - device_printf(dev, "Unable to create TX DMA map\n"); - goto fail; - } - } -fail: - return (error); -} - - -/********************************************************************* - * - * (Re)Initialize a queue transmit ring. - * - called by init, it clears the descriptor ring, - * and frees any stale mbufs - * - **********************************************************************/ -void -ixl_init_tx_ring(struct ixl_queue *que) -{ -#ifdef DEV_NETMAP - struct netmap_adapter *na = NA(que->vsi->ifp); - struct netmap_slot *slot; -#endif /* DEV_NETMAP */ - struct tx_ring *txr = &que->txr; - struct ixl_tx_buf *buf; - - /* Clear the old ring contents */ - IXL_TX_LOCK(txr); - -#ifdef DEV_NETMAP - /* - * (under lock): if in netmap mode, do some consistency - * checks and set slot to entry 0 of the netmap ring. - */ - slot = netmap_reset(na, NR_TX, que->me, 0); -#endif /* DEV_NETMAP */ - - bzero((void *)txr->base, - (sizeof(struct i40e_tx_desc)) * que->num_desc); - - /* Reset indices */ - txr->next_avail = 0; - txr->next_to_clean = 0; - -#ifdef IXL_FDIR - /* Initialize flow director */ - txr->atr_rate = ixl_atr_rate; - txr->atr_count = 0; -#endif - - /* Free any existing tx mbufs. */ - buf = txr->buffers; - for (int i = 0; i < que->num_desc; i++, buf++) { - if (buf->m_head != NULL) { - bus_dmamap_sync(buf->tag, buf->map, - BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(buf->tag, buf->map); - m_freem(buf->m_head); - buf->m_head = NULL; - } -#ifdef DEV_NETMAP - /* - * In netmap mode, set the map for the packet buffer. - * NOTE: Some drivers (not this one) also need to set - * the physical buffer address in the NIC ring. - * netmap_idx_n2k() maps a nic index, i, into the corresponding - * netmap slot index, si - */ - if (slot) { - int si = netmap_idx_n2k(&na->tx_rings[que->me], i); - netmap_load_map(na, buf->tag, buf->map, NMB(na, slot + si)); - } -#endif /* DEV_NETMAP */ - /* Clear the EOP index */ - buf->eop_index = -1; - } - - /* Set number of descriptors available */ - txr->avail = que->num_desc; - - bus_dmamap_sync(txr->dma.tag, txr->dma.map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - IXL_TX_UNLOCK(txr); -} - - -/********************************************************************* - * - * Free transmit ring related data structures. - * - **********************************************************************/ -void -ixl_free_que_tx(struct ixl_queue *que) -{ - struct tx_ring *txr = &que->txr; - struct ixl_tx_buf *buf; - - INIT_DBG_IF(que->vsi->ifp, "queue %d: begin", que->me); - - for (int i = 0; i < que->num_desc; i++) { - buf = &txr->buffers[i]; - if (buf->m_head != NULL) { - bus_dmamap_sync(buf->tag, buf->map, - BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(buf->tag, - buf->map); - m_freem(buf->m_head); - buf->m_head = NULL; - if (buf->map != NULL) { - bus_dmamap_destroy(buf->tag, - buf->map); - buf->map = NULL; - } - } else if (buf->map != NULL) { - bus_dmamap_unload(buf->tag, - buf->map); - bus_dmamap_destroy(buf->tag, - buf->map); - buf->map = NULL; - } - } - if (txr->br != NULL) - buf_ring_free(txr->br, M_DEVBUF); - if (txr->buffers != NULL) { - free(txr->buffers, M_DEVBUF); - txr->buffers = NULL; - } - if (txr->tx_tag != NULL) { - bus_dma_tag_destroy(txr->tx_tag); - txr->tx_tag = NULL; - } - if (txr->tso_tag != NULL) { - bus_dma_tag_destroy(txr->tso_tag); - txr->tso_tag = NULL; - } - - INIT_DBG_IF(que->vsi->ifp, "queue %d: end", que->me); - return; -} - -/********************************************************************* - * - * Setup descriptor for hw offloads - * - **********************************************************************/ - -static int -ixl_tx_setup_offload(struct ixl_queue *que, - struct mbuf *mp, u32 *cmd, u32 *off) -{ - struct ether_vlan_header *eh; -#ifdef INET - struct ip *ip = NULL; -#endif - struct tcphdr *th = NULL; -#ifdef INET6 - struct ip6_hdr *ip6; -#endif - int elen, ip_hlen = 0, tcp_hlen; - u16 etype; - u8 ipproto = 0; - bool tso = FALSE; - - - /* Set up the TSO context descriptor if required */ - if (mp->m_pkthdr.csum_flags & CSUM_TSO) { - tso = ixl_tso_setup(que, mp); - if (tso) - ++que->tso; - else - return (ENXIO); - } - - /* - * Determine where frame payload starts. - * Jump over vlan headers if already present, - * helpful for QinQ too. - */ - eh = mtod(mp, struct ether_vlan_header *); - if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { - etype = ntohs(eh->evl_proto); - elen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; - } else { - etype = ntohs(eh->evl_encap_proto); - elen = ETHER_HDR_LEN; - } - - switch (etype) { -#ifdef INET - case ETHERTYPE_IP: - ip = (struct ip *)(mp->m_data + elen); - ip_hlen = ip->ip_hl << 2; - ipproto = ip->ip_p; - th = (struct tcphdr *)((caddr_t)ip + ip_hlen); - /* The IP checksum must be recalculated with TSO */ - if (tso) - *cmd |= I40E_TX_DESC_CMD_IIPT_IPV4_CSUM; - else - *cmd |= I40E_TX_DESC_CMD_IIPT_IPV4; - break; -#endif -#ifdef INET6 - case ETHERTYPE_IPV6: - ip6 = (struct ip6_hdr *)(mp->m_data + elen); - ip_hlen = sizeof(struct ip6_hdr); - ipproto = ip6->ip6_nxt; - th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen); - *cmd |= I40E_TX_DESC_CMD_IIPT_IPV6; - break; -#endif - default: - break; - } - - *off |= (elen >> 1) << I40E_TX_DESC_LENGTH_MACLEN_SHIFT; - *off |= (ip_hlen >> 2) << I40E_TX_DESC_LENGTH_IPLEN_SHIFT; - - switch (ipproto) { - case IPPROTO_TCP: - tcp_hlen = th->th_off << 2; - if (mp->m_pkthdr.csum_flags & (CSUM_TCP|CSUM_TCP_IPV6)) { - *cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP; - *off |= (tcp_hlen >> 2) << - I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; - } -#ifdef IXL_FDIR - ixl_atr(que, th, etype); -#endif - break; - case IPPROTO_UDP: - if (mp->m_pkthdr.csum_flags & (CSUM_UDP|CSUM_UDP_IPV6)) { - *cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP; - *off |= (sizeof(struct udphdr) >> 2) << - I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; - } - break; - - case IPPROTO_SCTP: - if (mp->m_pkthdr.csum_flags & (CSUM_SCTP|CSUM_SCTP_IPV6)) { - *cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP; - *off |= (sizeof(struct sctphdr) >> 2) << - I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; - } - /* Fall Thru */ - default: - break; - } - - return (0); -} - - -/********************************************************************** - * - * Setup context for hardware segmentation offload (TSO) - * - **********************************************************************/ -static bool -ixl_tso_setup(struct ixl_queue *que, struct mbuf *mp) -{ - struct tx_ring *txr = &que->txr; - struct i40e_tx_context_desc *TXD; - struct ixl_tx_buf *buf; - u32 cmd, mss, type, tsolen; - u16 etype; - int idx, elen, ip_hlen, tcp_hlen; - struct ether_vlan_header *eh; -#ifdef INET - struct ip *ip; -#endif -#ifdef INET6 - struct ip6_hdr *ip6; -#endif -#if defined(INET6) || defined(INET) - struct tcphdr *th; -#endif - u64 type_cmd_tso_mss; - - /* - * Determine where frame payload starts. - * Jump over vlan headers if already present - */ - eh = mtod(mp, struct ether_vlan_header *); - if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { - elen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; - etype = eh->evl_proto; - } else { - elen = ETHER_HDR_LEN; - etype = eh->evl_encap_proto; - } - - switch (ntohs(etype)) { -#ifdef INET6 - case ETHERTYPE_IPV6: - ip6 = (struct ip6_hdr *)(mp->m_data + elen); - if (ip6->ip6_nxt != IPPROTO_TCP) - return (ENXIO); - ip_hlen = sizeof(struct ip6_hdr); - th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen); - th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0); - tcp_hlen = th->th_off << 2; - break; -#endif -#ifdef INET - case ETHERTYPE_IP: - ip = (struct ip *)(mp->m_data + elen); - if (ip->ip_p != IPPROTO_TCP) - return (ENXIO); - ip->ip_sum = 0; - ip_hlen = ip->ip_hl << 2; - th = (struct tcphdr *)((caddr_t)ip + ip_hlen); - th->th_sum = in_pseudo(ip->ip_src.s_addr, - ip->ip_dst.s_addr, htons(IPPROTO_TCP)); - tcp_hlen = th->th_off << 2; - break; -#endif - default: - printf("%s: CSUM_TSO but no supported IP version (0x%04x)", - __func__, ntohs(etype)); - return FALSE; - } - - /* Ensure we have at least the IP+TCP header in the first mbuf. */ - if (mp->m_len < elen + ip_hlen + sizeof(struct tcphdr)) - return FALSE; - - idx = txr->next_avail; - buf = &txr->buffers[idx]; - TXD = (struct i40e_tx_context_desc *) &txr->base[idx]; - tsolen = mp->m_pkthdr.len - (elen + ip_hlen + tcp_hlen); - - type = I40E_TX_DESC_DTYPE_CONTEXT; - cmd = I40E_TX_CTX_DESC_TSO; - mss = mp->m_pkthdr.tso_segsz; - - type_cmd_tso_mss = ((u64)type << I40E_TXD_CTX_QW1_DTYPE_SHIFT) | - ((u64)cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) | - ((u64)tsolen << I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) | - ((u64)mss << I40E_TXD_CTX_QW1_MSS_SHIFT); - TXD->type_cmd_tso_mss = htole64(type_cmd_tso_mss); - - TXD->tunneling_params = htole32(0); - buf->m_head = NULL; - buf->eop_index = -1; - - if (++idx == que->num_desc) - idx = 0; - - txr->avail--; - txr->next_avail = idx; - - return TRUE; -} - -/* -** ixl_get_tx_head - Retrieve the value from the -** location the HW records its HEAD index -*/ -static inline u32 -ixl_get_tx_head(struct ixl_queue *que) -{ - struct tx_ring *txr = &que->txr; - void *head = &txr->base[que->num_desc]; - return LE32_TO_CPU(*(volatile __le32 *)head); -} - -/********************************************************************** - * - * Examine each tx_buffer in the used queue. If the hardware is done - * processing the packet then free associated resources. The - * tx_buffer is put back on the free queue. - * - **********************************************************************/ -bool -ixl_txeof(struct ixl_queue *que) -{ - struct tx_ring *txr = &que->txr; - u32 first, last, head, done, processed; - struct ixl_tx_buf *buf; - struct i40e_tx_desc *tx_desc, *eop_desc; - - - mtx_assert(&txr->mtx, MA_OWNED); - -#ifdef DEV_NETMAP - // XXX todo: implement moderation - if (netmap_tx_irq(que->vsi->ifp, que->me)) - return FALSE; -#endif /* DEF_NETMAP */ - - /* These are not the descriptors you seek, move along :) */ - if (txr->avail == que->num_desc) { - que->busy = 0; - return FALSE; - } - - processed = 0; - first = txr->next_to_clean; - buf = &txr->buffers[first]; - tx_desc = (struct i40e_tx_desc *)&txr->base[first]; - last = buf->eop_index; - if (last == -1) - return FALSE; - eop_desc = (struct i40e_tx_desc *)&txr->base[last]; - - /* Get the Head WB value */ - head = ixl_get_tx_head(que); - - /* - ** Get the index of the first descriptor - ** BEYOND the EOP and call that 'done'. - ** I do this so the comparison in the - ** inner while loop below can be simple - */ - if (++last == que->num_desc) last = 0; - done = last; - - bus_dmamap_sync(txr->dma.tag, txr->dma.map, - BUS_DMASYNC_POSTREAD); - /* - ** The HEAD index of the ring is written in a - ** defined location, this rather than a done bit - ** is what is used to keep track of what must be - ** 'cleaned'. - */ - while (first != head) { - /* We clean the range of the packet */ - while (first != done) { - ++txr->avail; - ++processed; - - if (buf->m_head) { - txr->bytes += /* for ITR adjustment */ - buf->m_head->m_pkthdr.len; - txr->tx_bytes += /* for TX stats */ - buf->m_head->m_pkthdr.len; - bus_dmamap_sync(buf->tag, - buf->map, - BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(buf->tag, - buf->map); - m_freem(buf->m_head); - buf->m_head = NULL; - buf->map = NULL; - } - buf->eop_index = -1; - - if (++first == que->num_desc) - first = 0; - - buf = &txr->buffers[first]; - tx_desc = &txr->base[first]; - } - ++txr->packets; - /* See if there is more work now */ - last = buf->eop_index; - if (last != -1) { - eop_desc = &txr->base[last]; - /* Get next done point */ - if (++last == que->num_desc) last = 0; - done = last; - } else - break; - } - bus_dmamap_sync(txr->dma.tag, txr->dma.map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - - txr->next_to_clean = first; - - - /* - ** Hang detection, we know there's - ** work outstanding or the first return - ** would have been taken, so indicate an - ** unsuccessful pass, in local_timer if - ** the value is too great the queue will - ** be considered hung. If anything has been - ** cleaned then reset the state. - */ - if ((processed == 0) && (que->busy != IXL_QUEUE_HUNG)) - ++que->busy; - - if (processed) - que->busy = 1; /* Note this turns off HUNG */ - - /* - * If there are no pending descriptors, clear the timeout. - */ - if (txr->avail == que->num_desc) { - que->busy = 0; - return FALSE; - } - - return TRUE; -} - -/********************************************************************* - * - * Refresh mbuf buffers for RX descriptor rings - * - now keeps its own state so discards due to resource - * exhaustion are unnecessary, if an mbuf cannot be obtained - * it just returns, keeping its placeholder, thus it can simply - * be recalled to try again. - * - **********************************************************************/ -static void -ixl_refresh_mbufs(struct ixl_queue *que, int limit) -{ - struct ixl_vsi *vsi = que->vsi; - struct rx_ring *rxr = &que->rxr; - bus_dma_segment_t hseg[1]; - bus_dma_segment_t pseg[1]; - struct ixl_rx_buf *buf; - struct mbuf *mh, *mp; - int i, j, nsegs, error; - bool refreshed = FALSE; - - i = j = rxr->next_refresh; - /* Control the loop with one beyond */ - if (++j == que->num_desc) - j = 0; - - while (j != limit) { - buf = &rxr->buffers[i]; - if (rxr->hdr_split == FALSE) - goto no_split; - - if (buf->m_head == NULL) { - mh = m_gethdr(M_NOWAIT, MT_DATA); - if (mh == NULL) - goto update; - } else - mh = buf->m_head; - - mh->m_pkthdr.len = mh->m_len = MHLEN; - mh->m_len = MHLEN; - mh->m_flags |= M_PKTHDR; - /* Get the memory mapping */ - error = bus_dmamap_load_mbuf_sg(rxr->htag, - buf->hmap, mh, hseg, &nsegs, BUS_DMA_NOWAIT); - if (error != 0) { - printf("Refresh mbufs: hdr dmamap load" - " failure - %d\n", error); - m_free(mh); - buf->m_head = NULL; - goto update; - } - buf->m_head = mh; - bus_dmamap_sync(rxr->htag, buf->hmap, - BUS_DMASYNC_PREREAD); - rxr->base[i].read.hdr_addr = - htole64(hseg[0].ds_addr); - -no_split: - if (buf->m_pack == NULL) { - mp = m_getjcl(M_NOWAIT, MT_DATA, - M_PKTHDR, rxr->mbuf_sz); - if (mp == NULL) - goto update; - } else - mp = buf->m_pack; - - mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz; - /* Get the memory mapping */ - error = bus_dmamap_load_mbuf_sg(rxr->ptag, - buf->pmap, mp, pseg, &nsegs, BUS_DMA_NOWAIT); - if (error != 0) { - printf("Refresh mbufs: payload dmamap load" - " failure - %d\n", error); - m_free(mp); - buf->m_pack = NULL; - goto update; - } - buf->m_pack = mp; - bus_dmamap_sync(rxr->ptag, buf->pmap, - BUS_DMASYNC_PREREAD); - rxr->base[i].read.pkt_addr = - htole64(pseg[0].ds_addr); - /* Used only when doing header split */ - rxr->base[i].read.hdr_addr = 0; - - refreshed = TRUE; - /* Next is precalculated */ - i = j; - rxr->next_refresh = i; - if (++j == que->num_desc) - j = 0; - } -update: - if (refreshed) /* Update hardware tail index */ - wr32(vsi->hw, rxr->tail, rxr->next_refresh); - return; -} - - -/********************************************************************* - * - * Allocate memory for rx_buffer structures. Since we use one - * rx_buffer per descriptor, the maximum number of rx_buffer's - * that we'll need is equal to the number of receive descriptors - * that we've defined. - * - **********************************************************************/ -int -ixl_allocate_rx_data(struct ixl_queue *que) -{ - struct rx_ring *rxr = &que->rxr; - struct ixl_vsi *vsi = que->vsi; - device_t dev = vsi->dev; - struct ixl_rx_buf *buf; - int i, bsize, error; - - bsize = sizeof(struct ixl_rx_buf) * que->num_desc; - if (!(rxr->buffers = - (struct ixl_rx_buf *) malloc(bsize, - M_DEVBUF, M_NOWAIT | M_ZERO))) { - device_printf(dev, "Unable to allocate rx_buffer memory\n"); - error = ENOMEM; - return (error); - } - - if ((error = bus_dma_tag_create(NULL, /* parent */ - 1, 0, /* alignment, bounds */ - BUS_SPACE_MAXADDR, /* lowaddr */ - BUS_SPACE_MAXADDR, /* highaddr */ - NULL, NULL, /* filter, filterarg */ - MSIZE, /* maxsize */ - 1, /* nsegments */ - MSIZE, /* maxsegsize */ - 0, /* flags */ - NULL, /* lockfunc */ - NULL, /* lockfuncarg */ - &rxr->htag))) { - device_printf(dev, "Unable to create RX DMA htag\n"); - return (error); - } - - if ((error = bus_dma_tag_create(NULL, /* parent */ - 1, 0, /* alignment, bounds */ - BUS_SPACE_MAXADDR, /* lowaddr */ - BUS_SPACE_MAXADDR, /* highaddr */ - NULL, NULL, /* filter, filterarg */ - MJUM16BYTES, /* maxsize */ - 1, /* nsegments */ - MJUM16BYTES, /* maxsegsize */ - 0, /* flags */ - NULL, /* lockfunc */ - NULL, /* lockfuncarg */ - &rxr->ptag))) { - device_printf(dev, "Unable to create RX DMA ptag\n"); - return (error); - } - - for (i = 0; i < que->num_desc; i++) { - buf = &rxr->buffers[i]; - error = bus_dmamap_create(rxr->htag, - BUS_DMA_NOWAIT, &buf->hmap); - if (error) { - device_printf(dev, "Unable to create RX head map\n"); - break; - } - error = bus_dmamap_create(rxr->ptag, - BUS_DMA_NOWAIT, &buf->pmap); - if (error) { - device_printf(dev, "Unable to create RX pkt map\n"); - break; - } - } - - return (error); -} - - -/********************************************************************* - * - * (Re)Initialize the queue receive ring and its buffers. - * - **********************************************************************/ -int -ixl_init_rx_ring(struct ixl_queue *que) -{ - struct rx_ring *rxr = &que->rxr; - struct ixl_vsi *vsi = que->vsi; -#if defined(INET6) || defined(INET) - struct ifnet *ifp = vsi->ifp; - struct lro_ctrl *lro = &rxr->lro; -#endif - struct ixl_rx_buf *buf; - bus_dma_segment_t pseg[1], hseg[1]; - int rsize, nsegs, error = 0; -#ifdef DEV_NETMAP - struct netmap_adapter *na = NA(que->vsi->ifp); - struct netmap_slot *slot; -#endif /* DEV_NETMAP */ - - IXL_RX_LOCK(rxr); -#ifdef DEV_NETMAP - /* same as in ixl_init_tx_ring() */ - slot = netmap_reset(na, NR_RX, que->me, 0); -#endif /* DEV_NETMAP */ - /* Clear the ring contents */ - rsize = roundup2(que->num_desc * - sizeof(union i40e_rx_desc), DBA_ALIGN); - bzero((void *)rxr->base, rsize); - /* Cleanup any existing buffers */ - for (int i = 0; i < que->num_desc; i++) { - buf = &rxr->buffers[i]; - if (buf->m_head != NULL) { - bus_dmamap_sync(rxr->htag, buf->hmap, - BUS_DMASYNC_POSTREAD); - bus_dmamap_unload(rxr->htag, buf->hmap); - buf->m_head->m_flags |= M_PKTHDR; - m_freem(buf->m_head); - } - if (buf->m_pack != NULL) { - bus_dmamap_sync(rxr->ptag, buf->pmap, - BUS_DMASYNC_POSTREAD); - bus_dmamap_unload(rxr->ptag, buf->pmap); - buf->m_pack->m_flags |= M_PKTHDR; - m_freem(buf->m_pack); - } - buf->m_head = NULL; - buf->m_pack = NULL; - } - - /* header split is off */ - rxr->hdr_split = FALSE; - - /* Now replenish the mbufs */ - for (int j = 0; j != que->num_desc; ++j) { - struct mbuf *mh, *mp; - - buf = &rxr->buffers[j]; -#ifdef DEV_NETMAP - /* - * In netmap mode, fill the map and set the buffer - * address in the NIC ring, considering the offset - * between the netmap and NIC rings (see comment in - * ixgbe_setup_transmit_ring() ). No need to allocate - * an mbuf, so end the block with a continue; - */ - if (slot) { - int sj = netmap_idx_n2k(&na->rx_rings[que->me], j); - uint64_t paddr; - void *addr; - - addr = PNMB(na, slot + sj, &paddr); - netmap_load_map(na, rxr->dma.tag, buf->pmap, addr); - /* Update descriptor and the cached value */ - rxr->base[j].read.pkt_addr = htole64(paddr); - rxr->base[j].read.hdr_addr = 0; - continue; - } -#endif /* DEV_NETMAP */ - /* - ** Don't allocate mbufs if not - ** doing header split, its wasteful - */ - if (rxr->hdr_split == FALSE) - goto skip_head; - - /* First the header */ - buf->m_head = m_gethdr(M_NOWAIT, MT_DATA); - if (buf->m_head == NULL) { - error = ENOBUFS; - goto fail; - } - m_adj(buf->m_head, ETHER_ALIGN); - mh = buf->m_head; - mh->m_len = mh->m_pkthdr.len = MHLEN; - mh->m_flags |= M_PKTHDR; - /* Get the memory mapping */ - error = bus_dmamap_load_mbuf_sg(rxr->htag, - buf->hmap, buf->m_head, hseg, - &nsegs, BUS_DMA_NOWAIT); - if (error != 0) /* Nothing elegant to do here */ - goto fail; - bus_dmamap_sync(rxr->htag, - buf->hmap, BUS_DMASYNC_PREREAD); - /* Update descriptor */ - rxr->base[j].read.hdr_addr = htole64(hseg[0].ds_addr); - -skip_head: - /* Now the payload cluster */ - buf->m_pack = m_getjcl(M_NOWAIT, MT_DATA, - M_PKTHDR, rxr->mbuf_sz); - if (buf->m_pack == NULL) { - error = ENOBUFS; - goto fail; - } - mp = buf->m_pack; - mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz; - /* Get the memory mapping */ - error = bus_dmamap_load_mbuf_sg(rxr->ptag, - buf->pmap, mp, pseg, - &nsegs, BUS_DMA_NOWAIT); - if (error != 0) - goto fail; - bus_dmamap_sync(rxr->ptag, - buf->pmap, BUS_DMASYNC_PREREAD); - /* Update descriptor */ - rxr->base[j].read.pkt_addr = htole64(pseg[0].ds_addr); - rxr->base[j].read.hdr_addr = 0; - } - - - /* Setup our descriptor indices */ - rxr->next_check = 0; - rxr->next_refresh = 0; - rxr->lro_enabled = FALSE; - rxr->split = 0; - rxr->bytes = 0; - rxr->discard = FALSE; - - wr32(vsi->hw, rxr->tail, que->num_desc - 1); - ixl_flush(vsi->hw); - -#if defined(INET6) || defined(INET) - /* - ** Now set up the LRO interface: - */ - if (ifp->if_capenable & IFCAP_LRO) { - int err = tcp_lro_init(lro); - if (err) { - if_printf(ifp, "queue %d: LRO Initialization failed!\n", que->me); - goto fail; - } - INIT_DBG_IF(ifp, "queue %d: RX Soft LRO Initialized", que->me); - rxr->lro_enabled = TRUE; - lro->ifp = vsi->ifp; - } -#endif - - bus_dmamap_sync(rxr->dma.tag, rxr->dma.map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - -fail: - IXL_RX_UNLOCK(rxr); - return (error); -} - - -/********************************************************************* - * - * Free station receive ring data structures - * - **********************************************************************/ -void -ixl_free_que_rx(struct ixl_queue *que) -{ - struct rx_ring *rxr = &que->rxr; - struct ixl_rx_buf *buf; - - INIT_DBG_IF(que->vsi->ifp, "queue %d: begin", que->me); - - /* Cleanup any existing buffers */ - if (rxr->buffers != NULL) { - for (int i = 0; i < que->num_desc; i++) { - buf = &rxr->buffers[i]; - if (buf->m_head != NULL) { - bus_dmamap_sync(rxr->htag, buf->hmap, - BUS_DMASYNC_POSTREAD); - bus_dmamap_unload(rxr->htag, buf->hmap); - buf->m_head->m_flags |= M_PKTHDR; - m_freem(buf->m_head); - } - if (buf->m_pack != NULL) { - bus_dmamap_sync(rxr->ptag, buf->pmap, - BUS_DMASYNC_POSTREAD); - bus_dmamap_unload(rxr->ptag, buf->pmap); - buf->m_pack->m_flags |= M_PKTHDR; - m_freem(buf->m_pack); - } - buf->m_head = NULL; - buf->m_pack = NULL; - if (buf->hmap != NULL) { - bus_dmamap_destroy(rxr->htag, buf->hmap); - buf->hmap = NULL; - } - if (buf->pmap != NULL) { - bus_dmamap_destroy(rxr->ptag, buf->pmap); - buf->pmap = NULL; - } - } - if (rxr->buffers != NULL) { - free(rxr->buffers, M_DEVBUF); - rxr->buffers = NULL; - } - } - - if (rxr->htag != NULL) { - bus_dma_tag_destroy(rxr->htag); - rxr->htag = NULL; - } - if (rxr->ptag != NULL) { - bus_dma_tag_destroy(rxr->ptag); - rxr->ptag = NULL; - } - - INIT_DBG_IF(que->vsi->ifp, "queue %d: end", que->me); - return; -} - -static __inline void -ixl_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u8 ptype) -{ - -#if defined(INET6) || defined(INET) - /* - * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet - * should be computed by hardware. Also it should not have VLAN tag in - * ethernet header. - */ - if (rxr->lro_enabled && - (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 && - (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) == - (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) { - /* - * Send to the stack if: - ** - LRO not enabled, or - ** - no LRO resources, or - ** - lro enqueue fails - */ - if (rxr->lro.lro_cnt != 0) - if (tcp_lro_rx(&rxr->lro, m, 0) == 0) - return; - } -#endif - IXL_RX_UNLOCK(rxr); - (*ifp->if_input)(ifp, m); - IXL_RX_LOCK(rxr); -} - - -static __inline void -ixl_rx_discard(struct rx_ring *rxr, int i) -{ - struct ixl_rx_buf *rbuf; - - rbuf = &rxr->buffers[i]; - - if (rbuf->fmp != NULL) {/* Partial chain ? */ - rbuf->fmp->m_flags |= M_PKTHDR; - m_freem(rbuf->fmp); - rbuf->fmp = NULL; - } - - /* - ** With advanced descriptors the writeback - ** clobbers the buffer addrs, so its easier - ** to just free the existing mbufs and take - ** the normal refresh path to get new buffers - ** and mapping. - */ - if (rbuf->m_head) { - m_free(rbuf->m_head); - rbuf->m_head = NULL; - } - - if (rbuf->m_pack) { - m_free(rbuf->m_pack); - rbuf->m_pack = NULL; - } - - return; -} - -#ifdef RSS -/* -** i40e_ptype_to_hash: parse the packet type -** to determine the appropriate hash. -*/ -static inline int -ixl_ptype_to_hash(u8 ptype) -{ - struct i40e_rx_ptype_decoded decoded; - u8 ex = 0; - - decoded = decode_rx_desc_ptype(ptype); - ex = decoded.outer_frag; - - if (!decoded.known) - return M_HASHTYPE_OPAQUE; - - if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_L2) - return M_HASHTYPE_OPAQUE; - - /* Note: anything that gets to this point is IP */ - if (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6) { - switch (decoded.inner_prot) { - case I40E_RX_PTYPE_INNER_PROT_TCP: - if (ex) - return M_HASHTYPE_RSS_TCP_IPV6_EX; - else - return M_HASHTYPE_RSS_TCP_IPV6; - case I40E_RX_PTYPE_INNER_PROT_UDP: - if (ex) - return M_HASHTYPE_RSS_UDP_IPV6_EX; - else - return M_HASHTYPE_RSS_UDP_IPV6; - default: - if (ex) - return M_HASHTYPE_RSS_IPV6_EX; - else - return M_HASHTYPE_RSS_IPV6; - } - } - if (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV4) { - switch (decoded.inner_prot) { - case I40E_RX_PTYPE_INNER_PROT_TCP: - return M_HASHTYPE_RSS_TCP_IPV4; - case I40E_RX_PTYPE_INNER_PROT_UDP: - if (ex) - return M_HASHTYPE_RSS_UDP_IPV4_EX; - else - return M_HASHTYPE_RSS_UDP_IPV4; - default: - return M_HASHTYPE_RSS_IPV4; - } - } - /* We should never get here!! */ - return M_HASHTYPE_OPAQUE; -} -#endif /* RSS */ - -/********************************************************************* - * - * This routine executes in interrupt context. It replenishes - * the mbufs in the descriptor and sends data which has been - * dma'ed into host memory to upper layer. - * - * We loop at most count times if count is > 0, or until done if - * count < 0. - * - * Return TRUE for more work, FALSE for all clean. - *********************************************************************/ -bool -ixl_rxeof(struct ixl_queue *que, int count) -{ - struct ixl_vsi *vsi = que->vsi; - struct rx_ring *rxr = &que->rxr; - struct ifnet *ifp = vsi->ifp; -#if defined(INET6) || defined(INET) - struct lro_ctrl *lro = &rxr->lro; -#endif - int i, nextp, processed = 0; - union i40e_rx_desc *cur; - struct ixl_rx_buf *rbuf, *nbuf; - - - IXL_RX_LOCK(rxr); - -#ifdef DEV_NETMAP - if (netmap_rx_irq(ifp, que->me, &count)) { - IXL_RX_UNLOCK(rxr); - return (FALSE); - } -#endif /* DEV_NETMAP */ - - for (i = rxr->next_check; count != 0;) { - struct mbuf *sendmp, *mh, *mp; - u32 rsc, status, error; - u16 hlen, plen, vtag; - u64 qword; - u8 ptype; - bool eop; - - /* Sync the ring. */ - bus_dmamap_sync(rxr->dma.tag, rxr->dma.map, - BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); - - cur = &rxr->base[i]; - qword = le64toh(cur->wb.qword1.status_error_len); - status = (qword & I40E_RXD_QW1_STATUS_MASK) - >> I40E_RXD_QW1_STATUS_SHIFT; - error = (qword & I40E_RXD_QW1_ERROR_MASK) - >> I40E_RXD_QW1_ERROR_SHIFT; - plen = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) - >> I40E_RXD_QW1_LENGTH_PBUF_SHIFT; - hlen = (qword & I40E_RXD_QW1_LENGTH_HBUF_MASK) - >> I40E_RXD_QW1_LENGTH_HBUF_SHIFT; - ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) - >> I40E_RXD_QW1_PTYPE_SHIFT; - - if ((status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) == 0) { - ++rxr->not_done; - break; - } - if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) - break; - - count--; - sendmp = NULL; - nbuf = NULL; - rsc = 0; - cur->wb.qword1.status_error_len = 0; - rbuf = &rxr->buffers[i]; - mh = rbuf->m_head; - mp = rbuf->m_pack; - eop = (status & (1 << I40E_RX_DESC_STATUS_EOF_SHIFT)); - if (status & (1 << I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)) - vtag = le16toh(cur->wb.qword0.lo_dword.l2tag1); - else - vtag = 0; - - /* - ** Make sure bad packets are discarded, - ** note that only EOP descriptor has valid - ** error results. - */ - if (eop && (error & (1 << I40E_RX_DESC_ERROR_RXE_SHIFT))) { - rxr->discarded++; - ixl_rx_discard(rxr, i); - goto next_desc; - } - - /* Prefetch the next buffer */ - if (!eop) { - nextp = i + 1; - if (nextp == que->num_desc) - nextp = 0; - nbuf = &rxr->buffers[nextp]; - prefetch(nbuf); - } - - /* - ** The header mbuf is ONLY used when header - ** split is enabled, otherwise we get normal - ** behavior, ie, both header and payload - ** are DMA'd into the payload buffer. - ** - ** Rather than using the fmp/lmp global pointers - ** we now keep the head of a packet chain in the - ** buffer struct and pass this along from one - ** descriptor to the next, until we get EOP. - */ - if (rxr->hdr_split && (rbuf->fmp == NULL)) { - if (hlen > IXL_RX_HDR) - hlen = IXL_RX_HDR; - mh->m_len = hlen; - mh->m_flags |= M_PKTHDR; - mh->m_next = NULL; - mh->m_pkthdr.len = mh->m_len; - /* Null buf pointer so it is refreshed */ - rbuf->m_head = NULL; - /* - ** Check the payload length, this - ** could be zero if its a small - ** packet. - */ - if (plen > 0) { - mp->m_len = plen; - mp->m_next = NULL; - mp->m_flags &= ~M_PKTHDR; - mh->m_next = mp; - mh->m_pkthdr.len += mp->m_len; - /* Null buf pointer so it is refreshed */ - rbuf->m_pack = NULL; - rxr->split++; - } - /* - ** Now create the forward - ** chain so when complete - ** we wont have to. - */ - if (eop == 0) { - /* stash the chain head */ - nbuf->fmp = mh; - /* Make forward chain */ - if (plen) - mp->m_next = nbuf->m_pack; - else - mh->m_next = nbuf->m_pack; - } else { - /* Singlet, prepare to send */ - sendmp = mh; - if (vtag) { - sendmp->m_pkthdr.ether_vtag = vtag; - sendmp->m_flags |= M_VLANTAG; - } - } - } else { - /* - ** Either no header split, or a - ** secondary piece of a fragmented - ** split packet. - */ - mp->m_len = plen; - /* - ** See if there is a stored head - ** that determines what we are - */ - sendmp = rbuf->fmp; - rbuf->m_pack = rbuf->fmp = NULL; - - if (sendmp != NULL) /* secondary frag */ - sendmp->m_pkthdr.len += mp->m_len; - else { - /* first desc of a non-ps chain */ - sendmp = mp; - sendmp->m_flags |= M_PKTHDR; - sendmp->m_pkthdr.len = mp->m_len; - if (vtag) { - sendmp->m_pkthdr.ether_vtag = vtag; - sendmp->m_flags |= M_VLANTAG; - } - } - /* Pass the head pointer on */ - if (eop == 0) { - nbuf->fmp = sendmp; - sendmp = NULL; - mp->m_next = nbuf->m_pack; - } - } - ++processed; - /* Sending this frame? */ - if (eop) { - sendmp->m_pkthdr.rcvif = ifp; - /* gather stats */ - rxr->rx_packets++; - rxr->rx_bytes += sendmp->m_pkthdr.len; - /* capture data for dynamic ITR adjustment */ - rxr->packets++; - rxr->bytes += sendmp->m_pkthdr.len; - if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) - ixl_rx_checksum(sendmp, status, error, ptype); -#ifdef RSS - sendmp->m_pkthdr.flowid = - le32toh(cur->wb.qword0.hi_dword.rss); - M_HASHTYPE_SET(sendmp, ixl_ptype_to_hash(ptype)); +#include "opt_iflib.h" +#ifdef IFLIB +#include "iflib_ixl_txrx.c" #else - sendmp->m_pkthdr.flowid = que->msix; - M_HASHTYPE_SET(sendmp, M_HASHTYPE_OPAQUE); -#endif - } -next_desc: - bus_dmamap_sync(rxr->dma.tag, rxr->dma.map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - - /* Advance our pointers to the next descriptor. */ - if (++i == que->num_desc) - i = 0; - - /* Now send to the stack or do LRO */ - if (sendmp != NULL) { - rxr->next_check = i; - ixl_rx_input(rxr, ifp, sendmp, ptype); - i = rxr->next_check; - } - - /* Every 8 descriptors we go to refresh mbufs */ - if (processed == 8) { - ixl_refresh_mbufs(que, i); - processed = 0; - } - } - - /* Refresh any remaining buf structs */ - if (ixl_rx_unrefreshed(que)) - ixl_refresh_mbufs(que, i); - - rxr->next_check = i; - -#if defined(INET6) || defined(INET) - /* - * Flush any outstanding LRO work - */ - tcp_lro_flush_all(lro); -#endif - - IXL_RX_UNLOCK(rxr); - return (FALSE); -} - - -/********************************************************************* - * - * Verify that the hardware indicated that the checksum is valid. - * Inform the stack about the status of checksum so that stack - * doesn't spend time verifying the checksum. - * - *********************************************************************/ -static void -ixl_rx_checksum(struct mbuf * mp, u32 status, u32 error, u8 ptype) -{ - struct i40e_rx_ptype_decoded decoded; - - decoded = decode_rx_desc_ptype(ptype); - - /* Errors? */ - if (error & ((1 << I40E_RX_DESC_ERROR_IPE_SHIFT) | - (1 << I40E_RX_DESC_ERROR_L4E_SHIFT))) { - mp->m_pkthdr.csum_flags = 0; - return; - } - - /* IPv6 with extension headers likely have bad csum */ - if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP && - decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6) - if (status & - (1 << I40E_RX_DESC_STATUS_IPV6EXADD_SHIFT)) { - mp->m_pkthdr.csum_flags = 0; - return; - } - - - /* IP Checksum Good */ - mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED; - mp->m_pkthdr.csum_flags |= CSUM_IP_VALID; - - if (status & (1 << I40E_RX_DESC_STATUS_L3L4P_SHIFT)) { - mp->m_pkthdr.csum_flags |= - (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); - mp->m_pkthdr.csum_data |= htons(0xffff); - } - return; -} - -#if __FreeBSD_version >= 1100000 -uint64_t -ixl_get_counter(if_t ifp, ift_counter cnt) -{ - struct ixl_vsi *vsi; - - vsi = if_getsoftc(ifp); - - switch (cnt) { - case IFCOUNTER_IPACKETS: - return (vsi->ipackets); - case IFCOUNTER_IERRORS: - return (vsi->ierrors); - case IFCOUNTER_OPACKETS: - return (vsi->opackets); - case IFCOUNTER_OERRORS: - return (vsi->oerrors); - case IFCOUNTER_COLLISIONS: - /* Collisions are by standard impossible in 40G/10G Ethernet */ - return (0); - case IFCOUNTER_IBYTES: - return (vsi->ibytes); - case IFCOUNTER_OBYTES: - return (vsi->obytes); - case IFCOUNTER_IMCASTS: - return (vsi->imcasts); - case IFCOUNTER_OMCASTS: - return (vsi->omcasts); - case IFCOUNTER_IQDROPS: - return (vsi->iqdrops); - case IFCOUNTER_OQDROPS: - return (vsi->oqdrops); - case IFCOUNTER_NOPROTO: - return (vsi->noproto); - default: - return (if_get_counter_default(ifp, cnt)); - } -} +#include "legacy_ixl_txrx.c" #endif - Index: sys/dev/ixl/ixlv.h =================================================================== --- sys/dev/ixl/ixlv.h +++ sys/dev/ixl/ixlv.h @@ -1,213 +1,6 @@ -/****************************************************************************** - - Copyright (c) 2013-2015, Intel Corporation - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - 3. Neither the name of the Intel Corporation nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - -******************************************************************************/ -/*$FreeBSD$*/ - - -#ifndef _IXLV_H_ -#define _IXLV_H_ - -#include "ixlv_vc_mgr.h" - -#define IXLV_AQ_MAX_ERR 1000 -#define IXLV_MAX_FILTERS 128 -#define IXLV_MAX_QUEUES 16 -#define IXLV_AQ_TIMEOUT (1 * hz) -#define IXLV_CALLOUT_TIMO (hz / 50) /* 20 msec */ - -#define IXLV_FLAG_AQ_ENABLE_QUEUES (u32)(1) -#define IXLV_FLAG_AQ_DISABLE_QUEUES (u32)(1 << 1) -#define IXLV_FLAG_AQ_ADD_MAC_FILTER (u32)(1 << 2) -#define IXLV_FLAG_AQ_ADD_VLAN_FILTER (u32)(1 << 3) -#define IXLV_FLAG_AQ_DEL_MAC_FILTER (u32)(1 << 4) -#define IXLV_FLAG_AQ_DEL_VLAN_FILTER (u32)(1 << 5) -#define IXLV_FLAG_AQ_CONFIGURE_QUEUES (u32)(1 << 6) -#define IXLV_FLAG_AQ_MAP_VECTORS (u32)(1 << 7) -#define IXLV_FLAG_AQ_HANDLE_RESET (u32)(1 << 8) -#define IXLV_FLAG_AQ_CONFIGURE_PROMISC (u32)(1 << 9) -#define IXLV_FLAG_AQ_GET_STATS (u32)(1 << 10) - -/* printf %b arg */ -#define IXLV_FLAGS \ - "\20\1ENABLE_QUEUES\2DISABLE_QUEUES\3ADD_MAC_FILTER" \ - "\4ADD_VLAN_FILTER\5DEL_MAC_FILTER\6DEL_VLAN_FILTER" \ - "\7CONFIGURE_QUEUES\10MAP_VECTORS\11HANDLE_RESET" \ - "\12CONFIGURE_PROMISC\13GET_STATS" - -/* Hack for compatibility with 1.0.x linux pf driver */ -#define I40E_VIRTCHNL_OP_EVENT 17 - -/* Driver state */ -enum ixlv_state_t { - IXLV_START, - IXLV_FAILED, - IXLV_RESET_REQUIRED, - IXLV_RESET_PENDING, - IXLV_VERSION_CHECK, - IXLV_GET_RESOURCES, - IXLV_INIT_READY, - IXLV_INIT_START, - IXLV_INIT_CONFIG, - IXLV_INIT_MAPPING, - IXLV_INIT_ENABLE, - IXLV_INIT_COMPLETE, - IXLV_RUNNING, -}; - -struct ixlv_mac_filter { - SLIST_ENTRY(ixlv_mac_filter) next; - u8 macaddr[ETHER_ADDR_LEN]; - u16 flags; -}; -SLIST_HEAD(mac_list, ixlv_mac_filter); - -struct ixlv_vlan_filter { - SLIST_ENTRY(ixlv_vlan_filter) next; - u16 vlan; - u16 flags; -}; -SLIST_HEAD(vlan_list, ixlv_vlan_filter); - -/* Software controller structure */ -struct ixlv_sc { - struct i40e_hw hw; - struct i40e_osdep osdep; - struct device *dev; - - struct resource *pci_mem; - struct resource *msix_mem; - - enum ixlv_state_t init_state; - - /* - * Interrupt resources - */ - void *tag; - struct resource *res; /* For the AQ */ - - struct ifmedia media; - struct callout timer; - int msix; - int pf_version; - int if_flags; - - bool link_up; - u32 link_speed; - - struct mtx mtx; - - u32 qbase; - u32 admvec; - struct timeout_task timeout; - struct task aq_irq; - struct task aq_sched; - struct taskqueue *tq; - - struct ixl_vsi vsi; - - /* Filter lists */ - struct mac_list *mac_filters; - struct vlan_list *vlan_filters; - - /* Promiscuous mode */ - u32 promiscuous_flags; - - /* Admin queue task flags */ - u32 aq_wait_count; - - struct ixl_vc_mgr vc_mgr; - struct ixl_vc_cmd add_mac_cmd; - struct ixl_vc_cmd del_mac_cmd; - struct ixl_vc_cmd config_queues_cmd; - struct ixl_vc_cmd map_vectors_cmd; - struct ixl_vc_cmd enable_queues_cmd; - struct ixl_vc_cmd add_vlan_cmd; - struct ixl_vc_cmd del_vlan_cmd; - struct ixl_vc_cmd add_multi_cmd; - struct ixl_vc_cmd del_multi_cmd; - - /* Virtual comm channel */ - struct i40e_virtchnl_vf_resource *vf_res; - struct i40e_virtchnl_vsi_resource *vsi_res; - - /* Misc stats maintained by the driver */ - u64 watchdog_events; - u64 admin_irq; - - u8 aq_buffer[IXL_AQ_BUF_SZ]; -}; - -#define IXLV_CORE_LOCK_ASSERT(sc) mtx_assert(&(sc)->mtx, MA_OWNED) -/* -** This checks for a zero mac addr, something that will be likely -** unless the Admin on the Host has created one. -*/ -static inline bool -ixlv_check_ether_addr(u8 *addr) -{ - bool status = TRUE; - - if ((addr[0] == 0 && addr[1]== 0 && addr[2] == 0 && - addr[3] == 0 && addr[4]== 0 && addr[5] == 0)) - status = FALSE; - return (status); -} - -/* -** VF Common function prototypes -*/ -int ixlv_send_api_ver(struct ixlv_sc *); -int ixlv_verify_api_ver(struct ixlv_sc *); -int ixlv_send_vf_config_msg(struct ixlv_sc *); -int ixlv_get_vf_config(struct ixlv_sc *); -void ixlv_init(void *); -int ixlv_reinit_locked(struct ixlv_sc *); -void ixlv_configure_queues(struct ixlv_sc *); -void ixlv_enable_queues(struct ixlv_sc *); -void ixlv_disable_queues(struct ixlv_sc *); -void ixlv_map_queues(struct ixlv_sc *); -void ixlv_enable_intr(struct ixl_vsi *); -void ixlv_disable_intr(struct ixl_vsi *); -void ixlv_add_ether_filters(struct ixlv_sc *); -void ixlv_del_ether_filters(struct ixlv_sc *); -void ixlv_request_stats(struct ixlv_sc *); -void ixlv_request_reset(struct ixlv_sc *); -void ixlv_vc_completion(struct ixlv_sc *, - enum i40e_virtchnl_ops, i40e_status, u8 *, u16); -void ixlv_add_ether_filter(struct ixlv_sc *); -void ixlv_add_vlans(struct ixlv_sc *); -void ixlv_del_vlans(struct ixlv_sc *); -void ixlv_update_stats_counters(struct ixlv_sc *, - struct i40e_eth_stats *); -void ixlv_update_link_status(struct ixlv_sc *); - -#endif /* _IXLV_H_ */ +#include "opt_iflib.h" +#ifdef IFLIB +#include "iflib_ixlv.h" +#else +#include "legacy_ixlv.h" +#endif Index: sys/dev/ixl/ixlvc.c =================================================================== --- sys/dev/ixl/ixlvc.c +++ sys/dev/ixl/ixlvc.c @@ -1,1133 +1,8 @@ -/****************************************************************************** - Copyright (c) 2013-2015, Intel Corporation - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - 3. Neither the name of the Intel Corporation nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. +#include "opt_iflib.h" -******************************************************************************/ -/*$FreeBSD$*/ - -/* -** Virtual Channel support -** These are support functions to communication -** between the VF and PF drivers. -*/ - -#include "ixl.h" -#include "ixlv.h" -#include "i40e_prototype.h" - - -/* busy wait delay in msec */ -#define IXLV_BUSY_WAIT_DELAY 10 -#define IXLV_BUSY_WAIT_COUNT 50 - -static void ixl_vc_process_resp(struct ixl_vc_mgr *, uint32_t, - enum i40e_status_code); -static void ixl_vc_process_next(struct ixl_vc_mgr *mgr); -static void ixl_vc_schedule_retry(struct ixl_vc_mgr *mgr); -static void ixl_vc_send_current(struct ixl_vc_mgr *mgr); - -#ifdef IXL_DEBUG -/* -** Validate VF messages -*/ -static int ixl_vc_validate_vf_msg(struct ixlv_sc *sc, u32 v_opcode, - u8 *msg, u16 msglen) -{ - bool err_msg_format = false; - int valid_len; - - /* Validate message length. */ - switch (v_opcode) { - case I40E_VIRTCHNL_OP_VERSION: - valid_len = sizeof(struct i40e_virtchnl_version_info); - break; - case I40E_VIRTCHNL_OP_RESET_VF: - case I40E_VIRTCHNL_OP_GET_VF_RESOURCES: - valid_len = 0; - break; - case I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE: - valid_len = sizeof(struct i40e_virtchnl_txq_info); - break; - case I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE: - valid_len = sizeof(struct i40e_virtchnl_rxq_info); - break; - case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES: - valid_len = sizeof(struct i40e_virtchnl_vsi_queue_config_info); - if (msglen >= valid_len) { - struct i40e_virtchnl_vsi_queue_config_info *vqc = - (struct i40e_virtchnl_vsi_queue_config_info *)msg; - valid_len += (vqc->num_queue_pairs * - sizeof(struct - i40e_virtchnl_queue_pair_info)); - if (vqc->num_queue_pairs == 0) - err_msg_format = true; - } - break; - case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP: - valid_len = sizeof(struct i40e_virtchnl_irq_map_info); - if (msglen >= valid_len) { - struct i40e_virtchnl_irq_map_info *vimi = - (struct i40e_virtchnl_irq_map_info *)msg; - valid_len += (vimi->num_vectors * - sizeof(struct i40e_virtchnl_vector_map)); - if (vimi->num_vectors == 0) - err_msg_format = true; - } - break; - case I40E_VIRTCHNL_OP_ENABLE_QUEUES: - case I40E_VIRTCHNL_OP_DISABLE_QUEUES: - valid_len = sizeof(struct i40e_virtchnl_queue_select); - break; - case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS: - case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS: - valid_len = sizeof(struct i40e_virtchnl_ether_addr_list); - if (msglen >= valid_len) { - struct i40e_virtchnl_ether_addr_list *veal = - (struct i40e_virtchnl_ether_addr_list *)msg; - valid_len += veal->num_elements * - sizeof(struct i40e_virtchnl_ether_addr); - if (veal->num_elements == 0) - err_msg_format = true; - } - break; - case I40E_VIRTCHNL_OP_ADD_VLAN: - case I40E_VIRTCHNL_OP_DEL_VLAN: - valid_len = sizeof(struct i40e_virtchnl_vlan_filter_list); - if (msglen >= valid_len) { - struct i40e_virtchnl_vlan_filter_list *vfl = - (struct i40e_virtchnl_vlan_filter_list *)msg; - valid_len += vfl->num_elements * sizeof(u16); - if (vfl->num_elements == 0) - err_msg_format = true; - } - break; - case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE: - valid_len = sizeof(struct i40e_virtchnl_promisc_info); - break; - case I40E_VIRTCHNL_OP_GET_STATS: - valid_len = sizeof(struct i40e_virtchnl_queue_select); - break; - /* These are always errors coming from the VF. */ - case I40E_VIRTCHNL_OP_EVENT: - case I40E_VIRTCHNL_OP_UNKNOWN: - default: - return EPERM; - break; - } - /* few more checks */ - if ((valid_len != msglen) || (err_msg_format)) - return EINVAL; - else - return 0; -} -#endif - -/* -** ixlv_send_pf_msg -** -** Send message to PF and print status if failure. -*/ -static int -ixlv_send_pf_msg(struct ixlv_sc *sc, - enum i40e_virtchnl_ops op, u8 *msg, u16 len) -{ - struct i40e_hw *hw = &sc->hw; - device_t dev = sc->dev; - i40e_status err; - -#ifdef IXL_DEBUG - /* - ** Pre-validating messages to the PF - */ - int val_err; - val_err = ixl_vc_validate_vf_msg(sc, op, msg, len); - if (val_err) - device_printf(dev, "Error validating msg to PF for op %d," - " msglen %d: error %d\n", op, len, val_err); -#endif - - err = i40e_aq_send_msg_to_pf(hw, op, I40E_SUCCESS, msg, len, NULL); - if (err) - device_printf(dev, "Unable to send opcode %d to PF, " - "error %d, aq status %d\n", op, err, hw->aq.asq_last_status); - return err; -} - - -/* -** ixlv_send_api_ver -** -** Send API version admin queue message to the PF. The reply is not checked -** in this function. Returns 0 if the message was successfully -** sent, or one of the I40E_ADMIN_QUEUE_ERROR_ statuses if not. -*/ -int -ixlv_send_api_ver(struct ixlv_sc *sc) -{ - struct i40e_virtchnl_version_info vvi; - - vvi.major = I40E_VIRTCHNL_VERSION_MAJOR; - vvi.minor = I40E_VIRTCHNL_VERSION_MINOR; - - return ixlv_send_pf_msg(sc, I40E_VIRTCHNL_OP_VERSION, - (u8 *)&vvi, sizeof(vvi)); -} - -/* -** ixlv_verify_api_ver -** -** Compare API versions with the PF. Must be called after admin queue is -** initialized. Returns 0 if API versions match, EIO if -** they do not, or I40E_ERR_ADMIN_QUEUE_NO_WORK if the admin queue is empty. -*/ -int -ixlv_verify_api_ver(struct ixlv_sc *sc) -{ - struct i40e_virtchnl_version_info *pf_vvi; - struct i40e_hw *hw = &sc->hw; - struct i40e_arq_event_info event; - i40e_status err; - int retries = 0; - - event.buf_len = IXL_AQ_BUFSZ; - event.msg_buf = malloc(event.buf_len, M_DEVBUF, M_NOWAIT); - if (!event.msg_buf) { - err = ENOMEM; - goto out; - } - - do { - if (++retries > IXLV_AQ_MAX_ERR) - goto out_alloc; - - /* NOTE: initial delay is necessary */ - i40e_msec_delay(100); - err = i40e_clean_arq_element(hw, &event, NULL); - } while (err == I40E_ERR_ADMIN_QUEUE_NO_WORK); - if (err) - goto out_alloc; - - err = (i40e_status)le32toh(event.desc.cookie_low); - if (err) { - err = EIO; - goto out_alloc; - } - - if ((enum i40e_virtchnl_ops)le32toh(event.desc.cookie_high) != - I40E_VIRTCHNL_OP_VERSION) { - DDPRINTF(sc->dev, "Received unexpected op response: %d\n", - le32toh(event.desc.cookie_high)); - err = EIO; - goto out_alloc; - } - - pf_vvi = (struct i40e_virtchnl_version_info *)event.msg_buf; - if ((pf_vvi->major > I40E_VIRTCHNL_VERSION_MAJOR) || - ((pf_vvi->major == I40E_VIRTCHNL_VERSION_MAJOR) && - (pf_vvi->minor > I40E_VIRTCHNL_VERSION_MINOR))) - err = EIO; - else - sc->pf_version = pf_vvi->minor; - -out_alloc: - free(event.msg_buf, M_DEVBUF); -out: - return err; -} - -/* -** ixlv_send_vf_config_msg -** -** Send VF configuration request admin queue message to the PF. The reply -** is not checked in this function. Returns 0 if the message was -** successfully sent, or one of the I40E_ADMIN_QUEUE_ERROR_ statuses if not. -*/ -int -ixlv_send_vf_config_msg(struct ixlv_sc *sc) -{ - u32 caps; - - caps = I40E_VIRTCHNL_VF_OFFLOAD_L2 | - I40E_VIRTCHNL_VF_OFFLOAD_RSS_AQ | - I40E_VIRTCHNL_VF_OFFLOAD_RSS_REG | - I40E_VIRTCHNL_VF_OFFLOAD_VLAN; - - if (sc->pf_version) - return ixlv_send_pf_msg(sc, I40E_VIRTCHNL_OP_GET_VF_RESOURCES, - (u8 *)&caps, sizeof(caps)); - else - return ixlv_send_pf_msg(sc, I40E_VIRTCHNL_OP_GET_VF_RESOURCES, - NULL, 0); -} - -/* -** ixlv_get_vf_config -** -** Get VF configuration from PF and populate hw structure. Must be called after -** admin queue is initialized. Busy waits until response is received from PF, -** with maximum timeout. Response from PF is returned in the buffer for further -** processing by the caller. -*/ -int -ixlv_get_vf_config(struct ixlv_sc *sc) -{ - struct i40e_hw *hw = &sc->hw; - device_t dev = sc->dev; - struct i40e_arq_event_info event; - u16 len; - i40e_status err = 0; - u32 retries = 0; - - /* Note this assumes a single VSI */ - len = sizeof(struct i40e_virtchnl_vf_resource) + - sizeof(struct i40e_virtchnl_vsi_resource); - event.buf_len = len; - event.msg_buf = malloc(event.buf_len, M_DEVBUF, M_NOWAIT); - if (!event.msg_buf) { - err = ENOMEM; - goto out; - } - - for (;;) { - err = i40e_clean_arq_element(hw, &event, NULL); - if (err == I40E_ERR_ADMIN_QUEUE_NO_WORK) { - if (++retries <= IXLV_AQ_MAX_ERR) - i40e_msec_delay(10); - } else if ((enum i40e_virtchnl_ops)le32toh(event.desc.cookie_high) != - I40E_VIRTCHNL_OP_GET_VF_RESOURCES) { - DDPRINTF(dev, "Received a response from PF," - " opcode %d, error %d", - le32toh(event.desc.cookie_high), - le32toh(event.desc.cookie_low)); - retries++; - continue; - } else { - err = (i40e_status)le32toh(event.desc.cookie_low); - if (err) { - device_printf(dev, "%s: Error returned from PF," - " opcode %d, error %d\n", __func__, - le32toh(event.desc.cookie_high), - le32toh(event.desc.cookie_low)); - err = EIO; - goto out_alloc; - } - /* We retrieved the config message, with no errors */ - break; - } - - if (retries > IXLV_AQ_MAX_ERR) { - INIT_DBG_DEV(dev, "Did not receive response after %d tries.", - retries); - err = ETIMEDOUT; - goto out_alloc; - } - } - - memcpy(sc->vf_res, event.msg_buf, min(event.msg_len, len)); - i40e_vf_parse_hw_config(hw, sc->vf_res); - -out_alloc: - free(event.msg_buf, M_DEVBUF); -out: - return err; -} - -/* -** ixlv_configure_queues -** -** Request that the PF set up our queues. -*/ -void -ixlv_configure_queues(struct ixlv_sc *sc) -{ - device_t dev = sc->dev; - struct ixl_vsi *vsi = &sc->vsi; - struct ixl_queue *que = vsi->queues; - struct tx_ring *txr; - struct rx_ring *rxr; - int len, pairs; - - struct i40e_virtchnl_vsi_queue_config_info *vqci; - struct i40e_virtchnl_queue_pair_info *vqpi; - - pairs = vsi->num_queues; - len = sizeof(struct i40e_virtchnl_vsi_queue_config_info) + - (sizeof(struct i40e_virtchnl_queue_pair_info) * pairs); - vqci = malloc(len, M_DEVBUF, M_NOWAIT | M_ZERO); - if (!vqci) { - device_printf(dev, "%s: unable to allocate memory\n", __func__); - ixl_vc_schedule_retry(&sc->vc_mgr); - return; - } - vqci->vsi_id = sc->vsi_res->vsi_id; - vqci->num_queue_pairs = pairs; - vqpi = vqci->qpair; - /* Size check is not needed here - HW max is 16 queue pairs, and we - * can fit info for 31 of them into the AQ buffer before it overflows. - */ - for (int i = 0; i < pairs; i++, que++, vqpi++) { - txr = &que->txr; - rxr = &que->rxr; - vqpi->txq.vsi_id = vqci->vsi_id; - vqpi->txq.queue_id = i; - vqpi->txq.ring_len = que->num_desc; - vqpi->txq.dma_ring_addr = txr->dma.pa; - /* Enable Head writeback */ - vqpi->txq.headwb_enabled = 1; - vqpi->txq.dma_headwb_addr = txr->dma.pa + - (que->num_desc * sizeof(struct i40e_tx_desc)); - - vqpi->rxq.vsi_id = vqci->vsi_id; - vqpi->rxq.queue_id = i; - vqpi->rxq.ring_len = que->num_desc; - vqpi->rxq.dma_ring_addr = rxr->dma.pa; - vqpi->rxq.max_pkt_size = vsi->max_frame_size; - vqpi->rxq.databuffer_size = rxr->mbuf_sz; - vqpi->rxq.splithdr_enabled = 0; - } - - ixlv_send_pf_msg(sc, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, - (u8 *)vqci, len); - free(vqci, M_DEVBUF); -} - -/* -** ixlv_enable_queues -** -** Request that the PF enable all of our queues. -*/ -void -ixlv_enable_queues(struct ixlv_sc *sc) -{ - struct i40e_virtchnl_queue_select vqs; - - vqs.vsi_id = sc->vsi_res->vsi_id; - vqs.tx_queues = (1 << sc->vsi_res->num_queue_pairs) - 1; - vqs.rx_queues = vqs.tx_queues; - ixlv_send_pf_msg(sc, I40E_VIRTCHNL_OP_ENABLE_QUEUES, - (u8 *)&vqs, sizeof(vqs)); -} - -/* -** ixlv_disable_queues -** -** Request that the PF disable all of our queues. -*/ -void -ixlv_disable_queues(struct ixlv_sc *sc) -{ - struct i40e_virtchnl_queue_select vqs; - - vqs.vsi_id = sc->vsi_res->vsi_id; - vqs.tx_queues = (1 << sc->vsi_res->num_queue_pairs) - 1; - vqs.rx_queues = vqs.tx_queues; - ixlv_send_pf_msg(sc, I40E_VIRTCHNL_OP_DISABLE_QUEUES, - (u8 *)&vqs, sizeof(vqs)); -} - -/* -** ixlv_map_queues -** -** Request that the PF map queues to interrupt vectors. Misc causes, including -** admin queue, are always mapped to vector 0. -*/ -void -ixlv_map_queues(struct ixlv_sc *sc) -{ - struct i40e_virtchnl_irq_map_info *vm; - int i, q, len; - struct ixl_vsi *vsi = &sc->vsi; - struct ixl_queue *que = vsi->queues; - - /* How many queue vectors, adminq uses one */ - q = sc->msix - 1; - - len = sizeof(struct i40e_virtchnl_irq_map_info) + - (sc->msix * sizeof(struct i40e_virtchnl_vector_map)); - vm = malloc(len, M_DEVBUF, M_NOWAIT); - if (!vm) { - printf("%s: unable to allocate memory\n", __func__); - ixl_vc_schedule_retry(&sc->vc_mgr); - return; - } - - vm->num_vectors = sc->msix; - /* Queue vectors first */ - for (i = 0; i < q; i++, que++) { - vm->vecmap[i].vsi_id = sc->vsi_res->vsi_id; - vm->vecmap[i].vector_id = i + 1; /* first is adminq */ - vm->vecmap[i].txq_map = (1 << que->me); - vm->vecmap[i].rxq_map = (1 << que->me); - vm->vecmap[i].rxitr_idx = 0; - vm->vecmap[i].txitr_idx = 0; - } - - /* Misc vector last - this is only for AdminQ messages */ - vm->vecmap[i].vsi_id = sc->vsi_res->vsi_id; - vm->vecmap[i].vector_id = 0; - vm->vecmap[i].txq_map = 0; - vm->vecmap[i].rxq_map = 0; - vm->vecmap[i].rxitr_idx = 0; - vm->vecmap[i].txitr_idx = 0; - - ixlv_send_pf_msg(sc, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, - (u8 *)vm, len); - free(vm, M_DEVBUF); -} - -/* -** Scan the Filter List looking for vlans that need -** to be added, then create the data to hand to the AQ -** for handling. -*/ -void -ixlv_add_vlans(struct ixlv_sc *sc) -{ - struct i40e_virtchnl_vlan_filter_list *v; - struct ixlv_vlan_filter *f, *ftmp; - device_t dev = sc->dev; - int len, i = 0, cnt = 0; - - /* Get count of VLAN filters to add */ - SLIST_FOREACH(f, sc->vlan_filters, next) { - if (f->flags & IXL_FILTER_ADD) - cnt++; - } - - if (!cnt) { /* no work... */ - ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_ADD_VLAN_FILTER, - I40E_SUCCESS); - return; - } - - len = sizeof(struct i40e_virtchnl_vlan_filter_list) + - (cnt * sizeof(u16)); - - if (len > IXL_AQ_BUF_SZ) { - device_printf(dev, "%s: Exceeded Max AQ Buf size\n", - __func__); - ixl_vc_schedule_retry(&sc->vc_mgr); - return; - } - - v = malloc(len, M_DEVBUF, M_NOWAIT); - if (!v) { - device_printf(dev, "%s: unable to allocate memory\n", - __func__); - ixl_vc_schedule_retry(&sc->vc_mgr); - return; - } - - v->vsi_id = sc->vsi_res->vsi_id; - v->num_elements = cnt; - - /* Scan the filter array */ - SLIST_FOREACH_SAFE(f, sc->vlan_filters, next, ftmp) { - if (f->flags & IXL_FILTER_ADD) { - bcopy(&f->vlan, &v->vlan_id[i], sizeof(u16)); - f->flags = IXL_FILTER_USED; - i++; - } - if (i == cnt) - break; - } - // ERJ: Should this be taken out? - if (i == 0) { /* Should not happen... */ - device_printf(dev, "%s: i == 0?\n", __func__); - ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_ADD_VLAN_FILTER, - I40E_SUCCESS); - return; - } - - ixlv_send_pf_msg(sc, I40E_VIRTCHNL_OP_ADD_VLAN, (u8 *)v, len); - free(v, M_DEVBUF); - /* add stats? */ -} - -/* -** Scan the Filter Table looking for vlans that need -** to be removed, then create the data to hand to the AQ -** for handling. -*/ -void -ixlv_del_vlans(struct ixlv_sc *sc) -{ - device_t dev = sc->dev; - struct i40e_virtchnl_vlan_filter_list *v; - struct ixlv_vlan_filter *f, *ftmp; - int len, i = 0, cnt = 0; - - /* Get count of VLAN filters to delete */ - SLIST_FOREACH(f, sc->vlan_filters, next) { - if (f->flags & IXL_FILTER_DEL) - cnt++; - } - - if (!cnt) { /* no work... */ - ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_DEL_VLAN_FILTER, - I40E_SUCCESS); - return; - } - - len = sizeof(struct i40e_virtchnl_vlan_filter_list) + - (cnt * sizeof(u16)); - - if (len > IXL_AQ_BUF_SZ) { - device_printf(dev, "%s: Exceeded Max AQ Buf size\n", - __func__); - ixl_vc_schedule_retry(&sc->vc_mgr); - return; - } - - v = malloc(len, M_DEVBUF, M_NOWAIT | M_ZERO); - if (!v) { - device_printf(dev, "%s: unable to allocate memory\n", - __func__); - ixl_vc_schedule_retry(&sc->vc_mgr); - return; - } - - v->vsi_id = sc->vsi_res->vsi_id; - v->num_elements = cnt; - - /* Scan the filter array */ - SLIST_FOREACH_SAFE(f, sc->vlan_filters, next, ftmp) { - if (f->flags & IXL_FILTER_DEL) { - bcopy(&f->vlan, &v->vlan_id[i], sizeof(u16)); - i++; - SLIST_REMOVE(sc->vlan_filters, f, ixlv_vlan_filter, next); - free(f, M_DEVBUF); - } - if (i == cnt) - break; - } - // ERJ: Take this out? - if (i == 0) { /* Should not happen... */ - device_printf(dev, "%s: i == 0?\n", __func__); - ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_DEL_VLAN_FILTER, - I40E_SUCCESS); - return; - } - - ixlv_send_pf_msg(sc, I40E_VIRTCHNL_OP_DEL_VLAN, (u8 *)v, len); - free(v, M_DEVBUF); - /* add stats? */ -} - - -/* -** This routine takes additions to the vsi filter -** table and creates an Admin Queue call to create -** the filters in the hardware. -*/ -void -ixlv_add_ether_filters(struct ixlv_sc *sc) -{ - struct i40e_virtchnl_ether_addr_list *a; - struct ixlv_mac_filter *f; - device_t dev = sc->dev; - int len, j = 0, cnt = 0; - - /* Get count of MAC addresses to add */ - SLIST_FOREACH(f, sc->mac_filters, next) { - if (f->flags & IXL_FILTER_ADD) - cnt++; - } - if (cnt == 0) { /* Should not happen... */ - DDPRINTF(dev, "cnt == 0, exiting..."); - ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_ADD_MAC_FILTER, - I40E_SUCCESS); - return; - } - - len = sizeof(struct i40e_virtchnl_ether_addr_list) + - (cnt * sizeof(struct i40e_virtchnl_ether_addr)); - - a = malloc(len, M_DEVBUF, M_NOWAIT | M_ZERO); - if (a == NULL) { - device_printf(dev, "%s: Failed to get memory for " - "virtchnl_ether_addr_list\n", __func__); - ixl_vc_schedule_retry(&sc->vc_mgr); - return; - } - a->vsi_id = sc->vsi.id; - a->num_elements = cnt; - - /* Scan the filter array */ - SLIST_FOREACH(f, sc->mac_filters, next) { - if (f->flags & IXL_FILTER_ADD) { - bcopy(f->macaddr, a->list[j].addr, ETHER_ADDR_LEN); - f->flags &= ~IXL_FILTER_ADD; - j++; - - DDPRINTF(dev, "ADD: " MAC_FORMAT, - MAC_FORMAT_ARGS(f->macaddr)); - } - if (j == cnt) - break; - } - DDPRINTF(dev, "len %d, j %d, cnt %d", - len, j, cnt); - ixlv_send_pf_msg(sc, - I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, (u8 *)a, len); - /* add stats? */ - free(a, M_DEVBUF); - return; -} - -/* -** This routine takes filters flagged for deletion in the -** sc MAC filter list and creates an Admin Queue call -** to delete those filters in the hardware. -*/ -void -ixlv_del_ether_filters(struct ixlv_sc *sc) -{ - struct i40e_virtchnl_ether_addr_list *d; - device_t dev = sc->dev; - struct ixlv_mac_filter *f, *f_temp; - int len, j = 0, cnt = 0; - - /* Get count of MAC addresses to delete */ - SLIST_FOREACH(f, sc->mac_filters, next) { - if (f->flags & IXL_FILTER_DEL) - cnt++; - } - if (cnt == 0) { - DDPRINTF(dev, "cnt == 0, exiting..."); - ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_DEL_MAC_FILTER, - I40E_SUCCESS); - return; - } - - len = sizeof(struct i40e_virtchnl_ether_addr_list) + - (cnt * sizeof(struct i40e_virtchnl_ether_addr)); - - d = malloc(len, M_DEVBUF, M_NOWAIT | M_ZERO); - if (d == NULL) { - device_printf(dev, "%s: Failed to get memory for " - "virtchnl_ether_addr_list\n", __func__); - ixl_vc_schedule_retry(&sc->vc_mgr); - return; - } - d->vsi_id = sc->vsi.id; - d->num_elements = cnt; - - /* Scan the filter array */ - SLIST_FOREACH_SAFE(f, sc->mac_filters, next, f_temp) { - if (f->flags & IXL_FILTER_DEL) { - bcopy(f->macaddr, d->list[j].addr, ETHER_ADDR_LEN); - DDPRINTF(dev, "DEL: " MAC_FORMAT, - MAC_FORMAT_ARGS(f->macaddr)); - j++; - SLIST_REMOVE(sc->mac_filters, f, ixlv_mac_filter, next); - free(f, M_DEVBUF); - } - if (j == cnt) - break; - } - ixlv_send_pf_msg(sc, - I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS, (u8 *)d, len); - /* add stats? */ - free(d, M_DEVBUF); - return; -} - -/* -** ixlv_request_reset -** Request that the PF reset this VF. No response is expected. -*/ -void -ixlv_request_reset(struct ixlv_sc *sc) -{ - /* - ** Set the reset status to "in progress" before - ** the request, this avoids any possibility of - ** a mistaken early detection of completion. - */ - wr32(&sc->hw, I40E_VFGEN_RSTAT, I40E_VFR_INPROGRESS); - ixlv_send_pf_msg(sc, I40E_VIRTCHNL_OP_RESET_VF, NULL, 0); -} - -/* -** ixlv_request_stats -** Request the statistics for this VF's VSI from PF. -*/ -void -ixlv_request_stats(struct ixlv_sc *sc) -{ - struct i40e_virtchnl_queue_select vqs; - int error = 0; - - vqs.vsi_id = sc->vsi_res->vsi_id; - /* Low priority, we don't need to error check */ - error = ixlv_send_pf_msg(sc, I40E_VIRTCHNL_OP_GET_STATS, - (u8 *)&vqs, sizeof(vqs)); -#ifdef IXL_DEBUG - if (error) - device_printf(sc->dev, "Error sending stats request to PF: %d\n", error); -#endif -} - -/* -** Updates driver's stats counters with VSI stats returned from PF. -*/ -void -ixlv_update_stats_counters(struct ixlv_sc *sc, struct i40e_eth_stats *es) -{ - struct ixl_vsi *vsi = &sc->vsi; - uint64_t tx_discards; - - tx_discards = es->tx_discards; - for (int i = 0; i < vsi->num_queues; i++) - tx_discards += sc->vsi.queues[i].txr.br->br_drops; - - /* Update ifnet stats */ - IXL_SET_IPACKETS(vsi, es->rx_unicast + - es->rx_multicast + - es->rx_broadcast); - IXL_SET_OPACKETS(vsi, es->tx_unicast + - es->tx_multicast + - es->tx_broadcast); - IXL_SET_IBYTES(vsi, es->rx_bytes); - IXL_SET_OBYTES(vsi, es->tx_bytes); - IXL_SET_IMCASTS(vsi, es->rx_multicast); - IXL_SET_OMCASTS(vsi, es->tx_multicast); - - IXL_SET_OERRORS(vsi, es->tx_errors); - IXL_SET_IQDROPS(vsi, es->rx_discards); - IXL_SET_OQDROPS(vsi, tx_discards); - IXL_SET_NOPROTO(vsi, es->rx_unknown_protocol); - IXL_SET_COLLISIONS(vsi, 0); - - vsi->eth_stats = *es; -} - -/* -** ixlv_vc_completion -** -** Asynchronous completion function for admin queue messages. Rather than busy -** wait, we fire off our requests and assume that no errors will be returned. -** This function handles the reply messages. -*/ -void -ixlv_vc_completion(struct ixlv_sc *sc, - enum i40e_virtchnl_ops v_opcode, - i40e_status v_retval, u8 *msg, u16 msglen) -{ - device_t dev = sc->dev; - struct ixl_vsi *vsi = &sc->vsi; - - if (v_opcode == I40E_VIRTCHNL_OP_EVENT) { - struct i40e_virtchnl_pf_event *vpe = - (struct i40e_virtchnl_pf_event *)msg; - - switch (vpe->event) { - case I40E_VIRTCHNL_EVENT_LINK_CHANGE: -#ifdef IXL_DEBUG - device_printf(dev, "Link change: status %d, speed %d\n", - vpe->event_data.link_event.link_status, - vpe->event_data.link_event.link_speed); +#ifdef IFLIB +#include "iflib_ixlvc.c" +#else +#include "legacy_ixlvc.c" #endif - sc->link_up = - vpe->event_data.link_event.link_status; - sc->link_speed = - vpe->event_data.link_event.link_speed; - ixlv_update_link_status(sc); - break; - case I40E_VIRTCHNL_EVENT_RESET_IMPENDING: - device_printf(dev, "PF initiated reset!\n"); - sc->init_state = IXLV_RESET_PENDING; - ixlv_init(sc); - break; - default: - device_printf(dev, "%s: Unknown event %d from AQ\n", - __func__, vpe->event); - break; - } - - return; - } - - /* Catch-all error response */ - if (v_retval) { - device_printf(dev, - "%s: AQ returned error %d to our request %d!\n", - __func__, v_retval, v_opcode); - } - -#ifdef IXL_DEBUG - if (v_opcode != I40E_VIRTCHNL_OP_GET_STATS) - DDPRINTF(dev, "opcode %d", v_opcode); -#endif - - switch (v_opcode) { - case I40E_VIRTCHNL_OP_GET_STATS: - ixlv_update_stats_counters(sc, (struct i40e_eth_stats *)msg); - break; - case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS: - ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_ADD_MAC_FILTER, - v_retval); - if (v_retval) { - device_printf(dev, "WARNING: Error adding VF mac filter!\n"); - device_printf(dev, "WARNING: Device may not receive traffic!\n"); - } - break; - case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS: - ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_DEL_MAC_FILTER, - v_retval); - break; - case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE: - ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_CONFIGURE_PROMISC, - v_retval); - break; - case I40E_VIRTCHNL_OP_ADD_VLAN: - ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_ADD_VLAN_FILTER, - v_retval); - break; - case I40E_VIRTCHNL_OP_DEL_VLAN: - ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_DEL_VLAN_FILTER, - v_retval); - break; - case I40E_VIRTCHNL_OP_ENABLE_QUEUES: - ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_ENABLE_QUEUES, - v_retval); - if (v_retval == 0) { - /* Update link status */ - ixlv_update_link_status(sc); - /* Turn on all interrupts */ - ixlv_enable_intr(vsi); - /* And inform the stack we're ready */ - vsi->ifp->if_drv_flags |= IFF_DRV_RUNNING; - vsi->ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; - } - break; - case I40E_VIRTCHNL_OP_DISABLE_QUEUES: - ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_DISABLE_QUEUES, - v_retval); - if (v_retval == 0) { - /* Turn off all interrupts */ - ixlv_disable_intr(vsi); - /* Tell the stack that the interface is no longer active */ - vsi->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); - } - break; - case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES: - ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_CONFIGURE_QUEUES, - v_retval); - break; - case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP: - ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_MAP_VECTORS, - v_retval); - break; - default: - device_printf(dev, - "%s: Received unexpected message %d from PF.\n", - __func__, v_opcode); - break; - } - return; -} - -static void -ixl_vc_send_cmd(struct ixlv_sc *sc, uint32_t request) -{ - - switch (request) { - case IXLV_FLAG_AQ_MAP_VECTORS: - ixlv_map_queues(sc); - break; - - case IXLV_FLAG_AQ_ADD_MAC_FILTER: - ixlv_add_ether_filters(sc); - break; - - case IXLV_FLAG_AQ_ADD_VLAN_FILTER: - ixlv_add_vlans(sc); - break; - - case IXLV_FLAG_AQ_DEL_MAC_FILTER: - ixlv_del_ether_filters(sc); - break; - - case IXLV_FLAG_AQ_DEL_VLAN_FILTER: - ixlv_del_vlans(sc); - break; - - case IXLV_FLAG_AQ_CONFIGURE_QUEUES: - ixlv_configure_queues(sc); - break; - - case IXLV_FLAG_AQ_DISABLE_QUEUES: - ixlv_disable_queues(sc); - break; - - case IXLV_FLAG_AQ_ENABLE_QUEUES: - ixlv_enable_queues(sc); - break; - } -} - -void -ixl_vc_init_mgr(struct ixlv_sc *sc, struct ixl_vc_mgr *mgr) -{ - mgr->sc = sc; - mgr->current = NULL; - TAILQ_INIT(&mgr->pending); - callout_init_mtx(&mgr->callout, &sc->mtx, 0); -} - -static void -ixl_vc_process_completion(struct ixl_vc_mgr *mgr, enum i40e_status_code err) -{ - struct ixl_vc_cmd *cmd; - - cmd = mgr->current; - mgr->current = NULL; - cmd->flags &= ~IXLV_VC_CMD_FLAG_BUSY; - - cmd->callback(cmd, cmd->arg, err); - ixl_vc_process_next(mgr); -} - -static void -ixl_vc_process_resp(struct ixl_vc_mgr *mgr, uint32_t request, - enum i40e_status_code err) -{ - struct ixl_vc_cmd *cmd; - - cmd = mgr->current; - if (cmd == NULL || cmd->request != request) - return; - - callout_stop(&mgr->callout); - ixl_vc_process_completion(mgr, err); -} - -static void -ixl_vc_cmd_timeout(void *arg) -{ - struct ixl_vc_mgr *mgr = (struct ixl_vc_mgr *)arg; - - IXLV_CORE_LOCK_ASSERT(mgr->sc); - ixl_vc_process_completion(mgr, I40E_ERR_TIMEOUT); -} - -static void -ixl_vc_cmd_retry(void *arg) -{ - struct ixl_vc_mgr *mgr = (struct ixl_vc_mgr *)arg; - - IXLV_CORE_LOCK_ASSERT(mgr->sc); - ixl_vc_send_current(mgr); -} - -static void -ixl_vc_send_current(struct ixl_vc_mgr *mgr) -{ - struct ixl_vc_cmd *cmd; - - cmd = mgr->current; - ixl_vc_send_cmd(mgr->sc, cmd->request); - callout_reset(&mgr->callout, IXLV_VC_TIMEOUT, ixl_vc_cmd_timeout, mgr); -} - -static void -ixl_vc_process_next(struct ixl_vc_mgr *mgr) -{ - struct ixl_vc_cmd *cmd; - - if (mgr->current != NULL) - return; - - if (TAILQ_EMPTY(&mgr->pending)) - return; - - cmd = TAILQ_FIRST(&mgr->pending); - TAILQ_REMOVE(&mgr->pending, cmd, next); - - mgr->current = cmd; - ixl_vc_send_current(mgr); -} - -static void -ixl_vc_schedule_retry(struct ixl_vc_mgr *mgr) -{ - - callout_reset(&mgr->callout, howmany(hz, 100), ixl_vc_cmd_retry, mgr); -} - -void -ixl_vc_enqueue(struct ixl_vc_mgr *mgr, struct ixl_vc_cmd *cmd, - uint32_t req, ixl_vc_callback_t *callback, void *arg) -{ - IXLV_CORE_LOCK_ASSERT(mgr->sc); - - if (cmd->flags & IXLV_VC_CMD_FLAG_BUSY) { - if (mgr->current == cmd) - mgr->current = NULL; - else - TAILQ_REMOVE(&mgr->pending, cmd, next); - } - - cmd->request = req; - cmd->callback = callback; - cmd->arg = arg; - cmd->flags |= IXLV_VC_CMD_FLAG_BUSY; - TAILQ_INSERT_TAIL(&mgr->pending, cmd, next); - - ixl_vc_process_next(mgr); -} - -void -ixl_vc_flush(struct ixl_vc_mgr *mgr) -{ - struct ixl_vc_cmd *cmd; - - IXLV_CORE_LOCK_ASSERT(mgr->sc); - KASSERT(TAILQ_EMPTY(&mgr->pending) || mgr->current != NULL, - ("ixlv: pending commands waiting but no command in progress")); - - cmd = mgr->current; - if (cmd != NULL) { - mgr->current = NULL; - cmd->flags &= ~IXLV_VC_CMD_FLAG_BUSY; - cmd->callback(cmd, cmd->arg, I40E_ERR_ADAPTER_STOPPED); - } - - while ((cmd = TAILQ_FIRST(&mgr->pending)) != NULL) { - TAILQ_REMOVE(&mgr->pending, cmd, next); - cmd->flags &= ~IXLV_VC_CMD_FLAG_BUSY; - cmd->callback(cmd, cmd->arg, I40E_ERR_ADAPTER_STOPPED); - } - - callout_stop(&mgr->callout); -} - Index: sys/dev/ixl/legacy_if_ixl.c =================================================================== --- /dev/null +++ sys/dev/ixl/legacy_if_ixl.c @@ -0,0 +1,6721 @@ +/****************************************************************************** + + Copyright (c) 2013-2015, Intel Corporation + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of the Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +******************************************************************************/ +/*$FreeBSD$*/ + +#ifndef IXL_STANDALONE_BUILD +#include "opt_inet.h" +#include "opt_inet6.h" +#include "opt_rss.h" +#endif + +#include "ixl.h" +#include "ixl_pf.h" + +#ifdef RSS +#include +#endif + +/********************************************************************* + * Driver version + *********************************************************************/ +char ixl_driver_version[] = "1.4.3"; + +/********************************************************************* + * PCI Device ID Table + * + * Used by probe to select devices to load on + * Last field stores an index into ixl_strings + * Last entry must be all 0s + * + * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } + *********************************************************************/ + +static ixl_vendor_info_t ixl_vendor_info_array[] = +{ + {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_XL710, 0, 0, 0}, + {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_B, 0, 0, 0}, + {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_C, 0, 0, 0}, + {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_A, 0, 0, 0}, + {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_B, 0, 0, 0}, + {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_C, 0, 0, 0}, + {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T, 0, 0, 0}, + {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T4, 0, 0, 0}, +#ifdef X722_SUPPORT + {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_X722, 0, 0, 0}, + {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_1G_BASE_T_X722, 0, 0, 0}, + {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T_X722, 0, 0, 0}, +#endif + /* required last entry */ + {0, 0, 0, 0, 0} +}; + +/********************************************************************* + * Table of branding strings + *********************************************************************/ + +static char *ixl_strings[] = { + "Intel(R) Ethernet Connection XL710 Driver" +}; + + +/********************************************************************* + * Function prototypes + *********************************************************************/ +static int ixl_probe(device_t); +static int ixl_attach(device_t); +static int ixl_detach(device_t); +static int ixl_shutdown(device_t); +static int ixl_get_hw_capabilities(struct ixl_pf *); +static void ixl_cap_txcsum_tso(struct ixl_vsi *, struct ifnet *, int); +static int ixl_ioctl(struct ifnet *, u_long, caddr_t); +static void ixl_init(void *); +static void ixl_init_locked(struct ixl_pf *); +static void ixl_stop(struct ixl_pf *); +static void ixl_media_status(struct ifnet *, struct ifmediareq *); +static int ixl_media_change(struct ifnet *); +static void ixl_update_link_status(struct ixl_pf *); +static int ixl_allocate_pci_resources(struct ixl_pf *); +static u16 ixl_get_bus_info(struct i40e_hw *, device_t); +static int ixl_setup_stations(struct ixl_pf *); +static int ixl_switch_config(struct ixl_pf *); +static int ixl_initialize_vsi(struct ixl_vsi *); +static int ixl_assign_vsi_msix(struct ixl_pf *); +static int ixl_assign_vsi_legacy(struct ixl_pf *); +static int ixl_init_msix(struct ixl_pf *); +static void ixl_configure_msix(struct ixl_pf *); +static void ixl_configure_itr(struct ixl_pf *); +static void ixl_configure_legacy(struct ixl_pf *); +static void ixl_init_taskqueues(struct ixl_pf *); +static void ixl_free_taskqueues(struct ixl_pf *); +static void ixl_free_pci_resources(struct ixl_pf *); +static void ixl_local_timer(void *); +static int ixl_setup_interface(device_t, struct ixl_vsi *); +static void ixl_link_event(struct ixl_pf *, struct i40e_arq_event_info *); +static void ixl_config_rss(struct ixl_vsi *); +static void ixl_set_queue_rx_itr(struct ixl_queue *); +static void ixl_set_queue_tx_itr(struct ixl_queue *); +static int ixl_set_advertised_speeds(struct ixl_pf *, int); + +static int ixl_enable_rings(struct ixl_vsi *); +static int ixl_disable_rings(struct ixl_vsi *); +static void ixl_enable_intr(struct ixl_vsi *); +static void ixl_disable_intr(struct ixl_vsi *); +static void ixl_disable_rings_intr(struct ixl_vsi *); + +static void ixl_enable_adminq(struct i40e_hw *); +static void ixl_disable_adminq(struct i40e_hw *); +static void ixl_enable_queue(struct i40e_hw *, int); +static void ixl_disable_queue(struct i40e_hw *, int); +static void ixl_enable_legacy(struct i40e_hw *); +static void ixl_disable_legacy(struct i40e_hw *); + +static void ixl_set_promisc(struct ixl_vsi *); +static void ixl_add_multi(struct ixl_vsi *); +static void ixl_del_multi(struct ixl_vsi *); +static void ixl_register_vlan(void *, struct ifnet *, u16); +static void ixl_unregister_vlan(void *, struct ifnet *, u16); +static void ixl_setup_vlan_filters(struct ixl_vsi *); + +static void ixl_init_filters(struct ixl_vsi *); +static void ixl_reconfigure_filters(struct ixl_vsi *vsi); +static void ixl_add_filter(struct ixl_vsi *, u8 *, s16 vlan); +static void ixl_del_filter(struct ixl_vsi *, u8 *, s16 vlan); +static void ixl_add_hw_filters(struct ixl_vsi *, int, int); +static void ixl_del_hw_filters(struct ixl_vsi *, int); +static struct ixl_mac_filter * + ixl_find_filter(struct ixl_vsi *, u8 *, s16); +static void ixl_add_mc_filter(struct ixl_vsi *, u8 *); +static void ixl_free_mac_filters(struct ixl_vsi *vsi); + + +/* Sysctl debug interface */ +#ifdef IXL_DEBUG_SYSCTL +static int ixl_debug_info(SYSCTL_HANDLER_ARGS); +static void ixl_print_debug_info(struct ixl_pf *); +#endif + +/* The MSI/X Interrupt handlers */ +static void ixl_intr(void *); +static void ixl_msix_que(void *); +static void ixl_msix_adminq(void *); +static void ixl_handle_mdd_event(struct ixl_pf *); + +/* Deferred interrupt tasklets */ +static void ixl_do_adminq(void *, int); + +/* Sysctl handlers */ +static int ixl_set_flowcntl(SYSCTL_HANDLER_ARGS); +static int ixl_set_advertise(SYSCTL_HANDLER_ARGS); +static int ixl_current_speed(SYSCTL_HANDLER_ARGS); +static int ixl_sysctl_show_fw(SYSCTL_HANDLER_ARGS); + +/* Statistics */ +static void ixl_add_hw_stats(struct ixl_pf *); +static void ixl_add_sysctls_mac_stats(struct sysctl_ctx_list *, + struct sysctl_oid_list *, struct i40e_hw_port_stats *); +static void ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *, + struct sysctl_oid_list *, + struct i40e_eth_stats *); +static void ixl_update_stats_counters(struct ixl_pf *); +static void ixl_update_eth_stats(struct ixl_vsi *); +static void ixl_update_vsi_stats(struct ixl_vsi *); +static void ixl_pf_reset_stats(struct ixl_pf *); +static void ixl_vsi_reset_stats(struct ixl_vsi *); +static void ixl_stat_update48(struct i40e_hw *, u32, u32, bool, + u64 *, u64 *); +static void ixl_stat_update32(struct i40e_hw *, u32, bool, + u64 *, u64 *); + +#ifdef IXL_DEBUG_SYSCTL +static int ixl_sysctl_link_status(SYSCTL_HANDLER_ARGS); +static int ixl_sysctl_phy_abilities(SYSCTL_HANDLER_ARGS); +static int ixl_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS); +static int ixl_sysctl_hw_res_alloc(SYSCTL_HANDLER_ARGS); +static int ixl_sysctl_switch_config(SYSCTL_HANDLER_ARGS); +#endif + +#ifdef PCI_IOV +static int ixl_adminq_err_to_errno(enum i40e_admin_queue_err err); + +static int ixl_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t*); +static void ixl_iov_uninit(device_t dev); +static int ixl_add_vf(device_t dev, uint16_t vfnum, const nvlist_t*); + +static void ixl_handle_vf_msg(struct ixl_pf *, + struct i40e_arq_event_info *); +static void ixl_handle_vflr(void *arg, int pending); + +static void ixl_reset_vf(struct ixl_pf *pf, struct ixl_vf *vf); +static void ixl_reinit_vf(struct ixl_pf *pf, struct ixl_vf *vf); +#endif + +/********************************************************************* + * FreeBSD Device Interface Entry Points + *********************************************************************/ + +static device_method_t ixl_methods[] = { + /* Device interface */ + DEVMETHOD(device_probe, ixl_probe), + DEVMETHOD(device_attach, ixl_attach), + DEVMETHOD(device_detach, ixl_detach), + DEVMETHOD(device_shutdown, ixl_shutdown), +#ifdef PCI_IOV + DEVMETHOD(pci_iov_init, ixl_iov_init), + DEVMETHOD(pci_iov_uninit, ixl_iov_uninit), + DEVMETHOD(pci_iov_add_vf, ixl_add_vf), +#endif + {0, 0} +}; + +static driver_t ixl_driver = { + "ixl", ixl_methods, sizeof(struct ixl_pf), +}; + +devclass_t ixl_devclass; +DRIVER_MODULE(ixl, pci, ixl_driver, ixl_devclass, 0, 0); + +MODULE_DEPEND(ixl, pci, 1, 1, 1); +MODULE_DEPEND(ixl, ether, 1, 1, 1); +#ifdef DEV_NETMAP +MODULE_DEPEND(ixl, netmap, 1, 1, 1); +#endif /* DEV_NETMAP */ + +/* +** Global reset mutex +*/ +static struct mtx ixl_reset_mtx; + +/* +** TUNEABLE PARAMETERS: +*/ + +static SYSCTL_NODE(_hw, OID_AUTO, ixl, CTLFLAG_RD, 0, + "IXL driver parameters"); + +/* + * MSIX should be the default for best performance, + * but this allows it to be forced off for testing. + */ +static int ixl_enable_msix = 1; +TUNABLE_INT("hw.ixl.enable_msix", &ixl_enable_msix); +SYSCTL_INT(_hw_ixl, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &ixl_enable_msix, 0, + "Enable MSI-X interrupts"); + +/* +** Number of descriptors per ring: +** - TX and RX are the same size +*/ +static int ixl_ringsz = DEFAULT_RING; +TUNABLE_INT("hw.ixl.ringsz", &ixl_ringsz); +SYSCTL_INT(_hw_ixl, OID_AUTO, ring_size, CTLFLAG_RDTUN, + &ixl_ringsz, 0, "Descriptor Ring Size"); + +/* +** This can be set manually, if left as 0 the +** number of queues will be calculated based +** on cpus and msix vectors available. +*/ +int ixl_max_queues = 0; +TUNABLE_INT("hw.ixl.max_queues", &ixl_max_queues); +SYSCTL_INT(_hw_ixl, OID_AUTO, max_queues, CTLFLAG_RDTUN, + &ixl_max_queues, 0, "Number of Queues"); + +/* +** Controls for Interrupt Throttling +** - true/false for dynamic adjustment +** - default values for static ITR +*/ +int ixl_dynamic_rx_itr = 0; +TUNABLE_INT("hw.ixl.dynamic_rx_itr", &ixl_dynamic_rx_itr); +SYSCTL_INT(_hw_ixl, OID_AUTO, dynamic_rx_itr, CTLFLAG_RDTUN, + &ixl_dynamic_rx_itr, 0, "Dynamic RX Interrupt Rate"); + +int ixl_dynamic_tx_itr = 0; +TUNABLE_INT("hw.ixl.dynamic_tx_itr", &ixl_dynamic_tx_itr); +SYSCTL_INT(_hw_ixl, OID_AUTO, dynamic_tx_itr, CTLFLAG_RDTUN, + &ixl_dynamic_tx_itr, 0, "Dynamic TX Interrupt Rate"); + +int ixl_rx_itr = IXL_ITR_8K; +TUNABLE_INT("hw.ixl.rx_itr", &ixl_rx_itr); +SYSCTL_INT(_hw_ixl, OID_AUTO, rx_itr, CTLFLAG_RDTUN, + &ixl_rx_itr, 0, "RX Interrupt Rate"); + +int ixl_tx_itr = IXL_ITR_4K; +TUNABLE_INT("hw.ixl.tx_itr", &ixl_tx_itr); +SYSCTL_INT(_hw_ixl, OID_AUTO, tx_itr, CTLFLAG_RDTUN, + &ixl_tx_itr, 0, "TX Interrupt Rate"); + +#ifdef IXL_FDIR +static int ixl_enable_fdir = 1; +TUNABLE_INT("hw.ixl.enable_fdir", &ixl_enable_fdir); +/* Rate at which we sample */ +int ixl_atr_rate = 20; +TUNABLE_INT("hw.ixl.atr_rate", &ixl_atr_rate); +#endif + +#ifdef DEV_NETMAP +#define NETMAP_IXL_MAIN /* only bring in one part of the netmap code */ +#include +#endif /* DEV_NETMAP */ + +static char *ixl_fc_string[6] = { + "None", + "Rx", + "Tx", + "Full", + "Priority", + "Default" +}; + +MALLOC_DEFINE(M_IXL, "ixl", "ixl driver allocations"); + +static uint8_t ixl_bcast_addr[ETHER_ADDR_LEN] = + {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; + +/********************************************************************* + * Device identification routine + * + * ixl_probe determines if the driver should be loaded on + * the hardware based on PCI vendor/device id of the device. + * + * return BUS_PROBE_DEFAULT on success, positive on failure + *********************************************************************/ + +static int +ixl_probe(device_t dev) +{ + ixl_vendor_info_t *ent; + + u16 pci_vendor_id, pci_device_id; + u16 pci_subvendor_id, pci_subdevice_id; + char device_name[256]; + static bool lock_init = FALSE; + + INIT_DEBUGOUT("ixl_probe: begin"); + + pci_vendor_id = pci_get_vendor(dev); + if (pci_vendor_id != I40E_INTEL_VENDOR_ID) + return (ENXIO); + + pci_device_id = pci_get_device(dev); + pci_subvendor_id = pci_get_subvendor(dev); + pci_subdevice_id = pci_get_subdevice(dev); + + ent = ixl_vendor_info_array; + while (ent->vendor_id != 0) { + if ((pci_vendor_id == ent->vendor_id) && + (pci_device_id == ent->device_id) && + + ((pci_subvendor_id == ent->subvendor_id) || + (ent->subvendor_id == 0)) && + + ((pci_subdevice_id == ent->subdevice_id) || + (ent->subdevice_id == 0))) { + sprintf(device_name, "%s, Version - %s", + ixl_strings[ent->index], + ixl_driver_version); + device_set_desc_copy(dev, device_name); + /* One shot mutex init */ + if (lock_init == FALSE) { + lock_init = TRUE; + mtx_init(&ixl_reset_mtx, + "ixl_reset", + "IXL RESET Lock", MTX_DEF); + } + return (BUS_PROBE_DEFAULT); + } + ent++; + } + return (ENXIO); +} + +/********************************************************************* + * Device initialization routine + * + * The attach entry point is called when the driver is being loaded. + * This routine identifies the type of hardware, allocates all resources + * and initializes the hardware. + * + * return 0 on success, positive on failure + *********************************************************************/ + +static int +ixl_attach(device_t dev) +{ + struct ixl_pf *pf; + struct i40e_hw *hw; + struct ixl_vsi *vsi; + u16 bus; + int error = 0; +#ifdef PCI_IOV + nvlist_t *pf_schema, *vf_schema; + int iov_error; +#endif + + INIT_DEBUGOUT("ixl_attach: begin"); + + /* Allocate, clear, and link in our primary soft structure */ + pf = device_get_softc(dev); + pf->dev = pf->osdep.dev = dev; + hw = &pf->hw; + + /* + ** Note this assumes we have a single embedded VSI, + ** this could be enhanced later to allocate multiple + */ + vsi = &pf->vsi; + vsi->dev = pf->dev; + + /* Core Lock Init*/ + IXL_PF_LOCK_INIT(pf, device_get_nameunit(dev)); + + /* Set up the timer callout */ + callout_init_mtx(&pf->timer, &pf->pf_mtx, 0); + + /* Set up sysctls */ + SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW, + pf, 0, ixl_set_flowcntl, "I", "Flow Control"); + + SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW, + pf, 0, ixl_set_advertise, "I", "Advertised Speed"); + + SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "current_speed", CTLTYPE_STRING | CTLFLAG_RD, + pf, 0, ixl_current_speed, "A", "Current Port Speed"); + + SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "fw_version", CTLTYPE_STRING | CTLFLAG_RD, + pf, 0, ixl_sysctl_show_fw, "A", "Firmware version"); + + SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "rx_itr", CTLFLAG_RW, + &ixl_rx_itr, IXL_ITR_8K, "RX ITR"); + + SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "dynamic_rx_itr", CTLFLAG_RW, + &ixl_dynamic_rx_itr, 0, "Dynamic RX ITR"); + + SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "tx_itr", CTLFLAG_RW, + &ixl_tx_itr, IXL_ITR_4K, "TX ITR"); + + SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "dynamic_tx_itr", CTLFLAG_RW, + &ixl_dynamic_tx_itr, 0, "Dynamic TX ITR"); + +#ifdef IXL_DEBUG_SYSCTL + SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, pf, 0, + ixl_debug_info, "I", "Debug Information"); + + /* Debug shared-code message level */ + SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "debug_mask", CTLFLAG_RW, + &pf->hw.debug_mask, 0, "Debug Message Level"); + + SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "vc_debug_level", CTLFLAG_RW, &pf->vc_debug_lvl, + 0, "PF/VF Virtual Channel debug level"); + + SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "link_status", CTLTYPE_STRING | CTLFLAG_RD, + pf, 0, ixl_sysctl_link_status, "A", "Current Link Status"); + + SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "phy_abilities", CTLTYPE_STRING | CTLFLAG_RD, + pf, 0, ixl_sysctl_phy_abilities, "A", "PHY Abilities"); + + SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "filter_list", CTLTYPE_STRING | CTLFLAG_RD, + pf, 0, ixl_sysctl_sw_filter_list, "A", "SW Filter List"); + + SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "hw_res_alloc", CTLTYPE_STRING | CTLFLAG_RD, + pf, 0, ixl_sysctl_hw_res_alloc, "A", "HW Resource Allocation"); + + SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "switch_config", CTLTYPE_STRING | CTLFLAG_RD, + pf, 0, ixl_sysctl_switch_config, "A", "HW Switch Configuration"); +#endif + + /* Save off the PCI information */ + hw->vendor_id = pci_get_vendor(dev); + hw->device_id = pci_get_device(dev); + hw->revision_id = pci_read_config(dev, PCIR_REVID, 1); + hw->subsystem_vendor_id = + pci_read_config(dev, PCIR_SUBVEND_0, 2); + hw->subsystem_device_id = + pci_read_config(dev, PCIR_SUBDEV_0, 2); + + hw->bus.device = pci_get_slot(dev); + hw->bus.func = pci_get_function(dev); + + pf->vc_debug_lvl = 1; + + /* Do PCI setup - map BAR0, etc */ + if (ixl_allocate_pci_resources(pf)) { + device_printf(dev, "Allocation of PCI resources failed\n"); + error = ENXIO; + goto err_out; + } + + /* Establish a clean starting point */ + i40e_clear_hw(hw); + error = i40e_pf_reset(hw); + if (error) { + device_printf(dev,"PF reset failure %x\n", error); + error = EIO; + goto err_out; + } + + /* Set admin queue parameters */ + hw->aq.num_arq_entries = IXL_AQ_LEN; + hw->aq.num_asq_entries = IXL_AQ_LEN; + hw->aq.arq_buf_size = IXL_AQ_BUFSZ; + hw->aq.asq_buf_size = IXL_AQ_BUFSZ; + + /* Initialize the shared code */ + error = i40e_init_shared_code(hw); + if (error) { + device_printf(dev,"Unable to initialize the shared code\n"); + error = EIO; + goto err_out; + } + + /* Set up the admin queue */ + error = i40e_init_adminq(hw); + if (error) { + device_printf(dev, "The driver for the device stopped " + "because the NVM image is newer than expected.\n" + "You must install the most recent version of " + " the network driver.\n"); + goto err_out; + } + device_printf(dev, "%s\n", ixl_fw_version_str(hw)); + + if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR && + hw->aq.api_min_ver > I40E_FW_API_VERSION_MINOR) + device_printf(dev, "The driver for the device detected " + "a newer version of the NVM image than expected.\n" + "Please install the most recent version of the network driver.\n"); + else if (hw->aq.api_maj_ver < I40E_FW_API_VERSION_MAJOR || + hw->aq.api_min_ver < (I40E_FW_API_VERSION_MINOR - 1)) + device_printf(dev, "The driver for the device detected " + "an older version of the NVM image than expected.\n" + "Please update the NVM image.\n"); + + /* Clear PXE mode */ + i40e_clear_pxe_mode(hw); + + /* Get capabilities from the device */ + error = ixl_get_hw_capabilities(pf); + if (error) { + device_printf(dev, "HW capabilities failure!\n"); + goto err_get_cap; + } + + /* Set up host memory cache */ + error = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, + hw->func_caps.num_rx_qp, 0, 0); + if (error) { + device_printf(dev, "init_lan_hmc failed: %d\n", error); + goto err_get_cap; + } + + error = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); + if (error) { + device_printf(dev, "configure_lan_hmc failed: %d\n", error); + goto err_mac_hmc; + } + + /* Disable LLDP from the firmware */ + i40e_aq_stop_lldp(hw, TRUE, NULL); + + i40e_get_mac_addr(hw, hw->mac.addr); + error = i40e_validate_mac_addr(hw->mac.addr); + if (error) { + device_printf(dev, "validate_mac_addr failed: %d\n", error); + goto err_mac_hmc; + } + bcopy(hw->mac.addr, hw->mac.perm_addr, ETHER_ADDR_LEN); + i40e_get_port_mac_addr(hw, hw->mac.port_addr); + + /* Set up VSI and queues */ + if (ixl_setup_stations(pf) != 0) { + device_printf(dev, "setup stations failed!\n"); + error = ENOMEM; + goto err_mac_hmc; + } + + /* Initialize mac filter list for VSI */ + SLIST_INIT(&vsi->ftl); + + /* Set up interrupt routing here */ + if (pf->msix > 1) + error = ixl_assign_vsi_msix(pf); + else + error = ixl_assign_vsi_legacy(pf); + if (error) + goto err_mac_hmc; + + if (((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver < 33)) || + (hw->aq.fw_maj_ver < 4)) { + i40e_msec_delay(75); + error = i40e_aq_set_link_restart_an(hw, TRUE, NULL); + if (error) + device_printf(dev, "link restart failed, aq_err=%d\n", + pf->hw.aq.asq_last_status); + } + + /* Determine link state */ + i40e_aq_get_link_info(hw, TRUE, NULL, NULL); + i40e_get_link_status(hw, &pf->link_up); + + /* Setup OS specific network interface */ + if (ixl_setup_interface(dev, vsi) != 0) { + device_printf(dev, "interface setup failed!\n"); + error = EIO; + goto err_late; + } + + error = ixl_switch_config(pf); + if (error) { + device_printf(dev, "Initial switch config failed: %d\n", error); + goto err_late; + } + + /* Limit phy interrupts to link and modules failure */ + error = i40e_aq_set_phy_int_mask(hw, ~(I40E_AQ_EVENT_LINK_UPDOWN | + I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL); + if (error) + device_printf(dev, "set phy mask failed: %d\n", error); + + /* Get the bus configuration and set the shared code */ + bus = ixl_get_bus_info(hw, dev); + i40e_set_pci_config_data(hw, bus); + + /* Initialize taskqueues */ + ixl_init_taskqueues(pf); + + /* Initialize statistics */ + ixl_pf_reset_stats(pf); + ixl_update_stats_counters(pf); + ixl_add_hw_stats(pf); + + /* Register for VLAN events */ + vsi->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, + ixl_register_vlan, vsi, EVENTHANDLER_PRI_FIRST); + vsi->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, + ixl_unregister_vlan, vsi, EVENTHANDLER_PRI_FIRST); + +#ifdef PCI_IOV + /* SR-IOV is only supported when MSI-X is in use. */ + if (pf->msix > 1) { + pf_schema = pci_iov_schema_alloc_node(); + vf_schema = pci_iov_schema_alloc_node(); + pci_iov_schema_add_unicast_mac(vf_schema, "mac-addr", 0, NULL); + pci_iov_schema_add_bool(vf_schema, "mac-anti-spoof", + IOV_SCHEMA_HASDEFAULT, TRUE); + pci_iov_schema_add_bool(vf_schema, "allow-set-mac", + IOV_SCHEMA_HASDEFAULT, FALSE); + pci_iov_schema_add_bool(vf_schema, "allow-promisc", + IOV_SCHEMA_HASDEFAULT, FALSE); + + iov_error = pci_iov_attach(dev, pf_schema, vf_schema); + if (iov_error != 0) + device_printf(dev, + "Failed to initialize SR-IOV (error=%d)\n", + iov_error); + } +#endif + +#ifdef DEV_NETMAP + ixl_netmap_attach(vsi); +#endif /* DEV_NETMAP */ + INIT_DEBUGOUT("ixl_attach: end"); + return (0); + +err_late: + if (vsi->ifp != NULL) + if_free(vsi->ifp); +err_mac_hmc: + i40e_shutdown_lan_hmc(hw); +err_get_cap: + i40e_shutdown_adminq(hw); +err_out: + ixl_free_pci_resources(pf); + ixl_free_vsi(vsi); + IXL_PF_LOCK_DESTROY(pf); + return (error); +} + +/********************************************************************* + * Device removal routine + * + * The detach entry point is called when the driver is being removed. + * This routine stops the adapter and deallocates all the resources + * that were allocated for driver operation. + * + * return 0 on success, positive on failure + *********************************************************************/ + +static int +ixl_detach(device_t dev) +{ + struct ixl_pf *pf = device_get_softc(dev); + struct i40e_hw *hw = &pf->hw; + struct ixl_vsi *vsi = &pf->vsi; + i40e_status status; +#ifdef PCI_IOV + int error; +#endif + + INIT_DEBUGOUT("ixl_detach: begin"); + + /* Make sure VLANS are not using driver */ + if (vsi->ifp->if_vlantrunk != NULL) { + device_printf(dev,"Vlan in use, detach first\n"); + return (EBUSY); + } + +#ifdef PCI_IOV + error = pci_iov_detach(dev); + if (error != 0) { + device_printf(dev, "SR-IOV in use; detach first.\n"); + return (error); + } +#endif + + ether_ifdetach(vsi->ifp); + if (vsi->ifp->if_drv_flags & IFF_DRV_RUNNING) { + IXL_PF_LOCK(pf); + ixl_stop(pf); + IXL_PF_UNLOCK(pf); + } + + ixl_free_taskqueues(pf); + + /* Shutdown LAN HMC */ + status = i40e_shutdown_lan_hmc(hw); + if (status) + device_printf(dev, + "Shutdown LAN HMC failed with code %d\n", status); + + /* Shutdown admin queue */ + status = i40e_shutdown_adminq(hw); + if (status) + device_printf(dev, + "Shutdown Admin queue failed with code %d\n", status); + + /* Unregister VLAN events */ + if (vsi->vlan_attach != NULL) + EVENTHANDLER_DEREGISTER(vlan_config, vsi->vlan_attach); + if (vsi->vlan_detach != NULL) + EVENTHANDLER_DEREGISTER(vlan_unconfig, vsi->vlan_detach); + + callout_drain(&pf->timer); +#ifdef DEV_NETMAP + netmap_detach(vsi->ifp); +#endif /* DEV_NETMAP */ + ixl_free_pci_resources(pf); + bus_generic_detach(dev); + if_free(vsi->ifp); + ixl_free_vsi(vsi); + IXL_PF_LOCK_DESTROY(pf); + return (0); +} + +/********************************************************************* + * + * Shutdown entry point + * + **********************************************************************/ + +static int +ixl_shutdown(device_t dev) +{ + struct ixl_pf *pf = device_get_softc(dev); + IXL_PF_LOCK(pf); + ixl_stop(pf); + IXL_PF_UNLOCK(pf); + return (0); +} + + +/********************************************************************* + * + * Get the hardware capabilities + * + **********************************************************************/ + +static int +ixl_get_hw_capabilities(struct ixl_pf *pf) +{ + struct i40e_aqc_list_capabilities_element_resp *buf; + struct i40e_hw *hw = &pf->hw; + device_t dev = pf->dev; + int error, len; + u16 needed; + bool again = TRUE; + + len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp); +retry: + if (!(buf = (struct i40e_aqc_list_capabilities_element_resp *) + malloc(len, M_DEVBUF, M_NOWAIT | M_ZERO))) { + device_printf(dev, "Unable to allocate cap memory\n"); + return (ENOMEM); + } + + /* This populates the hw struct */ + error = i40e_aq_discover_capabilities(hw, buf, len, + &needed, i40e_aqc_opc_list_func_capabilities, NULL); + free(buf, M_DEVBUF); + if ((pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) && + (again == TRUE)) { + /* retry once with a larger buffer */ + again = FALSE; + len = needed; + goto retry; + } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) { + device_printf(dev, "capability discovery failed: %d\n", + pf->hw.aq.asq_last_status); + return (ENODEV); + } + + /* Capture this PF's starting queue pair */ + pf->qbase = hw->func_caps.base_queue; + +#ifdef IXL_DEBUG + device_printf(dev,"pf_id=%d, num_vfs=%d, msix_pf=%d, " + "msix_vf=%d, fd_g=%d, fd_b=%d, tx_qp=%d rx_qp=%d qbase=%d\n", + hw->pf_id, hw->func_caps.num_vfs, + hw->func_caps.num_msix_vectors, + hw->func_caps.num_msix_vectors_vf, + hw->func_caps.fd_filters_guaranteed, + hw->func_caps.fd_filters_best_effort, + hw->func_caps.num_tx_qp, + hw->func_caps.num_rx_qp, + hw->func_caps.base_queue); +#endif + return (error); +} + +static void +ixl_cap_txcsum_tso(struct ixl_vsi *vsi, struct ifnet *ifp, int mask) +{ + device_t dev = vsi->dev; + + /* Enable/disable TXCSUM/TSO4 */ + if (!(ifp->if_capenable & IFCAP_TXCSUM) + && !(ifp->if_capenable & IFCAP_TSO4)) { + if (mask & IFCAP_TXCSUM) { + ifp->if_capenable |= IFCAP_TXCSUM; + /* enable TXCSUM, restore TSO if previously enabled */ + if (vsi->flags & IXL_FLAGS_KEEP_TSO4) { + vsi->flags &= ~IXL_FLAGS_KEEP_TSO4; + ifp->if_capenable |= IFCAP_TSO4; + } + } + else if (mask & IFCAP_TSO4) { + ifp->if_capenable |= (IFCAP_TXCSUM | IFCAP_TSO4); + vsi->flags &= ~IXL_FLAGS_KEEP_TSO4; + device_printf(dev, + "TSO4 requires txcsum, enabling both...\n"); + } + } else if((ifp->if_capenable & IFCAP_TXCSUM) + && !(ifp->if_capenable & IFCAP_TSO4)) { + if (mask & IFCAP_TXCSUM) + ifp->if_capenable &= ~IFCAP_TXCSUM; + else if (mask & IFCAP_TSO4) + ifp->if_capenable |= IFCAP_TSO4; + } else if((ifp->if_capenable & IFCAP_TXCSUM) + && (ifp->if_capenable & IFCAP_TSO4)) { + if (mask & IFCAP_TXCSUM) { + vsi->flags |= IXL_FLAGS_KEEP_TSO4; + ifp->if_capenable &= ~(IFCAP_TXCSUM | IFCAP_TSO4); + device_printf(dev, + "TSO4 requires txcsum, disabling both...\n"); + } else if (mask & IFCAP_TSO4) + ifp->if_capenable &= ~IFCAP_TSO4; + } + + /* Enable/disable TXCSUM_IPV6/TSO6 */ + if (!(ifp->if_capenable & IFCAP_TXCSUM_IPV6) + && !(ifp->if_capenable & IFCAP_TSO6)) { + if (mask & IFCAP_TXCSUM_IPV6) { + ifp->if_capenable |= IFCAP_TXCSUM_IPV6; + if (vsi->flags & IXL_FLAGS_KEEP_TSO6) { + vsi->flags &= ~IXL_FLAGS_KEEP_TSO6; + ifp->if_capenable |= IFCAP_TSO6; + } + } else if (mask & IFCAP_TSO6) { + ifp->if_capenable |= (IFCAP_TXCSUM_IPV6 | IFCAP_TSO6); + vsi->flags &= ~IXL_FLAGS_KEEP_TSO6; + device_printf(dev, + "TSO6 requires txcsum6, enabling both...\n"); + } + } else if((ifp->if_capenable & IFCAP_TXCSUM_IPV6) + && !(ifp->if_capenable & IFCAP_TSO6)) { + if (mask & IFCAP_TXCSUM_IPV6) + ifp->if_capenable &= ~IFCAP_TXCSUM_IPV6; + else if (mask & IFCAP_TSO6) + ifp->if_capenable |= IFCAP_TSO6; + } else if ((ifp->if_capenable & IFCAP_TXCSUM_IPV6) + && (ifp->if_capenable & IFCAP_TSO6)) { + if (mask & IFCAP_TXCSUM_IPV6) { + vsi->flags |= IXL_FLAGS_KEEP_TSO6; + ifp->if_capenable &= ~(IFCAP_TXCSUM_IPV6 | IFCAP_TSO6); + device_printf(dev, + "TSO6 requires txcsum6, disabling both...\n"); + } else if (mask & IFCAP_TSO6) + ifp->if_capenable &= ~IFCAP_TSO6; + } +} + +/********************************************************************* + * Ioctl entry point + * + * ixl_ioctl is called when the user wants to configure the + * interface. + * + * return 0 on success, positive on failure + **********************************************************************/ + +static int +ixl_ioctl(struct ifnet * ifp, u_long command, caddr_t data) +{ + struct ixl_vsi *vsi = ifp->if_softc; + struct ixl_pf *pf = vsi->back; + struct ifreq *ifr = (struct ifreq *) data; +#if defined(INET) || defined(INET6) + struct ifaddr *ifa = (struct ifaddr *)data; + bool avoid_reset = FALSE; +#endif + int error = 0; + + switch (command) { + + case SIOCSIFADDR: +#ifdef INET + if (ifa->ifa_addr->sa_family == AF_INET) + avoid_reset = TRUE; +#endif +#ifdef INET6 + if (ifa->ifa_addr->sa_family == AF_INET6) + avoid_reset = TRUE; +#endif +#if defined(INET) || defined(INET6) + /* + ** Calling init results in link renegotiation, + ** so we avoid doing it when possible. + */ + if (avoid_reset) { + ifp->if_flags |= IFF_UP; + if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) + ixl_init(pf); +#ifdef INET + if (!(ifp->if_flags & IFF_NOARP)) + arp_ifinit(ifp, ifa); +#endif + } else + error = ether_ioctl(ifp, command, data); + break; +#endif + case SIOCSIFMTU: + IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); + if (ifr->ifr_mtu > IXL_MAX_FRAME - + ETHER_HDR_LEN - ETHER_CRC_LEN - ETHER_VLAN_ENCAP_LEN) { + error = EINVAL; + } else { + IXL_PF_LOCK(pf); + ifp->if_mtu = ifr->ifr_mtu; + vsi->max_frame_size = + ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN + + ETHER_VLAN_ENCAP_LEN; + ixl_init_locked(pf); + IXL_PF_UNLOCK(pf); + } + break; + case SIOCSIFFLAGS: + IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); + IXL_PF_LOCK(pf); + if (ifp->if_flags & IFF_UP) { + if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) { + if ((ifp->if_flags ^ pf->if_flags) & + (IFF_PROMISC | IFF_ALLMULTI)) { + ixl_set_promisc(vsi); + } + } else + ixl_init_locked(pf); + } else + if (ifp->if_drv_flags & IFF_DRV_RUNNING) + ixl_stop(pf); + pf->if_flags = ifp->if_flags; + IXL_PF_UNLOCK(pf); + break; + case SIOCADDMULTI: + IOCTL_DEBUGOUT("ioctl: SIOCADDMULTI"); + if (ifp->if_drv_flags & IFF_DRV_RUNNING) { + IXL_PF_LOCK(pf); + ixl_disable_intr(vsi); + ixl_add_multi(vsi); + ixl_enable_intr(vsi); + IXL_PF_UNLOCK(pf); + } + break; + case SIOCDELMULTI: + IOCTL_DEBUGOUT("ioctl: SIOCDELMULTI"); + if (ifp->if_drv_flags & IFF_DRV_RUNNING) { + IXL_PF_LOCK(pf); + ixl_disable_intr(vsi); + ixl_del_multi(vsi); + ixl_enable_intr(vsi); + IXL_PF_UNLOCK(pf); + } + break; + case SIOCSIFMEDIA: + case SIOCGIFMEDIA: +#ifdef IFM_ETH_XTYPE + case SIOCGIFXMEDIA: +#endif + IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); + error = ifmedia_ioctl(ifp, ifr, &vsi->media, command); + break; + case SIOCSIFCAP: + { + int mask = ifr->ifr_reqcap ^ ifp->if_capenable; + IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); + + ixl_cap_txcsum_tso(vsi, ifp, mask); + + if (mask & IFCAP_RXCSUM) + ifp->if_capenable ^= IFCAP_RXCSUM; + if (mask & IFCAP_RXCSUM_IPV6) + ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; + if (mask & IFCAP_LRO) + ifp->if_capenable ^= IFCAP_LRO; + if (mask & IFCAP_VLAN_HWTAGGING) + ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; + if (mask & IFCAP_VLAN_HWFILTER) + ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; + if (mask & IFCAP_VLAN_HWTSO) + ifp->if_capenable ^= IFCAP_VLAN_HWTSO; + if (ifp->if_drv_flags & IFF_DRV_RUNNING) { + IXL_PF_LOCK(pf); + ixl_init_locked(pf); + IXL_PF_UNLOCK(pf); + } + VLAN_CAPABILITIES(ifp); + + break; + } + + default: + IOCTL_DEBUGOUT("ioctl: UNKNOWN (0x%X)\n", (int)command); + error = ether_ioctl(ifp, command, data); + break; + } + + return (error); +} + + +/********************************************************************* + * Init entry point + * + * This routine is used in two ways. It is used by the stack as + * init entry point in network interface structure. It is also used + * by the driver as a hw/sw initialization routine to get to a + * consistent state. + * + * return 0 on success, positive on failure + **********************************************************************/ + +static void +ixl_init_locked(struct ixl_pf *pf) +{ + struct i40e_hw *hw = &pf->hw; + struct ixl_vsi *vsi = &pf->vsi; + struct ifnet *ifp = vsi->ifp; + device_t dev = pf->dev; + struct i40e_filter_control_settings filter; + u8 tmpaddr[ETHER_ADDR_LEN]; + int ret; + + mtx_assert(&pf->pf_mtx, MA_OWNED); + INIT_DEBUGOUT("ixl_init: begin"); + ixl_stop(pf); + + /* Get the latest mac address... User might use a LAA */ + bcopy(IF_LLADDR(vsi->ifp), tmpaddr, + I40E_ETH_LENGTH_OF_ADDRESS); + if (!cmp_etheraddr(hw->mac.addr, tmpaddr) && + (i40e_validate_mac_addr(tmpaddr) == I40E_SUCCESS)) { + ixl_del_filter(vsi, hw->mac.addr, IXL_VLAN_ANY); + bcopy(tmpaddr, hw->mac.addr, + I40E_ETH_LENGTH_OF_ADDRESS); + ret = i40e_aq_mac_address_write(hw, + I40E_AQC_WRITE_TYPE_LAA_ONLY, + hw->mac.addr, NULL); + if (ret) { + device_printf(dev, "LLA address" + "change failed!!\n"); + return; + } else { + ixl_add_filter(vsi, hw->mac.addr, IXL_VLAN_ANY); + } + } + + /* Set the various hardware offload abilities */ + ifp->if_hwassist = 0; + if (ifp->if_capenable & IFCAP_TSO) + ifp->if_hwassist |= CSUM_TSO; + if (ifp->if_capenable & IFCAP_TXCSUM) + ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); + if (ifp->if_capenable & IFCAP_TXCSUM_IPV6) + ifp->if_hwassist |= (CSUM_TCP_IPV6 | CSUM_UDP_IPV6); + + /* Set up the device filtering */ + bzero(&filter, sizeof(filter)); + filter.enable_ethtype = TRUE; + filter.enable_macvlan = TRUE; +#ifdef IXL_FDIR + filter.enable_fdir = TRUE; +#endif + filter.hash_lut_size = I40E_HASH_LUT_SIZE_512; + if (i40e_set_filter_control(hw, &filter)) + device_printf(dev, "set_filter_control() failed\n"); + + /* Set up RSS */ + ixl_config_rss(vsi); + + /* + ** Prepare the VSI: rings, hmc contexts, etc... + */ + if (ixl_initialize_vsi(vsi)) { + device_printf(dev, "initialize vsi failed!!\n"); + return; + } + + /* Add protocol filters to list */ + ixl_init_filters(vsi); + + /* Setup vlan's if needed */ + ixl_setup_vlan_filters(vsi); + + /* Start the local timer */ + callout_reset(&pf->timer, hz, ixl_local_timer, pf); + + /* Set up MSI/X routing and the ITR settings */ + if (ixl_enable_msix) { + ixl_configure_msix(pf); + ixl_configure_itr(pf); + } else + ixl_configure_legacy(pf); + + ixl_enable_rings(vsi); + + i40e_aq_set_default_vsi(hw, vsi->seid, NULL); + + ixl_reconfigure_filters(vsi); + + /* Set MTU in hardware*/ + int aq_error = i40e_aq_set_mac_config(hw, vsi->max_frame_size, + TRUE, 0, NULL); + if (aq_error) + device_printf(vsi->dev, + "aq_set_mac_config in init error, code %d\n", + aq_error); + + /* And now turn on interrupts */ + ixl_enable_intr(vsi); + + /* Now inform the stack we're ready */ + ifp->if_drv_flags |= IFF_DRV_RUNNING; + ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; + + return; +} + +static void +ixl_init(void *arg) +{ + struct ixl_pf *pf = arg; + + IXL_PF_LOCK(pf); + ixl_init_locked(pf); + IXL_PF_UNLOCK(pf); + return; +} + +/* +** +** MSIX Interrupt Handlers and Tasklets +** +*/ +static void +ixl_handle_que(void *context, int pending) +{ + struct ixl_queue *que = context; + struct ixl_vsi *vsi = que->vsi; + struct i40e_hw *hw = vsi->hw; + struct tx_ring *txr = &que->txr; + struct ifnet *ifp = vsi->ifp; + bool more; + + if (ifp->if_drv_flags & IFF_DRV_RUNNING) { + more = ixl_rxeof(que, IXL_RX_LIMIT); + IXL_TX_LOCK(txr); + ixl_txeof(que); + if (!drbr_empty(ifp, txr->br)) + ixl_mq_start_locked(ifp, txr); + IXL_TX_UNLOCK(txr); + if (more) { + taskqueue_enqueue(que->tq, &que->task); + return; + } + } + + /* Reenable this interrupt - hmmm */ + ixl_enable_queue(hw, que->me); + return; +} + + +/********************************************************************* + * + * Legacy Interrupt Service routine + * + **********************************************************************/ +void +ixl_intr(void *arg) +{ + struct ixl_pf *pf = arg; + struct i40e_hw *hw = &pf->hw; + struct ixl_vsi *vsi = &pf->vsi; + struct ixl_queue *que = vsi->queues; + struct ifnet *ifp = vsi->ifp; + struct tx_ring *txr = &que->txr; + u32 reg, icr0, mask; + bool more_tx, more_rx; + + ++que->irqs; + + /* Protect against spurious interrupts */ + if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) + return; + + icr0 = rd32(hw, I40E_PFINT_ICR0); + + reg = rd32(hw, I40E_PFINT_DYN_CTL0); + reg = reg | I40E_PFINT_DYN_CTL0_CLEARPBA_MASK; + wr32(hw, I40E_PFINT_DYN_CTL0, reg); + + mask = rd32(hw, I40E_PFINT_ICR0_ENA); + +#ifdef PCI_IOV + if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) + taskqueue_enqueue(pf->tq, &pf->vflr_task); +#endif + + if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) { + taskqueue_enqueue(pf->tq, &pf->adminq); + return; + } + + more_rx = ixl_rxeof(que, IXL_RX_LIMIT); + + IXL_TX_LOCK(txr); + more_tx = ixl_txeof(que); + if (!drbr_empty(vsi->ifp, txr->br)) + more_tx = 1; + IXL_TX_UNLOCK(txr); + + /* re-enable other interrupt causes */ + wr32(hw, I40E_PFINT_ICR0_ENA, mask); + + /* And now the queues */ + reg = rd32(hw, I40E_QINT_RQCTL(0)); + reg |= I40E_QINT_RQCTL_CAUSE_ENA_MASK; + wr32(hw, I40E_QINT_RQCTL(0), reg); + + reg = rd32(hw, I40E_QINT_TQCTL(0)); + reg |= I40E_QINT_TQCTL_CAUSE_ENA_MASK; + reg &= ~I40E_PFINT_ICR0_INTEVENT_MASK; + wr32(hw, I40E_QINT_TQCTL(0), reg); + + ixl_enable_legacy(hw); + + return; +} + + +/********************************************************************* + * + * MSIX VSI Interrupt Service routine + * + **********************************************************************/ +void +ixl_msix_que(void *arg) +{ + struct ixl_queue *que = arg; + struct ixl_vsi *vsi = que->vsi; + struct i40e_hw *hw = vsi->hw; + struct tx_ring *txr = &que->txr; + bool more_tx, more_rx; + + /* Protect against spurious interrupts */ + if (!(vsi->ifp->if_drv_flags & IFF_DRV_RUNNING)) + return; + + ++que->irqs; + + more_rx = ixl_rxeof(que, IXL_RX_LIMIT); + + IXL_TX_LOCK(txr); + more_tx = ixl_txeof(que); + /* + ** Make certain that if the stack + ** has anything queued the task gets + ** scheduled to handle it. + */ + if (!drbr_empty(vsi->ifp, txr->br)) + more_tx = 1; + IXL_TX_UNLOCK(txr); + + ixl_set_queue_rx_itr(que); + ixl_set_queue_tx_itr(que); + + if (more_tx || more_rx) + taskqueue_enqueue(que->tq, &que->task); + else + ixl_enable_queue(hw, que->me); + + return; +} + + +/********************************************************************* + * + * MSIX Admin Queue Interrupt Service routine + * + **********************************************************************/ +static void +ixl_msix_adminq(void *arg) +{ + struct ixl_pf *pf = arg; + struct i40e_hw *hw = &pf->hw; + u32 reg, mask; + + ++pf->admin_irq; + + reg = rd32(hw, I40E_PFINT_ICR0); + mask = rd32(hw, I40E_PFINT_ICR0_ENA); + + /* Check on the cause */ + if (reg & I40E_PFINT_ICR0_ADMINQ_MASK) + mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK; + + if (reg & I40E_PFINT_ICR0_MAL_DETECT_MASK) { + ixl_handle_mdd_event(pf); + mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; + } + +#ifdef PCI_IOV + if (reg & I40E_PFINT_ICR0_VFLR_MASK) { + mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK; + taskqueue_enqueue(pf->tq, &pf->vflr_task); + } +#endif + + reg = rd32(hw, I40E_PFINT_DYN_CTL0); + reg = reg | I40E_PFINT_DYN_CTL0_CLEARPBA_MASK; + wr32(hw, I40E_PFINT_DYN_CTL0, reg); + + taskqueue_enqueue(pf->tq, &pf->adminq); + return; +} + +/********************************************************************* + * + * Media Ioctl callback + * + * This routine is called whenever the user queries the status of + * the interface using ifconfig. + * + **********************************************************************/ +static void +ixl_media_status(struct ifnet * ifp, struct ifmediareq * ifmr) +{ + struct ixl_vsi *vsi = ifp->if_softc; + struct ixl_pf *pf = vsi->back; + struct i40e_hw *hw = &pf->hw; + + INIT_DEBUGOUT("ixl_media_status: begin"); + IXL_PF_LOCK(pf); + + hw->phy.get_link_info = TRUE; + i40e_get_link_status(hw, &pf->link_up); + ixl_update_link_status(pf); + + ifmr->ifm_status = IFM_AVALID; + ifmr->ifm_active = IFM_ETHER; + + if (!pf->link_up) { + IXL_PF_UNLOCK(pf); + return; + } + + ifmr->ifm_status |= IFM_ACTIVE; + /* Hardware is always full-duplex */ + ifmr->ifm_active |= IFM_FDX; + + switch (hw->phy.link_info.phy_type) { + /* 100 M */ + case I40E_PHY_TYPE_100BASE_TX: + ifmr->ifm_active |= IFM_100_TX; + break; + /* 1 G */ + case I40E_PHY_TYPE_1000BASE_T: + ifmr->ifm_active |= IFM_1000_T; + break; + case I40E_PHY_TYPE_1000BASE_SX: + ifmr->ifm_active |= IFM_1000_SX; + break; + case I40E_PHY_TYPE_1000BASE_LX: + ifmr->ifm_active |= IFM_1000_LX; + break; + /* 10 G */ + case I40E_PHY_TYPE_10GBASE_SFPP_CU: + ifmr->ifm_active |= IFM_10G_TWINAX; + break; + case I40E_PHY_TYPE_10GBASE_SR: + ifmr->ifm_active |= IFM_10G_SR; + break; + case I40E_PHY_TYPE_10GBASE_LR: + ifmr->ifm_active |= IFM_10G_LR; + break; + case I40E_PHY_TYPE_10GBASE_T: + ifmr->ifm_active |= IFM_10G_T; + break; + /* 40 G */ + case I40E_PHY_TYPE_40GBASE_CR4: + case I40E_PHY_TYPE_40GBASE_CR4_CU: + ifmr->ifm_active |= IFM_40G_CR4; + break; + case I40E_PHY_TYPE_40GBASE_SR4: + ifmr->ifm_active |= IFM_40G_SR4; + break; + case I40E_PHY_TYPE_40GBASE_LR4: + ifmr->ifm_active |= IFM_40G_LR4; + break; +#ifndef IFM_ETH_XTYPE + case I40E_PHY_TYPE_1000BASE_KX: + ifmr->ifm_active |= IFM_1000_CX; + break; + case I40E_PHY_TYPE_10GBASE_CR1_CU: + case I40E_PHY_TYPE_10GBASE_CR1: + ifmr->ifm_active |= IFM_10G_TWINAX; + break; + case I40E_PHY_TYPE_10GBASE_KX4: + ifmr->ifm_active |= IFM_10G_CX4; + break; + case I40E_PHY_TYPE_10GBASE_KR: + ifmr->ifm_active |= IFM_10G_SR; + break; + case I40E_PHY_TYPE_40GBASE_KR4: + case I40E_PHY_TYPE_XLPPI: + ifmr->ifm_active |= IFM_40G_SR4; + break; +#else + case I40E_PHY_TYPE_1000BASE_KX: + ifmr->ifm_active |= IFM_1000_KX; + break; + /* ERJ: What's the difference between these? */ + case I40E_PHY_TYPE_10GBASE_CR1_CU: + case I40E_PHY_TYPE_10GBASE_CR1: + ifmr->ifm_active |= IFM_10G_CR1; + break; + case I40E_PHY_TYPE_10GBASE_KX4: + ifmr->ifm_active |= IFM_10G_KX4; + break; + case I40E_PHY_TYPE_10GBASE_KR: + ifmr->ifm_active |= IFM_10G_KR; + break; + case I40E_PHY_TYPE_20GBASE_KR2: + ifmr->ifm_active |= IFM_20G_KR2; + break; + case I40E_PHY_TYPE_40GBASE_KR4: + ifmr->ifm_active |= IFM_40G_KR4; + break; + case I40E_PHY_TYPE_XLPPI: + ifmr->ifm_active |= IFM_40G_XLPPI; + break; +#endif + default: + ifmr->ifm_active |= IFM_UNKNOWN; + break; + } + /* Report flow control status as well */ + if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_TX) + ifmr->ifm_active |= IFM_ETH_TXPAUSE; + if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_RX) + ifmr->ifm_active |= IFM_ETH_RXPAUSE; + + IXL_PF_UNLOCK(pf); + + return; +} + +/********************************************************************* + * + * Media Ioctl callback + * + * This routine is called when the user changes speed/duplex using + * media/mediopt option with ifconfig. + * + **********************************************************************/ +static int +ixl_media_change(struct ifnet * ifp) +{ + struct ixl_vsi *vsi = ifp->if_softc; + struct ifmedia *ifm = &vsi->media; + + INIT_DEBUGOUT("ixl_media_change: begin"); + + if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) + return (EINVAL); + + if_printf(ifp, "Media change is currently not supported.\n"); + + return (ENODEV); +} + + +#ifdef IXL_FDIR +/* +** ATR: Application Targetted Receive - creates a filter +** based on TX flow info that will keep the receive +** portion of the flow on the same queue. Based on the +** implementation this is only available for TCP connections +*/ +void +ixl_atr(struct ixl_queue *que, struct tcphdr *th, int etype) +{ + struct ixl_vsi *vsi = que->vsi; + struct tx_ring *txr = &que->txr; + struct i40e_filter_program_desc *FDIR; + u32 ptype, dtype; + int idx; + + /* check if ATR is enabled and sample rate */ + if ((!ixl_enable_fdir) || (!txr->atr_rate)) + return; + /* + ** We sample all TCP SYN/FIN packets, + ** or at the selected sample rate + */ + txr->atr_count++; + if (((th->th_flags & (TH_FIN | TH_SYN)) == 0) && + (txr->atr_count < txr->atr_rate)) + return; + txr->atr_count = 0; + + /* Get a descriptor to use */ + idx = txr->next_avail; + FDIR = (struct i40e_filter_program_desc *) &txr->base[idx]; + if (++idx == que->num_desc) + idx = 0; + txr->avail--; + txr->next_avail = idx; + + ptype = (que->me << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) & + I40E_TXD_FLTR_QW0_QINDEX_MASK; + + ptype |= (etype == ETHERTYPE_IP) ? + (I40E_FILTER_PCTYPE_NONF_IPV4_TCP << + I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) : + (I40E_FILTER_PCTYPE_NONF_IPV6_TCP << + I40E_TXD_FLTR_QW0_PCTYPE_SHIFT); + + ptype |= vsi->id << I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT; + + dtype = I40E_TX_DESC_DTYPE_FILTER_PROG; + + /* + ** We use the TCP TH_FIN as a trigger to remove + ** the filter, otherwise its an update. + */ + dtype |= (th->th_flags & TH_FIN) ? + (I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE << + I40E_TXD_FLTR_QW1_PCMD_SHIFT) : + (I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE << + I40E_TXD_FLTR_QW1_PCMD_SHIFT); + + dtype |= I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX << + I40E_TXD_FLTR_QW1_DEST_SHIFT; + + dtype |= I40E_FILTER_PROGRAM_DESC_FD_STATUS_FD_ID << + I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT; + + FDIR->qindex_flex_ptype_vsi = htole32(ptype); + FDIR->dtype_cmd_cntindex = htole32(dtype); + return; +} +#endif + + +static void +ixl_set_promisc(struct ixl_vsi *vsi) +{ + struct ifnet *ifp = vsi->ifp; + struct i40e_hw *hw = vsi->hw; + int err, mcnt = 0; + bool uni = FALSE, multi = FALSE; + + if (ifp->if_flags & IFF_ALLMULTI) + multi = TRUE; + else { /* Need to count the multicast addresses */ + struct ifmultiaddr *ifma; + if_maddr_rlock(ifp); + TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + if (ifma->ifma_addr->sa_family != AF_LINK) + continue; + if (mcnt == MAX_MULTICAST_ADDR) + break; + mcnt++; + } + if_maddr_runlock(ifp); + } + + if (mcnt >= MAX_MULTICAST_ADDR) + multi = TRUE; + if (ifp->if_flags & IFF_PROMISC) + uni = TRUE; + + err = i40e_aq_set_vsi_unicast_promiscuous(hw, + vsi->seid, uni, NULL); + err = i40e_aq_set_vsi_multicast_promiscuous(hw, + vsi->seid, multi, NULL); + return; +} + +/********************************************************************* + * Filter Routines + * + * Routines for multicast and vlan filter management. + * + *********************************************************************/ +static void +ixl_add_multi(struct ixl_vsi *vsi) +{ + struct ifmultiaddr *ifma; + struct ifnet *ifp = vsi->ifp; + struct i40e_hw *hw = vsi->hw; + int mcnt = 0, flags; + + IOCTL_DEBUGOUT("ixl_add_multi: begin"); + + if_maddr_rlock(ifp); + /* + ** First just get a count, to decide if we + ** we simply use multicast promiscuous. + */ + TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + if (ifma->ifma_addr->sa_family != AF_LINK) + continue; + mcnt++; + } + if_maddr_runlock(ifp); + + if (__predict_false(mcnt >= MAX_MULTICAST_ADDR)) { + /* delete existing MC filters */ + ixl_del_hw_filters(vsi, mcnt); + i40e_aq_set_vsi_multicast_promiscuous(hw, + vsi->seid, TRUE, NULL); + return; + } + + mcnt = 0; + if_maddr_rlock(ifp); + TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + if (ifma->ifma_addr->sa_family != AF_LINK) + continue; + ixl_add_mc_filter(vsi, + (u8*)LLADDR((struct sockaddr_dl *) ifma->ifma_addr)); + mcnt++; + } + if_maddr_runlock(ifp); + if (mcnt > 0) { + flags = (IXL_FILTER_ADD | IXL_FILTER_USED | IXL_FILTER_MC); + ixl_add_hw_filters(vsi, flags, mcnt); + } + + IOCTL_DEBUGOUT("ixl_add_multi: end"); + return; +} + +static void +ixl_del_multi(struct ixl_vsi *vsi) +{ + struct ifnet *ifp = vsi->ifp; + struct ifmultiaddr *ifma; + struct ixl_mac_filter *f; + int mcnt = 0; + bool match = FALSE; + + IOCTL_DEBUGOUT("ixl_del_multi: begin"); + + /* Search for removed multicast addresses */ + if_maddr_rlock(ifp); + SLIST_FOREACH(f, &vsi->ftl, next) { + if ((f->flags & IXL_FILTER_USED) && (f->flags & IXL_FILTER_MC)) { + match = FALSE; + TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + if (ifma->ifma_addr->sa_family != AF_LINK) + continue; + u8 *mc_addr = (u8 *)LLADDR((struct sockaddr_dl *)ifma->ifma_addr); + if (cmp_etheraddr(f->macaddr, mc_addr)) { + match = TRUE; + break; + } + } + if (match == FALSE) { + f->flags |= IXL_FILTER_DEL; + mcnt++; + } + } + } + if_maddr_runlock(ifp); + + if (mcnt > 0) + ixl_del_hw_filters(vsi, mcnt); +} + + +/********************************************************************* + * Timer routine + * + * This routine checks for link status,updates statistics, + * and runs the watchdog check. + * + **********************************************************************/ + +static void +ixl_local_timer(void *arg) +{ + struct ixl_pf *pf = arg; + struct i40e_hw *hw = &pf->hw; + struct ixl_vsi *vsi = &pf->vsi; + struct ixl_queue *que = vsi->queues; + device_t dev = pf->dev; + int hung = 0; + u32 mask; + + mtx_assert(&pf->pf_mtx, MA_OWNED); + + /* Fire off the adminq task */ + taskqueue_enqueue(pf->tq, &pf->adminq); + + /* Update stats */ + ixl_update_stats_counters(pf); + + /* + ** Check status of the queues + */ + mask = (I40E_PFINT_DYN_CTLN_INTENA_MASK | + I40E_PFINT_DYN_CTLN_SWINT_TRIG_MASK); + + for (int i = 0; i < vsi->num_queues; i++,que++) { + /* Any queues with outstanding work get a sw irq */ + if (que->busy) + wr32(hw, I40E_PFINT_DYN_CTLN(que->me), mask); + /* + ** Each time txeof runs without cleaning, but there + ** are uncleaned descriptors it increments busy. If + ** we get to 5 we declare it hung. + */ + if (que->busy == IXL_QUEUE_HUNG) { + ++hung; + /* Mark the queue as inactive */ + vsi->active_queues &= ~((u64)1 << que->me); + continue; + } else { + /* Check if we've come back from hung */ + if ((vsi->active_queues & ((u64)1 << que->me)) == 0) + vsi->active_queues |= ((u64)1 << que->me); + } + if (que->busy >= IXL_MAX_TX_BUSY) { +#ifdef IXL_DEBUG + device_printf(dev,"Warning queue %d " + "appears to be hung!\n", i); +#endif + que->busy = IXL_QUEUE_HUNG; + ++hung; + } + } + /* Only reinit if all queues show hung */ + if (hung == vsi->num_queues) + goto hung; + + callout_reset(&pf->timer, hz, ixl_local_timer, pf); + return; + +hung: + device_printf(dev, "Local Timer: HANG DETECT - Resetting!!\n"); + ixl_init_locked(pf); +} + +/* +** Note: this routine updates the OS on the link state +** the real check of the hardware only happens with +** a link interrupt. +*/ +static void +ixl_update_link_status(struct ixl_pf *pf) +{ + struct ixl_vsi *vsi = &pf->vsi; + struct i40e_hw *hw = &pf->hw; + struct ifnet *ifp = vsi->ifp; + device_t dev = pf->dev; + + if (pf->link_up){ + if (vsi->link_active == FALSE) { + pf->fc = hw->fc.current_mode; + if (bootverbose) { + device_printf(dev,"Link is up %d Gbps %s," + " Flow Control: %s\n", + ((pf->link_speed == + I40E_LINK_SPEED_40GB)? 40:10), + "Full Duplex", ixl_fc_string[pf->fc]); + } + vsi->link_active = TRUE; + /* + ** Warn user if link speed on NPAR enabled + ** partition is not at least 10GB + */ + if (hw->func_caps.npar_enable && + (hw->phy.link_info.link_speed == + I40E_LINK_SPEED_1GB || + hw->phy.link_info.link_speed == + I40E_LINK_SPEED_100MB)) + device_printf(dev, "The partition detected" + "link speed that is less than 10Gbps\n"); + if_link_state_change(ifp, LINK_STATE_UP); + } + } else { /* Link down */ + if (vsi->link_active == TRUE) { + if (bootverbose) + device_printf(dev,"Link is Down\n"); + if_link_state_change(ifp, LINK_STATE_DOWN); + vsi->link_active = FALSE; + } + } + + return; +} + +/********************************************************************* + * + * This routine disables all traffic on the adapter by issuing a + * global reset on the MAC and deallocates TX/RX buffers. + * + **********************************************************************/ + +static void +ixl_stop(struct ixl_pf *pf) +{ + struct ixl_vsi *vsi = &pf->vsi; + struct ifnet *ifp = vsi->ifp; + + mtx_assert(&pf->pf_mtx, MA_OWNED); + + INIT_DEBUGOUT("ixl_stop: begin\n"); + if (pf->num_vfs == 0) + ixl_disable_intr(vsi); + else + ixl_disable_rings_intr(vsi); + ixl_disable_rings(vsi); + + /* Tell the stack that the interface is no longer active */ + ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); + + /* Stop the local timer */ + callout_stop(&pf->timer); + + return; +} + + +/********************************************************************* + * + * Setup MSIX Interrupt resources and handlers for the VSI + * + **********************************************************************/ +static int +ixl_assign_vsi_legacy(struct ixl_pf *pf) +{ + device_t dev = pf->dev; + struct ixl_vsi *vsi = &pf->vsi; + struct ixl_queue *que = vsi->queues; + int error, rid = 0; + + if (pf->msix == 1) + rid = 1; + pf->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, + &rid, RF_SHAREABLE | RF_ACTIVE); + if (pf->res == NULL) { + device_printf(dev,"Unable to allocate" + " bus resource: vsi legacy/msi interrupt\n"); + return (ENXIO); + } + + /* Set the handler function */ + error = bus_setup_intr(dev, pf->res, + INTR_TYPE_NET | INTR_MPSAFE, NULL, + ixl_intr, pf, &pf->tag); + if (error) { + pf->res = NULL; + device_printf(dev, "Failed to register legacy/msi handler"); + return (error); + } + bus_describe_intr(dev, pf->res, pf->tag, "irq0"); + TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que); + TASK_INIT(&que->task, 0, ixl_handle_que, que); + que->tq = taskqueue_create_fast("ixl_que", M_NOWAIT, + taskqueue_thread_enqueue, &que->tq); + taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que", + device_get_nameunit(dev)); + TASK_INIT(&pf->adminq, 0, ixl_do_adminq, pf); + +#ifdef PCI_IOV + TASK_INIT(&pf->vflr_task, 0, ixl_handle_vflr, pf); +#endif + + pf->tq = taskqueue_create_fast("ixl_adm", M_NOWAIT, + taskqueue_thread_enqueue, &pf->tq); + taskqueue_start_threads(&pf->tq, 1, PI_NET, "%s adminq", + device_get_nameunit(dev)); + + return (0); +} + +static void +ixl_init_taskqueues(struct ixl_pf *pf) +{ + struct ixl_vsi *vsi = &pf->vsi; + struct ixl_queue *que = vsi->queues; + device_t dev = pf->dev; +#ifdef RSS + int cpu_id; + cpuset_t cpu_mask; +#endif + + /* Tasklet for Admin Queue */ + TASK_INIT(&pf->adminq, 0, ixl_do_adminq, pf); +#ifdef PCI_IOV + /* VFLR Tasklet */ + TASK_INIT(&pf->vflr_task, 0, ixl_handle_vflr, pf); +#endif + + /* Create and start PF taskqueue */ + pf->tq = taskqueue_create_fast("ixl_adm", M_NOWAIT, + taskqueue_thread_enqueue, &pf->tq); + taskqueue_start_threads(&pf->tq, 1, PI_NET, "%s adminq", + device_get_nameunit(dev)); + + /* Create queue tasks and start queue taskqueues */ + for (int i = 0; i < vsi->num_queues; i++, que++) { + TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que); + TASK_INIT(&que->task, 0, ixl_handle_que, que); + que->tq = taskqueue_create_fast("ixl_que", M_NOWAIT, + taskqueue_thread_enqueue, &que->tq); +#ifdef RSS + cpu_id = rss_getcpu(i % rss_getnumbuckets()); + CPU_SETOF(cpu_id, &cpu_mask); + taskqueue_start_threads_cpuset(&que->tq, 1, PI_NET, + &cpu_mask, "%s (bucket %d)", + device_get_nameunit(dev), cpu_id); +#else + taskqueue_start_threads(&que->tq, 1, PI_NET, + "%s (que %d)", device_get_nameunit(dev), que->me); +#endif + } +} + +static void +ixl_free_taskqueues(struct ixl_pf *pf) +{ + struct ixl_vsi *vsi = &pf->vsi; + struct ixl_queue *que = vsi->queues; + + if (pf->tq) + taskqueue_free(pf->tq); + for (int i = 0; i < vsi->num_queues; i++, que++) { + if (que->tq) + taskqueue_free(que->tq); + } +} + +/********************************************************************* + * + * Setup MSIX Interrupt resources and handlers for the VSI + * + **********************************************************************/ +static int +ixl_assign_vsi_msix(struct ixl_pf *pf) +{ + device_t dev = pf->dev; + struct ixl_vsi *vsi = &pf->vsi; + struct ixl_queue *que = vsi->queues; + struct tx_ring *txr; + int error, rid, vector = 0; + + /* Admin Que is vector 0*/ + rid = vector + 1; + pf->res = bus_alloc_resource_any(dev, + SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); + if (!pf->res) { + device_printf(dev,"Unable to allocate" + " bus resource: Adminq interrupt [%d]\n", rid); + return (ENXIO); + } + /* Set the adminq vector and handler */ + error = bus_setup_intr(dev, pf->res, + INTR_TYPE_NET | INTR_MPSAFE, NULL, + ixl_msix_adminq, pf, &pf->tag); + if (error) { + pf->res = NULL; + device_printf(dev, "Failed to register Admin que handler"); + return (error); + } + bus_describe_intr(dev, pf->res, pf->tag, "aq"); + pf->admvec = vector; + ++vector; + + /* Now set up the stations */ + for (int i = 0; i < vsi->num_queues; i++, vector++, que++) { + int cpu_id = i; + rid = vector + 1; + txr = &que->txr; + que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, + RF_SHAREABLE | RF_ACTIVE); + if (que->res == NULL) { + device_printf(dev,"Unable to allocate" + " bus resource: que interrupt [%d]\n", vector); + return (ENXIO); + } + /* Set the handler function */ + error = bus_setup_intr(dev, que->res, + INTR_TYPE_NET | INTR_MPSAFE, NULL, + ixl_msix_que, que, &que->tag); + if (error) { + que->res = NULL; + device_printf(dev, "Failed to register que handler"); + return (error); + } + bus_describe_intr(dev, que->res, que->tag, "q%d", i); + /* Bind the vector to a CPU */ +#ifdef RSS + cpu_id = rss_getcpu(i % rss_getnumbuckets()); +#endif + bus_bind_intr(dev, que->res, cpu_id); + que->msix = vector; + } + + return (0); +} + + +/* + * Allocate MSI/X vectors + */ +static int +ixl_init_msix(struct ixl_pf *pf) +{ + device_t dev = pf->dev; + int rid, want, vectors, queues, available; + + /* Override by tuneable */ + if (ixl_enable_msix == 0) + goto msi; + + /* + ** When used in a virtualized environment + ** PCI BUSMASTER capability may not be set + ** so explicity set it here and rewrite + ** the ENABLE in the MSIX control register + ** at this point to cause the host to + ** successfully initialize us. + */ + { + u16 pci_cmd_word; + int msix_ctrl; + pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); + pci_cmd_word |= PCIM_CMD_BUSMASTEREN; + pci_write_config(dev, PCIR_COMMAND, pci_cmd_word, 2); + pci_find_cap(dev, PCIY_MSIX, &rid); + rid += PCIR_MSIX_CTRL; + msix_ctrl = pci_read_config(dev, rid, 2); + msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE; + pci_write_config(dev, rid, msix_ctrl, 2); + } + + /* First try MSI/X */ + rid = PCIR_BAR(IXL_BAR); + pf->msix_mem = bus_alloc_resource_any(dev, + SYS_RES_MEMORY, &rid, RF_ACTIVE); + if (!pf->msix_mem) { + /* May not be enabled */ + device_printf(pf->dev, + "Unable to map MSIX table \n"); + goto msi; + } + + available = pci_msix_count(dev); + if (available == 0) { /* system has msix disabled */ + bus_release_resource(dev, SYS_RES_MEMORY, + rid, pf->msix_mem); + pf->msix_mem = NULL; + goto msi; + } + + /* Figure out a reasonable auto config value */ + queues = (mp_ncpus > (available - 1)) ? (available - 1) : mp_ncpus; + + /* Override with hardcoded value if it's less than autoconfig count */ + if ((ixl_max_queues != 0) && (ixl_max_queues <= queues)) + queues = ixl_max_queues; + else if ((ixl_max_queues != 0) && (ixl_max_queues > queues)) + device_printf(dev, "ixl_max_queues > # of cpus, using " + "autoconfig amount...\n"); + /* Or limit maximum auto-configured queues to 8 */ + else if ((ixl_max_queues == 0) && (queues > 8)) + queues = 8; + +#ifdef RSS + /* If we're doing RSS, clamp at the number of RSS buckets */ + if (queues > rss_getnumbuckets()) + queues = rss_getnumbuckets(); +#endif + + /* + ** Want one vector (RX/TX pair) per queue + ** plus an additional for the admin queue. + */ + want = queues + 1; + if (want <= available) /* Have enough */ + vectors = want; + else { + device_printf(pf->dev, + "MSIX Configuration Problem, " + "%d vectors available but %d wanted!\n", + available, want); + return (0); /* Will go to Legacy setup */ + } + + if (pci_alloc_msix(dev, &vectors) == 0) { + device_printf(pf->dev, + "Using MSIX interrupts with %d vectors\n", vectors); + pf->msix = vectors; + pf->vsi.num_queues = queues; +#ifdef RSS + /* + * If we're doing RSS, the number of queues needs to + * match the number of RSS buckets that are configured. + * + * + If there's more queues than RSS buckets, we'll end + * up with queues that get no traffic. + * + * + If there's more RSS buckets than queues, we'll end + * up having multiple RSS buckets map to the same queue, + * so there'll be some contention. + */ + if (queues != rss_getnumbuckets()) { + device_printf(dev, + "%s: queues (%d) != RSS buckets (%d)" + "; performance will be impacted.\n", + __func__, queues, rss_getnumbuckets()); + } +#endif + return (vectors); + } +msi: + vectors = pci_msi_count(dev); + pf->vsi.num_queues = 1; + pf->msix = 1; + ixl_max_queues = 1; + ixl_enable_msix = 0; + if (vectors == 1 && pci_alloc_msi(dev, &vectors) == 0) + device_printf(pf->dev,"Using an MSI interrupt\n"); + else { + pf->msix = 0; + device_printf(pf->dev,"Using a Legacy interrupt\n"); + } + return (vectors); +} + + +/* + * Plumb MSI/X vectors + */ +static void +ixl_configure_msix(struct ixl_pf *pf) +{ + struct i40e_hw *hw = &pf->hw; + struct ixl_vsi *vsi = &pf->vsi; + u32 reg; + u16 vector = 1; + + /* First set up the adminq - vector 0 */ + wr32(hw, I40E_PFINT_ICR0_ENA, 0); /* disable all */ + rd32(hw, I40E_PFINT_ICR0); /* read to clear */ + + reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | + I40E_PFINT_ICR0_ENA_GRST_MASK | + I40E_PFINT_ICR0_HMC_ERR_MASK | + I40E_PFINT_ICR0_ENA_ADMINQ_MASK | + I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | + I40E_PFINT_ICR0_ENA_VFLR_MASK | + I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK; + wr32(hw, I40E_PFINT_ICR0_ENA, reg); + + wr32(hw, I40E_PFINT_LNKLST0, 0x7FF); + wr32(hw, I40E_PFINT_ITR0(IXL_RX_ITR), 0x003E); + + wr32(hw, I40E_PFINT_DYN_CTL0, + I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK | + I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK); + + wr32(hw, I40E_PFINT_STAT_CTL0, 0); + + /* Next configure the queues */ + for (int i = 0; i < vsi->num_queues; i++, vector++) { + wr32(hw, I40E_PFINT_DYN_CTLN(i), 0); + /* First queue type is RX / type 0 */ + wr32(hw, I40E_PFINT_LNKLSTN(i), i); + + reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK | + (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | + (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | + (i << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | + (I40E_QUEUE_TYPE_TX << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT); + wr32(hw, I40E_QINT_RQCTL(i), reg); + + reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK | + (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | + (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | + (IXL_QUEUE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | + (I40E_QUEUE_TYPE_RX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); + wr32(hw, I40E_QINT_TQCTL(i), reg); + } +} + +/* + * Configure for MSI single vector operation + */ +static void +ixl_configure_legacy(struct ixl_pf *pf) +{ + struct i40e_hw *hw = &pf->hw; + u32 reg; + + + wr32(hw, I40E_PFINT_ITR0(0), 0); + wr32(hw, I40E_PFINT_ITR0(1), 0); + + + /* Setup "other" causes */ + reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK + | I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK + | I40E_PFINT_ICR0_ENA_GRST_MASK + | I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK + | I40E_PFINT_ICR0_ENA_GPIO_MASK + | I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK + | I40E_PFINT_ICR0_ENA_HMC_ERR_MASK + | I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK + | I40E_PFINT_ICR0_ENA_VFLR_MASK + | I40E_PFINT_ICR0_ENA_ADMINQ_MASK + ; + wr32(hw, I40E_PFINT_ICR0_ENA, reg); + + /* SW_ITR_IDX = 0, but don't change INTENA */ + wr32(hw, I40E_PFINT_DYN_CTL0, + I40E_PFINT_DYN_CTLN_SW_ITR_INDX_MASK | + I40E_PFINT_DYN_CTLN_INTENA_MSK_MASK); + /* SW_ITR_IDX = 0, OTHER_ITR_IDX = 0 */ + wr32(hw, I40E_PFINT_STAT_CTL0, 0); + + /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */ + wr32(hw, I40E_PFINT_LNKLST0, 0); + + /* Associate the queue pair to the vector and enable the q int */ + reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK + | (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) + | (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); + wr32(hw, I40E_QINT_RQCTL(0), reg); + + reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK + | (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) + | (IXL_QUEUE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); + wr32(hw, I40E_QINT_TQCTL(0), reg); + + /* Next enable the queue pair */ + reg = rd32(hw, I40E_QTX_ENA(0)); + reg |= I40E_QTX_ENA_QENA_REQ_MASK; + wr32(hw, I40E_QTX_ENA(0), reg); + + reg = rd32(hw, I40E_QRX_ENA(0)); + reg |= I40E_QRX_ENA_QENA_REQ_MASK; + wr32(hw, I40E_QRX_ENA(0), reg); +} + + +/* + * Set the Initial ITR state + */ +static void +ixl_configure_itr(struct ixl_pf *pf) +{ + struct i40e_hw *hw = &pf->hw; + struct ixl_vsi *vsi = &pf->vsi; + struct ixl_queue *que = vsi->queues; + + vsi->rx_itr_setting = ixl_rx_itr; + if (ixl_dynamic_rx_itr) + vsi->rx_itr_setting |= IXL_ITR_DYNAMIC; + vsi->tx_itr_setting = ixl_tx_itr; + if (ixl_dynamic_tx_itr) + vsi->tx_itr_setting |= IXL_ITR_DYNAMIC; + + for (int i = 0; i < vsi->num_queues; i++, que++) { + struct tx_ring *txr = &que->txr; + struct rx_ring *rxr = &que->rxr; + + wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, i), + vsi->rx_itr_setting); + rxr->itr = vsi->rx_itr_setting; + rxr->latency = IXL_AVE_LATENCY; + wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, i), + vsi->tx_itr_setting); + txr->itr = vsi->tx_itr_setting; + txr->latency = IXL_AVE_LATENCY; + } +} + + +static int +ixl_allocate_pci_resources(struct ixl_pf *pf) +{ + int rid; + device_t dev = pf->dev; + + rid = PCIR_BAR(0); + pf->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, + &rid, RF_ACTIVE); + + if (!(pf->pci_mem)) { + device_printf(dev,"Unable to allocate bus resource: memory\n"); + return (ENXIO); + } + + pf->osdep.mem_bus_space_tag = + rman_get_bustag(pf->pci_mem); + pf->osdep.mem_bus_space_handle = + rman_get_bushandle(pf->pci_mem); + pf->osdep.mem_bus_space_size = rman_get_size(pf->pci_mem); + pf->osdep.flush_reg = I40E_GLGEN_STAT; + pf->hw.hw_addr = (u8 *) &pf->osdep.mem_bus_space_handle; + + pf->hw.back = &pf->osdep; + + /* + ** Now setup MSI or MSI/X, should + ** return us the number of supported + ** vectors. (Will be 1 for MSI) + */ + pf->msix = ixl_init_msix(pf); + return (0); +} + +static void +ixl_free_pci_resources(struct ixl_pf * pf) +{ + struct ixl_vsi *vsi = &pf->vsi; + struct ixl_queue *que = vsi->queues; + device_t dev = pf->dev; + int rid, memrid; + + memrid = PCIR_BAR(IXL_BAR); + + /* We may get here before stations are setup */ + if ((!ixl_enable_msix) || (que == NULL)) + goto early; + + /* + ** Release all msix VSI resources: + */ + for (int i = 0; i < vsi->num_queues; i++, que++) { + rid = que->msix + 1; + if (que->tag != NULL) { + bus_teardown_intr(dev, que->res, que->tag); + que->tag = NULL; + } + if (que->res != NULL) + bus_release_resource(dev, SYS_RES_IRQ, rid, que->res); + } + +early: + /* Clean the AdminQ interrupt last */ + if (pf->admvec) /* we are doing MSIX */ + rid = pf->admvec + 1; + else + (pf->msix != 0) ? (rid = 1):(rid = 0); + + if (pf->tag != NULL) { + bus_teardown_intr(dev, pf->res, pf->tag); + pf->tag = NULL; + } + if (pf->res != NULL) + bus_release_resource(dev, SYS_RES_IRQ, rid, pf->res); + + if (pf->msix) + pci_release_msi(dev); + + if (pf->msix_mem != NULL) + bus_release_resource(dev, SYS_RES_MEMORY, + memrid, pf->msix_mem); + + if (pf->pci_mem != NULL) + bus_release_resource(dev, SYS_RES_MEMORY, + PCIR_BAR(0), pf->pci_mem); + + return; +} + +static void +ixl_add_ifmedia(struct ixl_vsi *vsi, u32 phy_type) +{ + /* Display supported media types */ + if (phy_type & (1 << I40E_PHY_TYPE_100BASE_TX)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_100_TX, 0, NULL); + + if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_T)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_T, 0, NULL); + if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_SX)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_SX, 0, NULL); + if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_LX)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_LX, 0, NULL); + + if (phy_type & (1 << I40E_PHY_TYPE_XAUI) || + phy_type & (1 << I40E_PHY_TYPE_XFI) || + phy_type & (1 << I40E_PHY_TYPE_10GBASE_SFPP_CU)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX, 0, NULL); + + if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_SR)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SR, 0, NULL); + if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_LR)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_LR, 0, NULL); + if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_T)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_T, 0, NULL); + + if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_CR4) || + phy_type & (1 << I40E_PHY_TYPE_40GBASE_CR4_CU) || + phy_type & (1 << I40E_PHY_TYPE_40GBASE_AOC) || + phy_type & (1 << I40E_PHY_TYPE_XLAUI) || + phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_CR4, 0, NULL); + if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_SR4)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_SR4, 0, NULL); + if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_LR4)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_LR4, 0, NULL); + +#ifndef IFM_ETH_XTYPE + if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_KX)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_CX, 0, NULL); + + if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1_CU) || + phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1) || + phy_type & (1 << I40E_PHY_TYPE_10GBASE_AOC) || + phy_type & (1 << I40E_PHY_TYPE_SFI)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX, 0, NULL); + if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KX4)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_CX4, 0, NULL); + if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KR)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SR, 0, NULL); + + if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_SR4, 0, NULL); + if (phy_type & (1 << I40E_PHY_TYPE_XLPPI)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_CR4, 0, NULL); +#else + if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_KX)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_KX, 0, NULL); + + if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1_CU) + || phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_CR1, 0, NULL); + if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_AOC)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX_LONG, 0, NULL); + if (phy_type & (1 << I40E_PHY_TYPE_SFI)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SFI, 0, NULL); + if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KX4)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_KX4, 0, NULL); + if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KR)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_KR, 0, NULL); + + if (phy_type & (1 << I40E_PHY_TYPE_20GBASE_KR2)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_20G_KR2, 0, NULL); + + if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_KR4, 0, NULL); + if (phy_type & (1 << I40E_PHY_TYPE_XLPPI)) + ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_XLPPI, 0, NULL); +#endif +} + +/********************************************************************* + * + * Setup networking device structure and register an interface. + * + **********************************************************************/ +static int +ixl_setup_interface(device_t dev, struct ixl_vsi *vsi) +{ + struct ifnet *ifp; + struct i40e_hw *hw = vsi->hw; + struct ixl_queue *que = vsi->queues; + struct i40e_aq_get_phy_abilities_resp abilities; + enum i40e_status_code aq_error = 0; + + INIT_DEBUGOUT("ixl_setup_interface: begin"); + + ifp = vsi->ifp = if_alloc(IFT_ETHER); + if (ifp == NULL) { + device_printf(dev, "can not allocate ifnet structure\n"); + return (-1); + } + if_initname(ifp, device_get_name(dev), device_get_unit(dev)); + ifp->if_mtu = ETHERMTU; + ifp->if_baudrate = IF_Gbps(40); + ifp->if_init = ixl_init; + ifp->if_softc = vsi; + ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; + ifp->if_ioctl = ixl_ioctl; + +#if __FreeBSD_version >= 1100036 + if_setgetcounterfn(ifp, ixl_get_counter); +#endif + + ifp->if_transmit = ixl_mq_start; + + ifp->if_qflush = ixl_qflush; + + ifp->if_snd.ifq_maxlen = que->num_desc - 2; + + vsi->max_frame_size = + ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN + + ETHER_VLAN_ENCAP_LEN; + + /* + * Tell the upper layer(s) we support long frames. + */ + ifp->if_hdrlen = sizeof(struct ether_vlan_header); + + ifp->if_capabilities |= IFCAP_HWCSUM; + ifp->if_capabilities |= IFCAP_HWCSUM_IPV6; + ifp->if_capabilities |= IFCAP_TSO; + ifp->if_capabilities |= IFCAP_JUMBO_MTU; + ifp->if_capabilities |= IFCAP_LRO; + + /* VLAN capabilties */ + ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING + | IFCAP_VLAN_HWTSO + | IFCAP_VLAN_MTU + | IFCAP_VLAN_HWCSUM; + ifp->if_capenable = ifp->if_capabilities; + + /* + ** Don't turn this on by default, if vlans are + ** created on another pseudo device (eg. lagg) + ** then vlan events are not passed thru, breaking + ** operation, but with HW FILTER off it works. If + ** using vlans directly on the ixl driver you can + ** enable this and get full hardware tag filtering. + */ + ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; + + /* + * Specify the media types supported by this adapter and register + * callbacks to update media and link information + */ + ifmedia_init(&vsi->media, IFM_IMASK, ixl_media_change, + ixl_media_status); + + aq_error = i40e_aq_get_phy_capabilities(hw, + FALSE, TRUE, &abilities, NULL); + /* May need delay to detect fiber correctly */ + if (aq_error == I40E_ERR_UNKNOWN_PHY) { + i40e_msec_delay(200); + aq_error = i40e_aq_get_phy_capabilities(hw, FALSE, + TRUE, &abilities, NULL); + } + if (aq_error) { + if (aq_error == I40E_ERR_UNKNOWN_PHY) + device_printf(dev, "Unknown PHY type detected!\n"); + else + device_printf(dev, + "Error getting supported media types, err %d," + " AQ error %d\n", aq_error, hw->aq.asq_last_status); + return (0); + } + + ixl_add_ifmedia(vsi, abilities.phy_type); + + /* Use autoselect media by default */ + ifmedia_add(&vsi->media, IFM_ETHER | IFM_AUTO, 0, NULL); + ifmedia_set(&vsi->media, IFM_ETHER | IFM_AUTO); + + ether_ifattach(ifp, hw->mac.addr); + + return (0); +} + +/* +** Run when the Admin Queue gets a +** link transition interrupt. +*/ +static void +ixl_link_event(struct ixl_pf *pf, struct i40e_arq_event_info *e) +{ + struct i40e_hw *hw = &pf->hw; + struct i40e_aqc_get_link_status *status = + (struct i40e_aqc_get_link_status *)&e->desc.params.raw; + bool check; + + hw->phy.get_link_info = TRUE; + i40e_get_link_status(hw, &check); + pf->link_up = check; +#ifdef IXL_DEBUG + printf("Link is %s\n", check ? "up":"down"); +#endif + /* Report if Unqualified modules are found */ + if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) && + (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) && + (!(status->link_info & I40E_AQ_LINK_UP))) + device_printf(pf->dev, "Link failed because " + "an unqualified module was detected\n"); + + return; +} + +/********************************************************************* + * + * Get Firmware Switch configuration + * - this will need to be more robust when more complex + * switch configurations are enabled. + * + **********************************************************************/ +static int +ixl_switch_config(struct ixl_pf *pf) +{ + struct i40e_hw *hw = &pf->hw; + struct ixl_vsi *vsi = &pf->vsi; + device_t dev = vsi->dev; + struct i40e_aqc_get_switch_config_resp *sw_config; + u8 aq_buf[I40E_AQ_LARGE_BUF]; + int ret; + u16 next = 0; + + memset(&aq_buf, 0, sizeof(aq_buf)); + sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf; + ret = i40e_aq_get_switch_config(hw, sw_config, + sizeof(aq_buf), &next, NULL); + if (ret) { + device_printf(dev,"aq_get_switch_config failed (ret=%d)!!\n", + ret); + return (ret); + } +#ifdef IXL_DEBUG + device_printf(dev, + "Switch config: header reported: %d in structure, %d total\n", + sw_config->header.num_reported, sw_config->header.num_total); + for (int i = 0; i < sw_config->header.num_reported; i++) { + device_printf(dev, + "%d: type=%d seid=%d uplink=%d downlink=%d\n", i, + sw_config->element[i].element_type, + sw_config->element[i].seid, + sw_config->element[i].uplink_seid, + sw_config->element[i].downlink_seid); + } +#endif + /* Simplified due to a single VSI at the moment */ + vsi->uplink_seid = sw_config->element[0].uplink_seid; + vsi->downlink_seid = sw_config->element[0].downlink_seid; + vsi->seid = sw_config->element[0].seid; + return (ret); +} + +/********************************************************************* + * + * Initialize the VSI: this handles contexts, which means things + * like the number of descriptors, buffer size, + * plus we init the rings thru this function. + * + **********************************************************************/ +static int +ixl_initialize_vsi(struct ixl_vsi *vsi) +{ + struct ixl_pf *pf = vsi->back; + struct ixl_queue *que = vsi->queues; + device_t dev = vsi->dev; + struct i40e_hw *hw = vsi->hw; + struct i40e_vsi_context ctxt; + int err = 0; + + memset(&ctxt, 0, sizeof(ctxt)); + ctxt.seid = vsi->seid; + if (pf->veb_seid != 0) + ctxt.uplink_seid = pf->veb_seid; + ctxt.pf_num = hw->pf_id; + err = i40e_aq_get_vsi_params(hw, &ctxt, NULL); + if (err) { + device_printf(dev,"get vsi params failed %x!!\n", err); + return (err); + } +#ifdef IXL_DEBUG + printf("get_vsi_params: seid: %d, uplinkseid: %d, vsi_number: %d, " + "vsis_allocated: %d, vsis_unallocated: %d, flags: 0x%x, " + "pfnum: %d, vfnum: %d, stat idx: %d, enabled: %d\n", ctxt.seid, + ctxt.uplink_seid, ctxt.vsi_number, + ctxt.vsis_allocated, ctxt.vsis_unallocated, + ctxt.flags, ctxt.pf_num, ctxt.vf_num, + ctxt.info.stat_counter_idx, ctxt.info.up_enable_bits); +#endif + /* + ** Set the queue and traffic class bits + ** - when multiple traffic classes are supported + ** this will need to be more robust. + */ + ctxt.info.valid_sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; + ctxt.info.mapping_flags |= I40E_AQ_VSI_QUE_MAP_CONTIG; + /* In contig mode, que_mapping[0] is first queue index used by this VSI */ + ctxt.info.queue_mapping[0] = 0; + /* + * This VSI will only use traffic class 0; start traffic class 0's + * queue allocation at queue 0, and assign it 64 (2^6) queues (though + * the driver may not use all of them). + */ + ctxt.info.tc_mapping[0] = ((0 << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) + & I40E_AQ_VSI_TC_QUE_OFFSET_MASK) | + ((6 << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT) + & I40E_AQ_VSI_TC_QUE_NUMBER_MASK); + + /* Set VLAN receive stripping mode */ + ctxt.info.valid_sections |= I40E_AQ_VSI_PROP_VLAN_VALID; + ctxt.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL; + if (vsi->ifp->if_capenable & IFCAP_VLAN_HWTAGGING) + ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH; + else + ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_NOTHING; + + /* Keep copy of VSI info in VSI for statistic counters */ + memcpy(&vsi->info, &ctxt.info, sizeof(ctxt.info)); + + /* Reset VSI statistics */ + ixl_vsi_reset_stats(vsi); + vsi->hw_filters_add = 0; + vsi->hw_filters_del = 0; + + ctxt.flags = htole16(I40E_AQ_VSI_TYPE_PF); + + err = i40e_aq_update_vsi_params(hw, &ctxt, NULL); + if (err) { + device_printf(dev,"update vsi params failed %x!!\n", + hw->aq.asq_last_status); + return (err); + } + + for (int i = 0; i < vsi->num_queues; i++, que++) { + struct tx_ring *txr = &que->txr; + struct rx_ring *rxr = &que->rxr; + struct i40e_hmc_obj_txq tctx; + struct i40e_hmc_obj_rxq rctx; + u32 txctl; + u16 size; + + + /* Setup the HMC TX Context */ + size = que->num_desc * sizeof(struct i40e_tx_desc); + memset(&tctx, 0, sizeof(struct i40e_hmc_obj_txq)); + tctx.new_context = 1; + tctx.base = (txr->dma.pa/IXL_TX_CTX_BASE_UNITS); + tctx.qlen = que->num_desc; + tctx.fc_ena = 0; + tctx.rdylist = vsi->info.qs_handle[0]; /* index is TC */ + /* Enable HEAD writeback */ + tctx.head_wb_ena = 1; + tctx.head_wb_addr = txr->dma.pa + + (que->num_desc * sizeof(struct i40e_tx_desc)); + tctx.rdylist_act = 0; + err = i40e_clear_lan_tx_queue_context(hw, i); + if (err) { + device_printf(dev, "Unable to clear TX context\n"); + break; + } + err = i40e_set_lan_tx_queue_context(hw, i, &tctx); + if (err) { + device_printf(dev, "Unable to set TX context\n"); + break; + } + /* Associate the ring with this PF */ + txctl = I40E_QTX_CTL_PF_QUEUE; + txctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) & + I40E_QTX_CTL_PF_INDX_MASK); + wr32(hw, I40E_QTX_CTL(i), txctl); + ixl_flush(hw); + + /* Do ring (re)init */ + ixl_init_tx_ring(que); + + /* Next setup the HMC RX Context */ + if (vsi->max_frame_size <= MCLBYTES) + rxr->mbuf_sz = MCLBYTES; + else + rxr->mbuf_sz = MJUMPAGESIZE; + + u16 max_rxmax = rxr->mbuf_sz * hw->func_caps.rx_buf_chain_len; + + /* Set up an RX context for the HMC */ + memset(&rctx, 0, sizeof(struct i40e_hmc_obj_rxq)); + rctx.dbuff = rxr->mbuf_sz >> I40E_RXQ_CTX_DBUFF_SHIFT; + /* ignore header split for now */ + rctx.hbuff = 0 >> I40E_RXQ_CTX_HBUFF_SHIFT; + rctx.rxmax = (vsi->max_frame_size < max_rxmax) ? + vsi->max_frame_size : max_rxmax; + rctx.dtype = 0; + rctx.dsize = 1; /* do 32byte descriptors */ + rctx.hsplit_0 = 0; /* no HDR split initially */ + rctx.base = (rxr->dma.pa/IXL_RX_CTX_BASE_UNITS); + rctx.qlen = que->num_desc; + rctx.tphrdesc_ena = 1; + rctx.tphwdesc_ena = 1; + rctx.tphdata_ena = 0; + rctx.tphhead_ena = 0; + rctx.lrxqthresh = 2; + rctx.crcstrip = 1; + rctx.l2tsel = 1; + rctx.showiv = 1; + rctx.fc_ena = 0; + rctx.prefena = 1; + + err = i40e_clear_lan_rx_queue_context(hw, i); + if (err) { + device_printf(dev, + "Unable to clear RX context %d\n", i); + break; + } + err = i40e_set_lan_rx_queue_context(hw, i, &rctx); + if (err) { + device_printf(dev, "Unable to set RX context %d\n", i); + break; + } + err = ixl_init_rx_ring(que); + if (err) { + device_printf(dev, "Fail in init_rx_ring %d\n", i); + break; + } +#ifdef DEV_NETMAP + /* preserve queue */ + if (vsi->ifp->if_capenable & IFCAP_NETMAP) { + struct netmap_adapter *na = NA(vsi->ifp); + struct netmap_kring *kring = &na->rx_rings[i]; + int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring); + wr32(vsi->hw, I40E_QRX_TAIL(que->me), t); + } else +#endif /* DEV_NETMAP */ + wr32(vsi->hw, I40E_QRX_TAIL(que->me), que->num_desc - 1); + } + return (err); +} + + +/********************************************************************* + * + * Free all VSI structs. + * + **********************************************************************/ +void +ixl_free_vsi(struct ixl_vsi *vsi) +{ + struct ixl_pf *pf = (struct ixl_pf *)vsi->back; + struct ixl_queue *que = vsi->queues; + + /* Free station queues */ + for (int i = 0; i < vsi->num_queues; i++, que++) { + struct tx_ring *txr = &que->txr; + struct rx_ring *rxr = &que->rxr; + + if (!mtx_initialized(&txr->mtx)) /* uninitialized */ + continue; + IXL_TX_LOCK(txr); + ixl_free_que_tx(que); + if (txr->base) + i40e_free_dma_mem(&pf->hw, &txr->dma); + IXL_TX_UNLOCK(txr); + IXL_TX_LOCK_DESTROY(txr); + + if (!mtx_initialized(&rxr->mtx)) /* uninitialized */ + continue; + IXL_RX_LOCK(rxr); + ixl_free_que_rx(que); + if (rxr->base) + i40e_free_dma_mem(&pf->hw, &rxr->dma); + IXL_RX_UNLOCK(rxr); + IXL_RX_LOCK_DESTROY(rxr); + + } + free(vsi->queues, M_DEVBUF); + + /* Free VSI filter list */ + ixl_free_mac_filters(vsi); +} + +static void +ixl_free_mac_filters(struct ixl_vsi *vsi) +{ + struct ixl_mac_filter *f; + + while (!SLIST_EMPTY(&vsi->ftl)) { + f = SLIST_FIRST(&vsi->ftl); + SLIST_REMOVE_HEAD(&vsi->ftl, next); + free(f, M_DEVBUF); + } +} + + +/********************************************************************* + * + * Allocate memory for the VSI (virtual station interface) and their + * associated queues, rings and the descriptors associated with each, + * called only once at attach. + * + **********************************************************************/ +static int +ixl_setup_stations(struct ixl_pf *pf) +{ + device_t dev = pf->dev; + struct ixl_vsi *vsi; + struct ixl_queue *que; + struct tx_ring *txr; + struct rx_ring *rxr; + int rsize, tsize; + int error = I40E_SUCCESS; + + vsi = &pf->vsi; + vsi->back = (void *)pf; + vsi->hw = &pf->hw; + vsi->id = 0; + vsi->num_vlans = 0; + vsi->back = pf; + + /* Get memory for the station queues */ + if (!(vsi->queues = + (struct ixl_queue *) malloc(sizeof(struct ixl_queue) * + vsi->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { + device_printf(dev, "Unable to allocate queue memory\n"); + error = ENOMEM; + goto early; + } + + for (int i = 0; i < vsi->num_queues; i++) { + que = &vsi->queues[i]; + que->num_desc = ixl_ringsz; + que->me = i; + que->vsi = vsi; + /* mark the queue as active */ + vsi->active_queues |= (u64)1 << que->me; + txr = &que->txr; + txr->que = que; + txr->tail = I40E_QTX_TAIL(que->me); + + /* Initialize the TX lock */ + snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", + device_get_nameunit(dev), que->me); + mtx_init(&txr->mtx, txr->mtx_name, NULL, MTX_DEF); + /* Create the TX descriptor ring */ + tsize = roundup2((que->num_desc * + sizeof(struct i40e_tx_desc)) + + sizeof(u32), DBA_ALIGN); + if (i40e_allocate_dma_mem(&pf->hw, + &txr->dma, i40e_mem_reserved, tsize, DBA_ALIGN)) { + device_printf(dev, + "Unable to allocate TX Descriptor memory\n"); + error = ENOMEM; + goto fail; + } + txr->base = (struct i40e_tx_desc *)txr->dma.va; + bzero((void *)txr->base, tsize); + /* Now allocate transmit soft structs for the ring */ + if (ixl_allocate_tx_data(que)) { + device_printf(dev, + "Critical Failure setting up TX structures\n"); + error = ENOMEM; + goto fail; + } + /* Allocate a buf ring */ + txr->br = buf_ring_alloc(4096, M_DEVBUF, + M_WAITOK, &txr->mtx); + if (txr->br == NULL) { + device_printf(dev, + "Critical Failure setting up TX buf ring\n"); + error = ENOMEM; + goto fail; + } + + /* + * Next the RX queues... + */ + rsize = roundup2(que->num_desc * + sizeof(union i40e_rx_desc), DBA_ALIGN); + rxr = &que->rxr; + rxr->que = que; + rxr->tail = I40E_QRX_TAIL(que->me); + + /* Initialize the RX side lock */ + snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", + device_get_nameunit(dev), que->me); + mtx_init(&rxr->mtx, rxr->mtx_name, NULL, MTX_DEF); + + if (i40e_allocate_dma_mem(&pf->hw, + &rxr->dma, i40e_mem_reserved, rsize, 4096)) { + device_printf(dev, + "Unable to allocate RX Descriptor memory\n"); + error = ENOMEM; + goto fail; + } + rxr->base = (union i40e_rx_desc *)rxr->dma.va; + bzero((void *)rxr->base, rsize); + + /* Allocate receive soft structs for the ring*/ + if (ixl_allocate_rx_data(que)) { + device_printf(dev, + "Critical Failure setting up receive structs\n"); + error = ENOMEM; + goto fail; + } + } + + return (0); + +fail: + for (int i = 0; i < vsi->num_queues; i++) { + que = &vsi->queues[i]; + rxr = &que->rxr; + txr = &que->txr; + if (rxr->base) + i40e_free_dma_mem(&pf->hw, &rxr->dma); + if (txr->base) + i40e_free_dma_mem(&pf->hw, &txr->dma); + } + +early: + return (error); +} + +/* +** Provide a update to the queue RX +** interrupt moderation value. +*/ +static void +ixl_set_queue_rx_itr(struct ixl_queue *que) +{ + struct ixl_vsi *vsi = que->vsi; + struct i40e_hw *hw = vsi->hw; + struct rx_ring *rxr = &que->rxr; + u16 rx_itr; + u16 rx_latency = 0; + int rx_bytes; + + + /* Idle, do nothing */ + if (rxr->bytes == 0) + return; + + if (ixl_dynamic_rx_itr) { + rx_bytes = rxr->bytes/rxr->itr; + rx_itr = rxr->itr; + + /* Adjust latency range */ + switch (rxr->latency) { + case IXL_LOW_LATENCY: + if (rx_bytes > 10) { + rx_latency = IXL_AVE_LATENCY; + rx_itr = IXL_ITR_20K; + } + break; + case IXL_AVE_LATENCY: + if (rx_bytes > 20) { + rx_latency = IXL_BULK_LATENCY; + rx_itr = IXL_ITR_8K; + } else if (rx_bytes <= 10) { + rx_latency = IXL_LOW_LATENCY; + rx_itr = IXL_ITR_100K; + } + break; + case IXL_BULK_LATENCY: + if (rx_bytes <= 20) { + rx_latency = IXL_AVE_LATENCY; + rx_itr = IXL_ITR_20K; + } + break; + } + + rxr->latency = rx_latency; + + if (rx_itr != rxr->itr) { + /* do an exponential smoothing */ + rx_itr = (10 * rx_itr * rxr->itr) / + ((9 * rx_itr) + rxr->itr); + rxr->itr = rx_itr & IXL_MAX_ITR; + wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, + que->me), rxr->itr); + } + } else { /* We may have have toggled to non-dynamic */ + if (vsi->rx_itr_setting & IXL_ITR_DYNAMIC) + vsi->rx_itr_setting = ixl_rx_itr; + /* Update the hardware if needed */ + if (rxr->itr != vsi->rx_itr_setting) { + rxr->itr = vsi->rx_itr_setting; + wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, + que->me), rxr->itr); + } + } + rxr->bytes = 0; + rxr->packets = 0; + return; +} + + +/* +** Provide a update to the queue TX +** interrupt moderation value. +*/ +static void +ixl_set_queue_tx_itr(struct ixl_queue *que) +{ + struct ixl_vsi *vsi = que->vsi; + struct i40e_hw *hw = vsi->hw; + struct tx_ring *txr = &que->txr; + u16 tx_itr; + u16 tx_latency = 0; + int tx_bytes; + + + /* Idle, do nothing */ + if (txr->bytes == 0) + return; + + if (ixl_dynamic_tx_itr) { + tx_bytes = txr->bytes/txr->itr; + tx_itr = txr->itr; + + switch (txr->latency) { + case IXL_LOW_LATENCY: + if (tx_bytes > 10) { + tx_latency = IXL_AVE_LATENCY; + tx_itr = IXL_ITR_20K; + } + break; + case IXL_AVE_LATENCY: + if (tx_bytes > 20) { + tx_latency = IXL_BULK_LATENCY; + tx_itr = IXL_ITR_8K; + } else if (tx_bytes <= 10) { + tx_latency = IXL_LOW_LATENCY; + tx_itr = IXL_ITR_100K; + } + break; + case IXL_BULK_LATENCY: + if (tx_bytes <= 20) { + tx_latency = IXL_AVE_LATENCY; + tx_itr = IXL_ITR_20K; + } + break; + } + + txr->latency = tx_latency; + + if (tx_itr != txr->itr) { + /* do an exponential smoothing */ + tx_itr = (10 * tx_itr * txr->itr) / + ((9 * tx_itr) + txr->itr); + txr->itr = tx_itr & IXL_MAX_ITR; + wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, + que->me), txr->itr); + } + + } else { /* We may have have toggled to non-dynamic */ + if (vsi->tx_itr_setting & IXL_ITR_DYNAMIC) + vsi->tx_itr_setting = ixl_tx_itr; + /* Update the hardware if needed */ + if (txr->itr != vsi->tx_itr_setting) { + txr->itr = vsi->tx_itr_setting; + wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, + que->me), txr->itr); + } + } + txr->bytes = 0; + txr->packets = 0; + return; +} + +#define QUEUE_NAME_LEN 32 + +static void +ixl_add_vsi_sysctls(struct ixl_pf *pf, struct ixl_vsi *vsi, + struct sysctl_ctx_list *ctx, const char *sysctl_name) +{ + struct sysctl_oid *tree; + struct sysctl_oid_list *child; + struct sysctl_oid_list *vsi_list; + + tree = device_get_sysctl_tree(pf->dev); + child = SYSCTL_CHILDREN(tree); + vsi->vsi_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, sysctl_name, + CTLFLAG_RD, NULL, "VSI Number"); + vsi_list = SYSCTL_CHILDREN(vsi->vsi_node); + + ixl_add_sysctls_eth_stats(ctx, vsi_list, &vsi->eth_stats); +} + +static void +ixl_add_hw_stats(struct ixl_pf *pf) +{ + device_t dev = pf->dev; + struct ixl_vsi *vsi = &pf->vsi; + struct ixl_queue *queues = vsi->queues; + struct i40e_hw_port_stats *pf_stats = &pf->stats; + + struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); + struct sysctl_oid *tree = device_get_sysctl_tree(dev); + struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree); + struct sysctl_oid_list *vsi_list; + + struct sysctl_oid *queue_node; + struct sysctl_oid_list *queue_list; + + struct tx_ring *txr; + struct rx_ring *rxr; + char queue_namebuf[QUEUE_NAME_LEN]; + + /* Driver statistics */ + SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events", + CTLFLAG_RD, &pf->watchdog_events, + "Watchdog timeouts"); + SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "admin_irq", + CTLFLAG_RD, &pf->admin_irq, + "Admin Queue IRQ Handled"); + + ixl_add_vsi_sysctls(pf, &pf->vsi, ctx, "pf"); + vsi_list = SYSCTL_CHILDREN(pf->vsi.vsi_node); + + /* Queue statistics */ + for (int q = 0; q < vsi->num_queues; q++) { + snprintf(queue_namebuf, QUEUE_NAME_LEN, "que%d", q); + queue_node = SYSCTL_ADD_NODE(ctx, vsi_list, + OID_AUTO, queue_namebuf, CTLFLAG_RD, NULL, "Queue #"); + queue_list = SYSCTL_CHILDREN(queue_node); + + txr = &(queues[q].txr); + rxr = &(queues[q].rxr); + + SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "mbuf_defrag_failed", + CTLFLAG_RD, &(queues[q].mbuf_defrag_failed), + "m_defrag() failed"); + SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "dropped", + CTLFLAG_RD, &(queues[q].dropped_pkts), + "Driver dropped packets"); + SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs", + CTLFLAG_RD, &(queues[q].irqs), + "irqs on this queue"); + SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tso_tx", + CTLFLAG_RD, &(queues[q].tso), + "TSO"); + SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_dma_setup", + CTLFLAG_RD, &(queues[q].tx_dma_setup), + "Driver tx dma failure in xmit"); + SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail", + CTLFLAG_RD, &(txr->no_desc), + "Queue No Descriptor Available"); + SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets", + CTLFLAG_RD, &(txr->total_packets), + "Queue Packets Transmitted"); + SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_bytes", + CTLFLAG_RD, &(txr->tx_bytes), + "Queue Bytes Transmitted"); + SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets", + CTLFLAG_RD, &(rxr->rx_packets), + "Queue Packets Received"); + SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes", + CTLFLAG_RD, &(rxr->rx_bytes), + "Queue Bytes Received"); + } + + /* MAC stats */ + ixl_add_sysctls_mac_stats(ctx, child, pf_stats); +} + +static void +ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *ctx, + struct sysctl_oid_list *child, + struct i40e_eth_stats *eth_stats) +{ + struct ixl_sysctl_info ctls[] = + { + {ð_stats->rx_bytes, "good_octets_rcvd", "Good Octets Received"}, + {ð_stats->rx_unicast, "ucast_pkts_rcvd", + "Unicast Packets Received"}, + {ð_stats->rx_multicast, "mcast_pkts_rcvd", + "Multicast Packets Received"}, + {ð_stats->rx_broadcast, "bcast_pkts_rcvd", + "Broadcast Packets Received"}, + {ð_stats->rx_discards, "rx_discards", "Discarded RX packets"}, + {ð_stats->tx_bytes, "good_octets_txd", "Good Octets Transmitted"}, + {ð_stats->tx_unicast, "ucast_pkts_txd", "Unicast Packets Transmitted"}, + {ð_stats->tx_multicast, "mcast_pkts_txd", + "Multicast Packets Transmitted"}, + {ð_stats->tx_broadcast, "bcast_pkts_txd", + "Broadcast Packets Transmitted"}, + // end + {0,0,0} + }; + + struct ixl_sysctl_info *entry = ctls; + while (entry->stat != NULL) + { + SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, entry->name, + CTLFLAG_RD, entry->stat, + entry->description); + entry++; + } +} + +static void +ixl_add_sysctls_mac_stats(struct sysctl_ctx_list *ctx, + struct sysctl_oid_list *child, + struct i40e_hw_port_stats *stats) +{ + struct sysctl_oid *stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac", + CTLFLAG_RD, NULL, "Mac Statistics"); + struct sysctl_oid_list *stat_list = SYSCTL_CHILDREN(stat_node); + + struct i40e_eth_stats *eth_stats = &stats->eth; + ixl_add_sysctls_eth_stats(ctx, stat_list, eth_stats); + + struct ixl_sysctl_info ctls[] = + { + {&stats->crc_errors, "crc_errors", "CRC Errors"}, + {&stats->illegal_bytes, "illegal_bytes", "Illegal Byte Errors"}, + {&stats->mac_local_faults, "local_faults", "MAC Local Faults"}, + {&stats->mac_remote_faults, "remote_faults", "MAC Remote Faults"}, + {&stats->rx_length_errors, "rx_length_errors", "Receive Length Errors"}, + /* Packet Reception Stats */ + {&stats->rx_size_64, "rx_frames_64", "64 byte frames received"}, + {&stats->rx_size_127, "rx_frames_65_127", "65-127 byte frames received"}, + {&stats->rx_size_255, "rx_frames_128_255", "128-255 byte frames received"}, + {&stats->rx_size_511, "rx_frames_256_511", "256-511 byte frames received"}, + {&stats->rx_size_1023, "rx_frames_512_1023", "512-1023 byte frames received"}, + {&stats->rx_size_1522, "rx_frames_1024_1522", "1024-1522 byte frames received"}, + {&stats->rx_size_big, "rx_frames_big", "1523-9522 byte frames received"}, + {&stats->rx_undersize, "rx_undersize", "Undersized packets received"}, + {&stats->rx_fragments, "rx_fragmented", "Fragmented packets received"}, + {&stats->rx_oversize, "rx_oversized", "Oversized packets received"}, + {&stats->rx_jabber, "rx_jabber", "Received Jabber"}, + {&stats->checksum_error, "checksum_errors", "Checksum Errors"}, + /* Packet Transmission Stats */ + {&stats->tx_size_64, "tx_frames_64", "64 byte frames transmitted"}, + {&stats->tx_size_127, "tx_frames_65_127", "65-127 byte frames transmitted"}, + {&stats->tx_size_255, "tx_frames_128_255", "128-255 byte frames transmitted"}, + {&stats->tx_size_511, "tx_frames_256_511", "256-511 byte frames transmitted"}, + {&stats->tx_size_1023, "tx_frames_512_1023", "512-1023 byte frames transmitted"}, + {&stats->tx_size_1522, "tx_frames_1024_1522", "1024-1522 byte frames transmitted"}, + {&stats->tx_size_big, "tx_frames_big", "1523-9522 byte frames transmitted"}, + /* Flow control */ + {&stats->link_xon_tx, "xon_txd", "Link XON transmitted"}, + {&stats->link_xon_rx, "xon_recvd", "Link XON received"}, + {&stats->link_xoff_tx, "xoff_txd", "Link XOFF transmitted"}, + {&stats->link_xoff_rx, "xoff_recvd", "Link XOFF received"}, + /* End */ + {0,0,0} + }; + + struct ixl_sysctl_info *entry = ctls; + while (entry->stat != NULL) + { + SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, entry->name, + CTLFLAG_RD, entry->stat, + entry->description); + entry++; + } +} + + +/* +** ixl_config_rss - setup RSS +** - note this is done for the single vsi +*/ +static void ixl_config_rss(struct ixl_vsi *vsi) +{ + struct ixl_pf *pf = (struct ixl_pf *)vsi->back; + struct i40e_hw *hw = vsi->hw; + u32 lut = 0; + u64 set_hena = 0, hena; + int i, j, que_id; +#ifdef RSS + u32 rss_hash_config; + u32 rss_seed[IXL_KEYSZ]; +#else + u32 rss_seed[IXL_KEYSZ] = {0x41b01687, + 0x183cfd8c, 0xce880440, 0x580cbc3c, + 0x35897377, 0x328b25e1, 0x4fa98922, + 0xb7d90c14, 0xd5bad70d, 0xcd15a2c1}; +#endif + +#ifdef RSS + /* Fetch the configured RSS key */ + rss_getkey((uint8_t *) &rss_seed); +#endif + + /* Fill out hash function seed */ + for (i = 0; i < IXL_KEYSZ; i++) + wr32(hw, I40E_PFQF_HKEY(i), rss_seed[i]); + + /* Enable PCTYPES for RSS: */ +#ifdef RSS + rss_hash_config = rss_gethashconfig(); + if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4) + set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER); + if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4) + set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP); + if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4) + set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP); + if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6) + set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER); + if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX) + set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6); + if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6) + set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP); + if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6) + set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP); +#else + set_hena = + ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP) | + ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP) | + ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) | + ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) | + ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV4) | + ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP) | + ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP) | + ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) | + ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) | + ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6) | + ((u64)1 << I40E_FILTER_PCTYPE_L2_PAYLOAD); +#endif + hena = (u64)rd32(hw, I40E_PFQF_HENA(0)) | + ((u64)rd32(hw, I40E_PFQF_HENA(1)) << 32); + hena |= set_hena; + wr32(hw, I40E_PFQF_HENA(0), (u32)hena); + wr32(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32)); + + /* Populate the LUT with max no. of queues in round robin fashion */ + for (i = j = 0; i < pf->hw.func_caps.rss_table_size; i++, j++) { + if (j == vsi->num_queues) + j = 0; +#ifdef RSS + /* + * Fetch the RSS bucket id for the given indirection entry. + * Cap it at the number of configured buckets (which is + * num_queues.) + */ + que_id = rss_get_indirection_to_bucket(i); + que_id = que_id % vsi->num_queues; +#else + que_id = j; +#endif + /* lut = 4-byte sliding window of 4 lut entries */ + lut = (lut << 8) | (que_id & + ((0x1 << pf->hw.func_caps.rss_table_entry_width) - 1)); + /* On i = 3, we have 4 entries in lut; write to the register */ + if ((i & 3) == 3) + wr32(hw, I40E_PFQF_HLUT(i >> 2), lut); + } + ixl_flush(hw); +} + + +/* +** This routine is run via an vlan config EVENT, +** it enables us to use the HW Filter table since +** we can get the vlan id. This just creates the +** entry in the soft version of the VFTA, init will +** repopulate the real table. +*/ +static void +ixl_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) +{ + struct ixl_vsi *vsi = ifp->if_softc; + struct i40e_hw *hw = vsi->hw; + struct ixl_pf *pf = (struct ixl_pf *)vsi->back; + + if (ifp->if_softc != arg) /* Not our event */ + return; + + if ((vtag == 0) || (vtag > 4095)) /* Invalid */ + return; + + IXL_PF_LOCK(pf); + ++vsi->num_vlans; + ixl_add_filter(vsi, hw->mac.addr, vtag); + IXL_PF_UNLOCK(pf); +} + +/* +** This routine is run via an vlan +** unconfig EVENT, remove our entry +** in the soft vfta. +*/ +static void +ixl_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) +{ + struct ixl_vsi *vsi = ifp->if_softc; + struct i40e_hw *hw = vsi->hw; + struct ixl_pf *pf = (struct ixl_pf *)vsi->back; + + if (ifp->if_softc != arg) + return; + + if ((vtag == 0) || (vtag > 4095)) /* Invalid */ + return; + + IXL_PF_LOCK(pf); + --vsi->num_vlans; + ixl_del_filter(vsi, hw->mac.addr, vtag); + IXL_PF_UNLOCK(pf); +} + +/* +** This routine updates vlan filters, called by init +** it scans the filter table and then updates the hw +** after a soft reset. +*/ +static void +ixl_setup_vlan_filters(struct ixl_vsi *vsi) +{ + struct ixl_mac_filter *f; + int cnt = 0, flags; + + if (vsi->num_vlans == 0) + return; + /* + ** Scan the filter list for vlan entries, + ** mark them for addition and then call + ** for the AQ update. + */ + SLIST_FOREACH(f, &vsi->ftl, next) { + if (f->flags & IXL_FILTER_VLAN) { + f->flags |= + (IXL_FILTER_ADD | + IXL_FILTER_USED); + cnt++; + } + } + if (cnt == 0) { + printf("setup vlan: no filters found!\n"); + return; + } + flags = IXL_FILTER_VLAN; + flags |= (IXL_FILTER_ADD | IXL_FILTER_USED); + ixl_add_hw_filters(vsi, flags, cnt); + return; +} + +/* +** Initialize filter list and add filters that the hardware +** needs to know about. +*/ +static void +ixl_init_filters(struct ixl_vsi *vsi) +{ + /* Add broadcast address */ + ixl_add_filter(vsi, ixl_bcast_addr, IXL_VLAN_ANY); +} + +/* +** This routine adds mulicast filters +*/ +static void +ixl_add_mc_filter(struct ixl_vsi *vsi, u8 *macaddr) +{ + struct ixl_mac_filter *f; + + /* Does one already exist */ + f = ixl_find_filter(vsi, macaddr, IXL_VLAN_ANY); + if (f != NULL) + return; + + f = ixl_get_filter(vsi); + if (f == NULL) { + printf("WARNING: no filter available!!\n"); + return; + } + bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN); + f->vlan = IXL_VLAN_ANY; + f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED + | IXL_FILTER_MC); + + return; +} + +static void +ixl_reconfigure_filters(struct ixl_vsi *vsi) +{ + + ixl_add_hw_filters(vsi, IXL_FILTER_USED, vsi->num_macs); +} + +/* +** This routine adds macvlan filters +*/ +static void +ixl_add_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan) +{ + struct ixl_mac_filter *f, *tmp; + struct ixl_pf *pf; + device_t dev; + + DEBUGOUT("ixl_add_filter: begin"); + + pf = vsi->back; + dev = pf->dev; + + /* Does one already exist */ + f = ixl_find_filter(vsi, macaddr, vlan); + if (f != NULL) + return; + /* + ** Is this the first vlan being registered, if so we + ** need to remove the ANY filter that indicates we are + ** not in a vlan, and replace that with a 0 filter. + */ + if ((vlan != IXL_VLAN_ANY) && (vsi->num_vlans == 1)) { + tmp = ixl_find_filter(vsi, macaddr, IXL_VLAN_ANY); + if (tmp != NULL) { + ixl_del_filter(vsi, macaddr, IXL_VLAN_ANY); + ixl_add_filter(vsi, macaddr, 0); + } + } + + f = ixl_get_filter(vsi); + if (f == NULL) { + device_printf(dev, "WARNING: no filter available!!\n"); + return; + } + bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN); + f->vlan = vlan; + f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED); + if (f->vlan != IXL_VLAN_ANY) + f->flags |= IXL_FILTER_VLAN; + else + vsi->num_macs++; + + ixl_add_hw_filters(vsi, f->flags, 1); + return; +} + +static void +ixl_del_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan) +{ + struct ixl_mac_filter *f; + + f = ixl_find_filter(vsi, macaddr, vlan); + if (f == NULL) + return; + + f->flags |= IXL_FILTER_DEL; + ixl_del_hw_filters(vsi, 1); + vsi->num_macs--; + + /* Check if this is the last vlan removal */ + if (vlan != IXL_VLAN_ANY && vsi->num_vlans == 0) { + /* Switch back to a non-vlan filter */ + ixl_del_filter(vsi, macaddr, 0); + ixl_add_filter(vsi, macaddr, IXL_VLAN_ANY); + } + return; +} + +/* +** Find the filter with both matching mac addr and vlan id +*/ +static struct ixl_mac_filter * +ixl_find_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan) +{ + struct ixl_mac_filter *f; + bool match = FALSE; + + SLIST_FOREACH(f, &vsi->ftl, next) { + if (!cmp_etheraddr(f->macaddr, macaddr)) + continue; + if (f->vlan == vlan) { + match = TRUE; + break; + } + } + + if (!match) + f = NULL; + return (f); +} + +/* +** This routine takes additions to the vsi filter +** table and creates an Admin Queue call to create +** the filters in the hardware. +*/ +static void +ixl_add_hw_filters(struct ixl_vsi *vsi, int flags, int cnt) +{ + struct i40e_aqc_add_macvlan_element_data *a, *b; + struct ixl_mac_filter *f; + struct ixl_pf *pf; + struct i40e_hw *hw; + device_t dev; + int err, j = 0; + + pf = vsi->back; + dev = pf->dev; + hw = &pf->hw; + IXL_PF_LOCK_ASSERT(pf); + + a = malloc(sizeof(struct i40e_aqc_add_macvlan_element_data) * cnt, + M_DEVBUF, M_NOWAIT | M_ZERO); + if (a == NULL) { + device_printf(dev, "add_hw_filters failed to get memory\n"); + return; + } + + /* + ** Scan the filter list, each time we find one + ** we add it to the admin queue array and turn off + ** the add bit. + */ + SLIST_FOREACH(f, &vsi->ftl, next) { + if (f->flags == flags) { + b = &a[j]; // a pox on fvl long names :) + bcopy(f->macaddr, b->mac_addr, ETHER_ADDR_LEN); + if (f->vlan == IXL_VLAN_ANY) { + b->vlan_tag = 0; + b->flags = I40E_AQC_MACVLAN_ADD_IGNORE_VLAN; + } else { + b->vlan_tag = f->vlan; + b->flags = 0; + } + b->flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH; + f->flags &= ~IXL_FILTER_ADD; + j++; + } + if (j == cnt) + break; + } + if (j > 0) { + err = i40e_aq_add_macvlan(hw, vsi->seid, a, j, NULL); + if (err) + device_printf(dev, "aq_add_macvlan err %d, " + "aq_error %d\n", err, hw->aq.asq_last_status); + else + vsi->hw_filters_add += j; + } + free(a, M_DEVBUF); + return; +} + +/* +** This routine takes removals in the vsi filter +** table and creates an Admin Queue call to delete +** the filters in the hardware. +*/ +static void +ixl_del_hw_filters(struct ixl_vsi *vsi, int cnt) +{ + struct i40e_aqc_remove_macvlan_element_data *d, *e; + struct ixl_pf *pf; + struct i40e_hw *hw; + device_t dev; + struct ixl_mac_filter *f, *f_temp; + int err, j = 0; + + DEBUGOUT("ixl_del_hw_filters: begin\n"); + + pf = vsi->back; + hw = &pf->hw; + dev = pf->dev; + + d = malloc(sizeof(struct i40e_aqc_remove_macvlan_element_data) * cnt, + M_DEVBUF, M_NOWAIT | M_ZERO); + if (d == NULL) { + printf("del hw filter failed to get memory\n"); + return; + } + + SLIST_FOREACH_SAFE(f, &vsi->ftl, next, f_temp) { + if (f->flags & IXL_FILTER_DEL) { + e = &d[j]; // a pox on fvl long names :) + bcopy(f->macaddr, e->mac_addr, ETHER_ADDR_LEN); + e->vlan_tag = (f->vlan == IXL_VLAN_ANY ? 0 : f->vlan); + e->flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; + /* delete entry from vsi list */ + SLIST_REMOVE(&vsi->ftl, f, ixl_mac_filter, next); + free(f, M_DEVBUF); + j++; + } + if (j == cnt) + break; + } + if (j > 0) { + err = i40e_aq_remove_macvlan(hw, vsi->seid, d, j, NULL); + /* NOTE: returns ENOENT every time but seems to work fine, + so we'll ignore that specific error. */ + // TODO: Does this still occur on current firmwares? + if (err && hw->aq.asq_last_status != I40E_AQ_RC_ENOENT) { + int sc = 0; + for (int i = 0; i < j; i++) + sc += (!d[i].error_code); + vsi->hw_filters_del += sc; + device_printf(dev, + "Failed to remove %d/%d filters, aq error %d\n", + j - sc, j, hw->aq.asq_last_status); + } else + vsi->hw_filters_del += j; + } + free(d, M_DEVBUF); + + DEBUGOUT("ixl_del_hw_filters: end\n"); + return; +} + +static int +ixl_enable_rings(struct ixl_vsi *vsi) +{ + struct ixl_pf *pf = vsi->back; + struct i40e_hw *hw = &pf->hw; + int index, error; + u32 reg; + + error = 0; + for (int i = 0; i < vsi->num_queues; i++) { + index = vsi->first_queue + i; + i40e_pre_tx_queue_cfg(hw, index, TRUE); + + reg = rd32(hw, I40E_QTX_ENA(index)); + reg |= I40E_QTX_ENA_QENA_REQ_MASK | + I40E_QTX_ENA_QENA_STAT_MASK; + wr32(hw, I40E_QTX_ENA(index), reg); + /* Verify the enable took */ + for (int j = 0; j < 10; j++) { + reg = rd32(hw, I40E_QTX_ENA(index)); + if (reg & I40E_QTX_ENA_QENA_STAT_MASK) + break; + i40e_msec_delay(10); + } + if ((reg & I40E_QTX_ENA_QENA_STAT_MASK) == 0) { + device_printf(pf->dev, "TX queue %d disabled!\n", + index); + error = ETIMEDOUT; + } + + reg = rd32(hw, I40E_QRX_ENA(index)); + reg |= I40E_QRX_ENA_QENA_REQ_MASK | + I40E_QRX_ENA_QENA_STAT_MASK; + wr32(hw, I40E_QRX_ENA(index), reg); + /* Verify the enable took */ + for (int j = 0; j < 10; j++) { + reg = rd32(hw, I40E_QRX_ENA(index)); + if (reg & I40E_QRX_ENA_QENA_STAT_MASK) + break; + i40e_msec_delay(10); + } + if ((reg & I40E_QRX_ENA_QENA_STAT_MASK) == 0) { + device_printf(pf->dev, "RX queue %d disabled!\n", + index); + error = ETIMEDOUT; + } + } + + return (error); +} + +static int +ixl_disable_rings(struct ixl_vsi *vsi) +{ + struct ixl_pf *pf = vsi->back; + struct i40e_hw *hw = &pf->hw; + int index, error; + u32 reg; + + error = 0; + for (int i = 0; i < vsi->num_queues; i++) { + index = vsi->first_queue + i; + + i40e_pre_tx_queue_cfg(hw, index, FALSE); + i40e_usec_delay(500); + + reg = rd32(hw, I40E_QTX_ENA(index)); + reg &= ~I40E_QTX_ENA_QENA_REQ_MASK; + wr32(hw, I40E_QTX_ENA(index), reg); + /* Verify the disable took */ + for (int j = 0; j < 10; j++) { + reg = rd32(hw, I40E_QTX_ENA(index)); + if (!(reg & I40E_QTX_ENA_QENA_STAT_MASK)) + break; + i40e_msec_delay(10); + } + if (reg & I40E_QTX_ENA_QENA_STAT_MASK) { + device_printf(pf->dev, "TX queue %d still enabled!\n", + index); + error = ETIMEDOUT; + } + + reg = rd32(hw, I40E_QRX_ENA(index)); + reg &= ~I40E_QRX_ENA_QENA_REQ_MASK; + wr32(hw, I40E_QRX_ENA(index), reg); + /* Verify the disable took */ + for (int j = 0; j < 10; j++) { + reg = rd32(hw, I40E_QRX_ENA(index)); + if (!(reg & I40E_QRX_ENA_QENA_STAT_MASK)) + break; + i40e_msec_delay(10); + } + if (reg & I40E_QRX_ENA_QENA_STAT_MASK) { + device_printf(pf->dev, "RX queue %d still enabled!\n", + index); + error = ETIMEDOUT; + } + } + + return (error); +} + +/** + * ixl_handle_mdd_event + * + * Called from interrupt handler to identify possibly malicious vfs + * (But also detects events from the PF, as well) + **/ +static void ixl_handle_mdd_event(struct ixl_pf *pf) +{ + struct i40e_hw *hw = &pf->hw; + device_t dev = pf->dev; + bool mdd_detected = false; + bool pf_mdd_detected = false; + u32 reg; + + /* find what triggered the MDD event */ + reg = rd32(hw, I40E_GL_MDET_TX); + if (reg & I40E_GL_MDET_TX_VALID_MASK) { + u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >> + I40E_GL_MDET_TX_PF_NUM_SHIFT; + u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >> + I40E_GL_MDET_TX_EVENT_SHIFT; + u8 queue = (reg & I40E_GL_MDET_TX_QUEUE_MASK) >> + I40E_GL_MDET_TX_QUEUE_SHIFT; + device_printf(dev, + "Malicious Driver Detection event 0x%02x" + " on TX queue %d pf number 0x%02x\n", + event, queue, pf_num); + wr32(hw, I40E_GL_MDET_TX, 0xffffffff); + mdd_detected = true; + } + reg = rd32(hw, I40E_GL_MDET_RX); + if (reg & I40E_GL_MDET_RX_VALID_MASK) { + u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >> + I40E_GL_MDET_RX_FUNCTION_SHIFT; + u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >> + I40E_GL_MDET_RX_EVENT_SHIFT; + u8 queue = (reg & I40E_GL_MDET_RX_QUEUE_MASK) >> + I40E_GL_MDET_RX_QUEUE_SHIFT; + device_printf(dev, + "Malicious Driver Detection event 0x%02x" + " on RX queue %d of function 0x%02x\n", + event, queue, func); + wr32(hw, I40E_GL_MDET_RX, 0xffffffff); + mdd_detected = true; + } + + if (mdd_detected) { + reg = rd32(hw, I40E_PF_MDET_TX); + if (reg & I40E_PF_MDET_TX_VALID_MASK) { + wr32(hw, I40E_PF_MDET_TX, 0xFFFF); + device_printf(dev, + "MDD TX event is for this function 0x%08x", + reg); + pf_mdd_detected = true; + } + reg = rd32(hw, I40E_PF_MDET_RX); + if (reg & I40E_PF_MDET_RX_VALID_MASK) { + wr32(hw, I40E_PF_MDET_RX, 0xFFFF); + device_printf(dev, + "MDD RX event is for this function 0x%08x", + reg); + pf_mdd_detected = true; + } + } + + /* re-enable mdd interrupt cause */ + reg = rd32(hw, I40E_PFINT_ICR0_ENA); + reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; + wr32(hw, I40E_PFINT_ICR0_ENA, reg); + ixl_flush(hw); +} + +static void +ixl_enable_intr(struct ixl_vsi *vsi) +{ + struct i40e_hw *hw = vsi->hw; + struct ixl_queue *que = vsi->queues; + + if (ixl_enable_msix) { + ixl_enable_adminq(hw); + for (int i = 0; i < vsi->num_queues; i++, que++) + ixl_enable_queue(hw, que->me); + } else + ixl_enable_legacy(hw); +} + +static void +ixl_disable_rings_intr(struct ixl_vsi *vsi) +{ + struct i40e_hw *hw = vsi->hw; + struct ixl_queue *que = vsi->queues; + + for (int i = 0; i < vsi->num_queues; i++, que++) + ixl_disable_queue(hw, que->me); +} + +static void +ixl_disable_intr(struct ixl_vsi *vsi) +{ + struct i40e_hw *hw = vsi->hw; + + if (ixl_enable_msix) + ixl_disable_adminq(hw); + else + ixl_disable_legacy(hw); +} + +static void +ixl_enable_adminq(struct i40e_hw *hw) +{ + u32 reg; + + reg = I40E_PFINT_DYN_CTL0_INTENA_MASK | + I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | + (IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT); + wr32(hw, I40E_PFINT_DYN_CTL0, reg); + ixl_flush(hw); + return; +} + +static void +ixl_disable_adminq(struct i40e_hw *hw) +{ + u32 reg; + + reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT; + wr32(hw, I40E_PFINT_DYN_CTL0, reg); + + return; +} + +static void +ixl_enable_queue(struct i40e_hw *hw, int id) +{ + u32 reg; + + reg = I40E_PFINT_DYN_CTLN_INTENA_MASK | + I40E_PFINT_DYN_CTLN_CLEARPBA_MASK | + (IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT); + wr32(hw, I40E_PFINT_DYN_CTLN(id), reg); +} + +static void +ixl_disable_queue(struct i40e_hw *hw, int id) +{ + u32 reg; + + reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT; + wr32(hw, I40E_PFINT_DYN_CTLN(id), reg); + + return; +} + +static void +ixl_enable_legacy(struct i40e_hw *hw) +{ + u32 reg; + reg = I40E_PFINT_DYN_CTL0_INTENA_MASK | + I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | + (IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT); + wr32(hw, I40E_PFINT_DYN_CTL0, reg); +} + +static void +ixl_disable_legacy(struct i40e_hw *hw) +{ + u32 reg; + + reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT; + wr32(hw, I40E_PFINT_DYN_CTL0, reg); + + return; +} + +static void +ixl_update_stats_counters(struct ixl_pf *pf) +{ + struct i40e_hw *hw = &pf->hw; + struct ixl_vsi *vsi = &pf->vsi; + struct ixl_vf *vf; + + struct i40e_hw_port_stats *nsd = &pf->stats; + struct i40e_hw_port_stats *osd = &pf->stats_offsets; + + /* Update hw stats */ + ixl_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port), + pf->stat_offsets_loaded, + &osd->crc_errors, &nsd->crc_errors); + ixl_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port), + pf->stat_offsets_loaded, + &osd->illegal_bytes, &nsd->illegal_bytes); + ixl_stat_update48(hw, I40E_GLPRT_GORCH(hw->port), + I40E_GLPRT_GORCL(hw->port), + pf->stat_offsets_loaded, + &osd->eth.rx_bytes, &nsd->eth.rx_bytes); + ixl_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port), + I40E_GLPRT_GOTCL(hw->port), + pf->stat_offsets_loaded, + &osd->eth.tx_bytes, &nsd->eth.tx_bytes); + ixl_stat_update32(hw, I40E_GLPRT_RDPC(hw->port), + pf->stat_offsets_loaded, + &osd->eth.rx_discards, + &nsd->eth.rx_discards); + ixl_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port), + I40E_GLPRT_UPRCL(hw->port), + pf->stat_offsets_loaded, + &osd->eth.rx_unicast, + &nsd->eth.rx_unicast); + ixl_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port), + I40E_GLPRT_UPTCL(hw->port), + pf->stat_offsets_loaded, + &osd->eth.tx_unicast, + &nsd->eth.tx_unicast); + ixl_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port), + I40E_GLPRT_MPRCL(hw->port), + pf->stat_offsets_loaded, + &osd->eth.rx_multicast, + &nsd->eth.rx_multicast); + ixl_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port), + I40E_GLPRT_MPTCL(hw->port), + pf->stat_offsets_loaded, + &osd->eth.tx_multicast, + &nsd->eth.tx_multicast); + ixl_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port), + I40E_GLPRT_BPRCL(hw->port), + pf->stat_offsets_loaded, + &osd->eth.rx_broadcast, + &nsd->eth.rx_broadcast); + ixl_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port), + I40E_GLPRT_BPTCL(hw->port), + pf->stat_offsets_loaded, + &osd->eth.tx_broadcast, + &nsd->eth.tx_broadcast); + + ixl_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port), + pf->stat_offsets_loaded, + &osd->tx_dropped_link_down, + &nsd->tx_dropped_link_down); + ixl_stat_update32(hw, I40E_GLPRT_MLFC(hw->port), + pf->stat_offsets_loaded, + &osd->mac_local_faults, + &nsd->mac_local_faults); + ixl_stat_update32(hw, I40E_GLPRT_MRFC(hw->port), + pf->stat_offsets_loaded, + &osd->mac_remote_faults, + &nsd->mac_remote_faults); + ixl_stat_update32(hw, I40E_GLPRT_RLEC(hw->port), + pf->stat_offsets_loaded, + &osd->rx_length_errors, + &nsd->rx_length_errors); + + /* Flow control (LFC) stats */ + ixl_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port), + pf->stat_offsets_loaded, + &osd->link_xon_rx, &nsd->link_xon_rx); + ixl_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port), + pf->stat_offsets_loaded, + &osd->link_xon_tx, &nsd->link_xon_tx); + ixl_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port), + pf->stat_offsets_loaded, + &osd->link_xoff_rx, &nsd->link_xoff_rx); + ixl_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port), + pf->stat_offsets_loaded, + &osd->link_xoff_tx, &nsd->link_xoff_tx); + + /* Packet size stats rx */ + ixl_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port), + I40E_GLPRT_PRC64L(hw->port), + pf->stat_offsets_loaded, + &osd->rx_size_64, &nsd->rx_size_64); + ixl_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port), + I40E_GLPRT_PRC127L(hw->port), + pf->stat_offsets_loaded, + &osd->rx_size_127, &nsd->rx_size_127); + ixl_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port), + I40E_GLPRT_PRC255L(hw->port), + pf->stat_offsets_loaded, + &osd->rx_size_255, &nsd->rx_size_255); + ixl_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port), + I40E_GLPRT_PRC511L(hw->port), + pf->stat_offsets_loaded, + &osd->rx_size_511, &nsd->rx_size_511); + ixl_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port), + I40E_GLPRT_PRC1023L(hw->port), + pf->stat_offsets_loaded, + &osd->rx_size_1023, &nsd->rx_size_1023); + ixl_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port), + I40E_GLPRT_PRC1522L(hw->port), + pf->stat_offsets_loaded, + &osd->rx_size_1522, &nsd->rx_size_1522); + ixl_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port), + I40E_GLPRT_PRC9522L(hw->port), + pf->stat_offsets_loaded, + &osd->rx_size_big, &nsd->rx_size_big); + + /* Packet size stats tx */ + ixl_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port), + I40E_GLPRT_PTC64L(hw->port), + pf->stat_offsets_loaded, + &osd->tx_size_64, &nsd->tx_size_64); + ixl_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port), + I40E_GLPRT_PTC127L(hw->port), + pf->stat_offsets_loaded, + &osd->tx_size_127, &nsd->tx_size_127); + ixl_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port), + I40E_GLPRT_PTC255L(hw->port), + pf->stat_offsets_loaded, + &osd->tx_size_255, &nsd->tx_size_255); + ixl_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port), + I40E_GLPRT_PTC511L(hw->port), + pf->stat_offsets_loaded, + &osd->tx_size_511, &nsd->tx_size_511); + ixl_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port), + I40E_GLPRT_PTC1023L(hw->port), + pf->stat_offsets_loaded, + &osd->tx_size_1023, &nsd->tx_size_1023); + ixl_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port), + I40E_GLPRT_PTC1522L(hw->port), + pf->stat_offsets_loaded, + &osd->tx_size_1522, &nsd->tx_size_1522); + ixl_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port), + I40E_GLPRT_PTC9522L(hw->port), + pf->stat_offsets_loaded, + &osd->tx_size_big, &nsd->tx_size_big); + + ixl_stat_update32(hw, I40E_GLPRT_RUC(hw->port), + pf->stat_offsets_loaded, + &osd->rx_undersize, &nsd->rx_undersize); + ixl_stat_update32(hw, I40E_GLPRT_RFC(hw->port), + pf->stat_offsets_loaded, + &osd->rx_fragments, &nsd->rx_fragments); + ixl_stat_update32(hw, I40E_GLPRT_ROC(hw->port), + pf->stat_offsets_loaded, + &osd->rx_oversize, &nsd->rx_oversize); + ixl_stat_update32(hw, I40E_GLPRT_RJC(hw->port), + pf->stat_offsets_loaded, + &osd->rx_jabber, &nsd->rx_jabber); + pf->stat_offsets_loaded = true; + /* End hw stats */ + + /* Update vsi stats */ + ixl_update_vsi_stats(vsi); + + for (int i = 0; i < pf->num_vfs; i++) { + vf = &pf->vfs[i]; + if (vf->vf_flags & VF_FLAG_ENABLED) + ixl_update_eth_stats(&pf->vfs[i].vsi); + } +} + +/* +** Tasklet handler for MSIX Adminq interrupts +** - do outside interrupt since it might sleep +*/ +static void +ixl_do_adminq(void *context, int pending) +{ + struct ixl_pf *pf = context; + struct i40e_hw *hw = &pf->hw; + struct ixl_vsi *vsi = &pf->vsi; + struct i40e_arq_event_info event; + i40e_status ret; + u32 reg, loop = 0; + u16 opcode, result; + + event.buf_len = IXL_AQ_BUF_SZ; + event.msg_buf = malloc(event.buf_len, + M_DEVBUF, M_NOWAIT | M_ZERO); + if (!event.msg_buf) { + printf("Unable to allocate adminq memory\n"); + return; + } + + IXL_PF_LOCK(pf); + /* clean and process any events */ + do { + ret = i40e_clean_arq_element(hw, &event, &result); + if (ret) + break; + opcode = LE16_TO_CPU(event.desc.opcode); + switch (opcode) { + case i40e_aqc_opc_get_link_status: + ixl_link_event(pf, &event); + ixl_update_link_status(pf); + break; + case i40e_aqc_opc_send_msg_to_pf: +#ifdef PCI_IOV + ixl_handle_vf_msg(pf, &event); +#endif + break; + case i40e_aqc_opc_event_lan_overflow: + break; + default: +#ifdef IXL_DEBUG + printf("AdminQ unknown event %x\n", opcode); +#endif + break; + } + + } while (result && (loop++ < IXL_ADM_LIMIT)); + + reg = rd32(hw, I40E_PFINT_ICR0_ENA); + reg |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK; + wr32(hw, I40E_PFINT_ICR0_ENA, reg); + free(event.msg_buf, M_DEVBUF); + + /* + * If there are still messages to process, reschedule ourselves. + * Otherwise, re-enable our interrupt and go to sleep. + */ + if (result > 0) + taskqueue_enqueue(pf->tq, &pf->adminq); + else + ixl_enable_intr(vsi); + + IXL_PF_UNLOCK(pf); +} + +#ifdef IXL_DEBUG_SYSCTL +static int +ixl_debug_info(SYSCTL_HANDLER_ARGS) +{ + struct ixl_pf *pf; + int error, input = 0; + + error = sysctl_handle_int(oidp, &input, 0, req); + + if (error || !req->newptr) + return (error); + + if (input == 1) { + pf = (struct ixl_pf *)arg1; + ixl_print_debug_info(pf); + } + + return (error); +} + +static void +ixl_print_debug_info(struct ixl_pf *pf) +{ + struct i40e_hw *hw = &pf->hw; + struct ixl_vsi *vsi = &pf->vsi; + struct ixl_queue *que = vsi->queues; + struct rx_ring *rxr = &que->rxr; + struct tx_ring *txr = &que->txr; + u32 reg; + + + printf("Queue irqs = %jx\n", (uintmax_t)que->irqs); + printf("AdminQ irqs = %jx\n", (uintmax_t)pf->admin_irq); + printf("RX next check = %x\n", rxr->next_check); + printf("RX not ready = %jx\n", (uintmax_t)rxr->not_done); + printf("RX packets = %jx\n", (uintmax_t)rxr->rx_packets); + printf("TX desc avail = %x\n", txr->avail); + + reg = rd32(hw, I40E_GLV_GORCL(0xc)); + printf("RX Bytes = %x\n", reg); + reg = rd32(hw, I40E_GLPRT_GORCL(hw->port)); + printf("Port RX Bytes = %x\n", reg); + reg = rd32(hw, I40E_GLV_RDPC(0xc)); + printf("RX discard = %x\n", reg); + reg = rd32(hw, I40E_GLPRT_RDPC(hw->port)); + printf("Port RX discard = %x\n", reg); + + reg = rd32(hw, I40E_GLV_TEPC(0xc)); + printf("TX errors = %x\n", reg); + reg = rd32(hw, I40E_GLV_GOTCL(0xc)); + printf("TX Bytes = %x\n", reg); + + reg = rd32(hw, I40E_GLPRT_RUC(hw->port)); + printf("RX undersize = %x\n", reg); + reg = rd32(hw, I40E_GLPRT_RFC(hw->port)); + printf("RX fragments = %x\n", reg); + reg = rd32(hw, I40E_GLPRT_ROC(hw->port)); + printf("RX oversize = %x\n", reg); + reg = rd32(hw, I40E_GLPRT_RLEC(hw->port)); + printf("RX length error = %x\n", reg); + reg = rd32(hw, I40E_GLPRT_MRFC(hw->port)); + printf("mac remote fault = %x\n", reg); + reg = rd32(hw, I40E_GLPRT_MLFC(hw->port)); + printf("mac local fault = %x\n", reg); +} +#endif + +/** + * Update VSI-specific ethernet statistics counters. + **/ +void ixl_update_eth_stats(struct ixl_vsi *vsi) +{ + struct ixl_pf *pf = (struct ixl_pf *)vsi->back; + struct i40e_hw *hw = &pf->hw; + struct i40e_eth_stats *es; + struct i40e_eth_stats *oes; + struct i40e_hw_port_stats *nsd; + u16 stat_idx = vsi->info.stat_counter_idx; + + es = &vsi->eth_stats; + oes = &vsi->eth_stats_offsets; + nsd = &pf->stats; + + /* Gather up the stats that the hw collects */ + ixl_stat_update32(hw, I40E_GLV_TEPC(stat_idx), + vsi->stat_offsets_loaded, + &oes->tx_errors, &es->tx_errors); + ixl_stat_update32(hw, I40E_GLV_RDPC(stat_idx), + vsi->stat_offsets_loaded, + &oes->rx_discards, &es->rx_discards); + + ixl_stat_update48(hw, I40E_GLV_GORCH(stat_idx), + I40E_GLV_GORCL(stat_idx), + vsi->stat_offsets_loaded, + &oes->rx_bytes, &es->rx_bytes); + ixl_stat_update48(hw, I40E_GLV_UPRCH(stat_idx), + I40E_GLV_UPRCL(stat_idx), + vsi->stat_offsets_loaded, + &oes->rx_unicast, &es->rx_unicast); + ixl_stat_update48(hw, I40E_GLV_MPRCH(stat_idx), + I40E_GLV_MPRCL(stat_idx), + vsi->stat_offsets_loaded, + &oes->rx_multicast, &es->rx_multicast); + ixl_stat_update48(hw, I40E_GLV_BPRCH(stat_idx), + I40E_GLV_BPRCL(stat_idx), + vsi->stat_offsets_loaded, + &oes->rx_broadcast, &es->rx_broadcast); + + ixl_stat_update48(hw, I40E_GLV_GOTCH(stat_idx), + I40E_GLV_GOTCL(stat_idx), + vsi->stat_offsets_loaded, + &oes->tx_bytes, &es->tx_bytes); + ixl_stat_update48(hw, I40E_GLV_UPTCH(stat_idx), + I40E_GLV_UPTCL(stat_idx), + vsi->stat_offsets_loaded, + &oes->tx_unicast, &es->tx_unicast); + ixl_stat_update48(hw, I40E_GLV_MPTCH(stat_idx), + I40E_GLV_MPTCL(stat_idx), + vsi->stat_offsets_loaded, + &oes->tx_multicast, &es->tx_multicast); + ixl_stat_update48(hw, I40E_GLV_BPTCH(stat_idx), + I40E_GLV_BPTCL(stat_idx), + vsi->stat_offsets_loaded, + &oes->tx_broadcast, &es->tx_broadcast); + vsi->stat_offsets_loaded = true; +} + +static void +ixl_update_vsi_stats(struct ixl_vsi *vsi) +{ + struct ixl_pf *pf; + struct ifnet *ifp; + struct i40e_eth_stats *es; + u64 tx_discards; + + struct i40e_hw_port_stats *nsd; + + pf = vsi->back; + ifp = vsi->ifp; + es = &vsi->eth_stats; + nsd = &pf->stats; + + ixl_update_eth_stats(vsi); + + tx_discards = es->tx_discards + nsd->tx_dropped_link_down; + for (int i = 0; i < vsi->num_queues; i++) + tx_discards += vsi->queues[i].txr.br->br_drops; + + /* Update ifnet stats */ + IXL_SET_IPACKETS(vsi, es->rx_unicast + + es->rx_multicast + + es->rx_broadcast); + IXL_SET_OPACKETS(vsi, es->tx_unicast + + es->tx_multicast + + es->tx_broadcast); + IXL_SET_IBYTES(vsi, es->rx_bytes); + IXL_SET_OBYTES(vsi, es->tx_bytes); + IXL_SET_IMCASTS(vsi, es->rx_multicast); + IXL_SET_OMCASTS(vsi, es->tx_multicast); + + IXL_SET_IERRORS(vsi, nsd->crc_errors + nsd->illegal_bytes + + nsd->rx_undersize + nsd->rx_oversize + nsd->rx_fragments + + nsd->rx_jabber); + IXL_SET_OERRORS(vsi, es->tx_errors); + IXL_SET_IQDROPS(vsi, es->rx_discards + nsd->eth.rx_discards); + IXL_SET_OQDROPS(vsi, tx_discards); + IXL_SET_NOPROTO(vsi, es->rx_unknown_protocol); + IXL_SET_COLLISIONS(vsi, 0); +} + +/** + * Reset all of the stats for the given pf + **/ +void ixl_pf_reset_stats(struct ixl_pf *pf) +{ + bzero(&pf->stats, sizeof(struct i40e_hw_port_stats)); + bzero(&pf->stats_offsets, sizeof(struct i40e_hw_port_stats)); + pf->stat_offsets_loaded = false; +} + +/** + * Resets all stats of the given vsi + **/ +void ixl_vsi_reset_stats(struct ixl_vsi *vsi) +{ + bzero(&vsi->eth_stats, sizeof(struct i40e_eth_stats)); + bzero(&vsi->eth_stats_offsets, sizeof(struct i40e_eth_stats)); + vsi->stat_offsets_loaded = false; +} + +/** + * Read and update a 48 bit stat from the hw + * + * Since the device stats are not reset at PFReset, they likely will not + * be zeroed when the driver starts. We'll save the first values read + * and use them as offsets to be subtracted from the raw values in order + * to report stats that count from zero. + **/ +static void +ixl_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg, + bool offset_loaded, u64 *offset, u64 *stat) +{ + u64 new_data; + +#if defined(__FreeBSD__) && (__FreeBSD_version >= 1000000) && defined(__amd64__) + new_data = rd64(hw, loreg); +#else + /* + * Use two rd32's instead of one rd64; FreeBSD versions before + * 10 don't support 8 byte bus reads/writes. + */ + new_data = rd32(hw, loreg); + new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32; +#endif + + if (!offset_loaded) + *offset = new_data; + if (new_data >= *offset) + *stat = new_data - *offset; + else + *stat = (new_data + ((u64)1 << 48)) - *offset; + *stat &= 0xFFFFFFFFFFFFULL; +} + +/** + * Read and update a 32 bit stat from the hw + **/ +static void +ixl_stat_update32(struct i40e_hw *hw, u32 reg, + bool offset_loaded, u64 *offset, u64 *stat) +{ + u32 new_data; + + new_data = rd32(hw, reg); + if (!offset_loaded) + *offset = new_data; + if (new_data >= *offset) + *stat = (u32)(new_data - *offset); + else + *stat = (u32)((new_data + ((u64)1 << 32)) - *offset); +} + +/* +** Set flow control using sysctl: +** 0 - off +** 1 - rx pause +** 2 - tx pause +** 3 - full +*/ +static int +ixl_set_flowcntl(SYSCTL_HANDLER_ARGS) +{ + /* + * TODO: ensure flow control is disabled if + * priority flow control is enabled + * + * TODO: ensure tx CRC by hardware should be enabled + * if tx flow control is enabled. + */ + struct ixl_pf *pf = (struct ixl_pf *)arg1; + struct i40e_hw *hw = &pf->hw; + device_t dev = pf->dev; + int error = 0; + enum i40e_status_code aq_error = 0; + u8 fc_aq_err = 0; + + /* Get request */ + error = sysctl_handle_int(oidp, &pf->fc, 0, req); + if ((error) || (req->newptr == NULL)) + return (error); + if (pf->fc < 0 || pf->fc > 3) { + device_printf(dev, + "Invalid fc mode; valid modes are 0 through 3\n"); + return (EINVAL); + } + + /* + ** Changing flow control mode currently does not work on + ** 40GBASE-CR4 PHYs + */ + if (hw->phy.link_info.phy_type == I40E_PHY_TYPE_40GBASE_CR4 + || hw->phy.link_info.phy_type == I40E_PHY_TYPE_40GBASE_CR4_CU) { + device_printf(dev, "Changing flow control mode unsupported" + " on 40GBase-CR4 media.\n"); + return (ENODEV); + } + + /* Set fc ability for port */ + hw->fc.requested_mode = pf->fc; + aq_error = i40e_set_fc(hw, &fc_aq_err, TRUE); + if (aq_error) { + device_printf(dev, + "%s: Error setting new fc mode %d; fc_err %#x\n", + __func__, aq_error, fc_aq_err); + return (EAGAIN); + } + + return (0); +} + +static int +ixl_current_speed(SYSCTL_HANDLER_ARGS) +{ + struct ixl_pf *pf = (struct ixl_pf *)arg1; + struct i40e_hw *hw = &pf->hw; + int error = 0, index = 0; + + char *speeds[] = { + "Unknown", + "100M", + "1G", + "10G", + "40G", + "20G" + }; + + ixl_update_link_status(pf); + + switch (hw->phy.link_info.link_speed) { + case I40E_LINK_SPEED_100MB: + index = 1; + break; + case I40E_LINK_SPEED_1GB: + index = 2; + break; + case I40E_LINK_SPEED_10GB: + index = 3; + break; + case I40E_LINK_SPEED_40GB: + index = 4; + break; + case I40E_LINK_SPEED_20GB: + index = 5; + break; + case I40E_LINK_SPEED_UNKNOWN: + default: + index = 0; + break; + } + + error = sysctl_handle_string(oidp, speeds[index], + strlen(speeds[index]), req); + return (error); +} + +static int +ixl_set_advertised_speeds(struct ixl_pf *pf, int speeds) +{ + struct i40e_hw *hw = &pf->hw; + device_t dev = pf->dev; + struct i40e_aq_get_phy_abilities_resp abilities; + struct i40e_aq_set_phy_config config; + enum i40e_status_code aq_error = 0; + + /* Get current capability information */ + aq_error = i40e_aq_get_phy_capabilities(hw, + FALSE, FALSE, &abilities, NULL); + if (aq_error) { + device_printf(dev, + "%s: Error getting phy capabilities %d," + " aq error: %d\n", __func__, aq_error, + hw->aq.asq_last_status); + return (EAGAIN); + } + + /* Prepare new config */ + bzero(&config, sizeof(config)); + config.phy_type = abilities.phy_type; + config.abilities = abilities.abilities + | I40E_AQ_PHY_ENABLE_ATOMIC_LINK; + config.eee_capability = abilities.eee_capability; + config.eeer = abilities.eeer_val; + config.low_power_ctrl = abilities.d3_lpan; + /* Translate into aq cmd link_speed */ + if (speeds & 0x8) + config.link_speed |= I40E_LINK_SPEED_20GB; + if (speeds & 0x4) + config.link_speed |= I40E_LINK_SPEED_10GB; + if (speeds & 0x2) + config.link_speed |= I40E_LINK_SPEED_1GB; + if (speeds & 0x1) + config.link_speed |= I40E_LINK_SPEED_100MB; + + /* Do aq command & restart link */ + aq_error = i40e_aq_set_phy_config(hw, &config, NULL); + if (aq_error) { + device_printf(dev, + "%s: Error setting new phy config %d," + " aq error: %d\n", __func__, aq_error, + hw->aq.asq_last_status); + return (EAGAIN); + } + + /* + ** This seems a bit heavy handed, but we + ** need to get a reinit on some devices + */ + IXL_PF_LOCK(pf); + ixl_stop(pf); + ixl_init_locked(pf); + IXL_PF_UNLOCK(pf); + + return (0); +} + +/* +** Control link advertise speed: +** Flags: +** 0x1 - advertise 100 Mb +** 0x2 - advertise 1G +** 0x4 - advertise 10G +** 0x8 - advertise 20G +** +** Does not work on 40G devices. +*/ +static int +ixl_set_advertise(SYSCTL_HANDLER_ARGS) +{ + struct ixl_pf *pf = (struct ixl_pf *)arg1; + struct i40e_hw *hw = &pf->hw; + device_t dev = pf->dev; + int requested_ls = 0; + int error = 0; + + /* + ** FW doesn't support changing advertised speed + ** for 40G devices; speed is always 40G. + */ + if (i40e_is_40G_device(hw->device_id)) + return (ENODEV); + + /* Read in new mode */ + requested_ls = pf->advertised_speed; + error = sysctl_handle_int(oidp, &requested_ls, 0, req); + if ((error) || (req->newptr == NULL)) + return (error); + /* Check for sane value */ + if (requested_ls < 0x1 || requested_ls > 0xE) { + device_printf(dev, "Invalid advertised speed; " + "valid modes are 0x1 through 0xE\n"); + return (EINVAL); + } + /* Then check for validity based on adapter type */ + switch (hw->device_id) { + case I40E_DEV_ID_10G_BASE_T: + if (requested_ls & 0x8) { + device_printf(dev, + "20Gbs speed not supported on this device.\n"); + return (EINVAL); + } + break; + case I40E_DEV_ID_20G_KR2: + if (requested_ls & 0x1) { + device_printf(dev, + "100Mbs speed not supported on this device.\n"); + return (EINVAL); + } + break; + default: + if (requested_ls & ~0x6) { + device_printf(dev, + "Only 1/10Gbs speeds are supported on this device.\n"); + return (EINVAL); + } + break; + } + + /* Exit if no change */ + if (pf->advertised_speed == requested_ls) + return (0); + + error = ixl_set_advertised_speeds(pf, requested_ls); + if (error) + return (error); + + pf->advertised_speed = requested_ls; + ixl_update_link_status(pf); + return (0); +} + +/* +** Get the width and transaction speed of +** the bus this adapter is plugged into. +*/ +static u16 +ixl_get_bus_info(struct i40e_hw *hw, device_t dev) +{ + u16 link; + u32 offset; + + + /* Get the PCI Express Capabilities offset */ + pci_find_cap(dev, PCIY_EXPRESS, &offset); + + /* ...and read the Link Status Register */ + link = pci_read_config(dev, offset + PCIER_LINK_STA, 2); + + switch (link & I40E_PCI_LINK_WIDTH) { + case I40E_PCI_LINK_WIDTH_1: + hw->bus.width = i40e_bus_width_pcie_x1; + break; + case I40E_PCI_LINK_WIDTH_2: + hw->bus.width = i40e_bus_width_pcie_x2; + break; + case I40E_PCI_LINK_WIDTH_4: + hw->bus.width = i40e_bus_width_pcie_x4; + break; + case I40E_PCI_LINK_WIDTH_8: + hw->bus.width = i40e_bus_width_pcie_x8; + break; + default: + hw->bus.width = i40e_bus_width_unknown; + break; + } + + switch (link & I40E_PCI_LINK_SPEED) { + case I40E_PCI_LINK_SPEED_2500: + hw->bus.speed = i40e_bus_speed_2500; + break; + case I40E_PCI_LINK_SPEED_5000: + hw->bus.speed = i40e_bus_speed_5000; + break; + case I40E_PCI_LINK_SPEED_8000: + hw->bus.speed = i40e_bus_speed_8000; + break; + default: + hw->bus.speed = i40e_bus_speed_unknown; + break; + } + + + device_printf(dev,"PCI Express Bus: Speed %s %s\n", + ((hw->bus.speed == i40e_bus_speed_8000) ? "8.0GT/s": + (hw->bus.speed == i40e_bus_speed_5000) ? "5.0GT/s": + (hw->bus.speed == i40e_bus_speed_2500) ? "2.5GT/s":"Unknown"), + (hw->bus.width == i40e_bus_width_pcie_x8) ? "Width x8" : + (hw->bus.width == i40e_bus_width_pcie_x4) ? "Width x4" : + (hw->bus.width == i40e_bus_width_pcie_x1) ? "Width x1" : + ("Unknown")); + + if ((hw->bus.width <= i40e_bus_width_pcie_x8) && + (hw->bus.speed < i40e_bus_speed_8000)) { + device_printf(dev, "PCI-Express bandwidth available" + " for this device\n may be insufficient for" + " optimal performance.\n"); + device_printf(dev, "For expected performance a x8 " + "PCIE Gen3 slot is required.\n"); + } + + return (link); +} + +static int +ixl_sysctl_show_fw(SYSCTL_HANDLER_ARGS) +{ + struct ixl_pf *pf = (struct ixl_pf *)arg1; + struct i40e_hw *hw = &pf->hw; + char buf[32]; + + snprintf(buf, sizeof(buf), + "f%d.%d a%d.%d n%02x.%02x e%08x", + hw->aq.fw_maj_ver, hw->aq.fw_min_ver, + hw->aq.api_maj_ver, hw->aq.api_min_ver, + (hw->nvm.version & IXL_NVM_VERSION_HI_MASK) >> + IXL_NVM_VERSION_HI_SHIFT, + (hw->nvm.version & IXL_NVM_VERSION_LO_MASK) >> + IXL_NVM_VERSION_LO_SHIFT, + hw->nvm.eetrack); + return (sysctl_handle_string(oidp, buf, strlen(buf), req)); +} + + +#ifdef IXL_DEBUG_SYSCTL +static int +ixl_sysctl_link_status(SYSCTL_HANDLER_ARGS) +{ + struct ixl_pf *pf = (struct ixl_pf *)arg1; + struct i40e_hw *hw = &pf->hw; + struct i40e_link_status link_status; + char buf[512]; + + enum i40e_status_code aq_error = 0; + + aq_error = i40e_aq_get_link_info(hw, TRUE, &link_status, NULL); + if (aq_error) { + printf("i40e_aq_get_link_info() error %d\n", aq_error); + return (EPERM); + } + + sprintf(buf, "\n" + "PHY Type : %#04x\n" + "Speed : %#04x\n" + "Link info: %#04x\n" + "AN info : %#04x\n" + "Ext info : %#04x", + link_status.phy_type, link_status.link_speed, + link_status.link_info, link_status.an_info, + link_status.ext_info); + + return (sysctl_handle_string(oidp, buf, strlen(buf), req)); +} + +static int +ixl_sysctl_phy_abilities(SYSCTL_HANDLER_ARGS) +{ + struct ixl_pf *pf = (struct ixl_pf *)arg1; + struct i40e_hw *hw = &pf->hw; + char buf[512]; + enum i40e_status_code aq_error = 0; + + struct i40e_aq_get_phy_abilities_resp abilities; + + aq_error = i40e_aq_get_phy_capabilities(hw, + TRUE, FALSE, &abilities, NULL); + if (aq_error) { + printf("i40e_aq_get_phy_capabilities() error %d\n", aq_error); + return (EPERM); + } + + sprintf(buf, "\n" + "PHY Type : %#010x\n" + "Speed : %#04x\n" + "Abilities: %#04x\n" + "EEE cap : %#06x\n" + "EEER reg : %#010x\n" + "D3 Lpan : %#04x", + abilities.phy_type, abilities.link_speed, + abilities.abilities, abilities.eee_capability, + abilities.eeer_val, abilities.d3_lpan); + + return (sysctl_handle_string(oidp, buf, strlen(buf), req)); +} + +static int +ixl_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS) +{ + struct ixl_pf *pf = (struct ixl_pf *)arg1; + struct ixl_vsi *vsi = &pf->vsi; + struct ixl_mac_filter *f; + char *buf, *buf_i; + + int error = 0; + int ftl_len = 0; + int ftl_counter = 0; + int buf_len = 0; + int entry_len = 42; + + SLIST_FOREACH(f, &vsi->ftl, next) { + ftl_len++; + } + + if (ftl_len < 1) { + sysctl_handle_string(oidp, "(none)", 6, req); + return (0); + } + + buf_len = sizeof(char) * (entry_len + 1) * ftl_len + 2; + buf = buf_i = malloc(buf_len, M_DEVBUF, M_NOWAIT); + + sprintf(buf_i++, "\n"); + SLIST_FOREACH(f, &vsi->ftl, next) { + sprintf(buf_i, + MAC_FORMAT ", vlan %4d, flags %#06x", + MAC_FORMAT_ARGS(f->macaddr), f->vlan, f->flags); + buf_i += entry_len; + /* don't print '\n' for last entry */ + if (++ftl_counter != ftl_len) { + sprintf(buf_i, "\n"); + buf_i++; + } + } + + error = sysctl_handle_string(oidp, buf, strlen(buf), req); + if (error) + printf("sysctl error: %d\n", error); + free(buf, M_DEVBUF); + return error; +} + +#define IXL_SW_RES_SIZE 0x14 +static int +ixl_res_alloc_cmp(const void *a, const void *b) +{ + const struct i40e_aqc_switch_resource_alloc_element_resp *one, *two; + one = (const struct i40e_aqc_switch_resource_alloc_element_resp *)a; + two = (const struct i40e_aqc_switch_resource_alloc_element_resp *)b; + + return ((int)one->resource_type - (int)two->resource_type); +} + +static int +ixl_sysctl_hw_res_alloc(SYSCTL_HANDLER_ARGS) +{ + struct ixl_pf *pf = (struct ixl_pf *)arg1; + struct i40e_hw *hw = &pf->hw; + device_t dev = pf->dev; + struct sbuf *buf; + int error = 0; + + u8 num_entries; + struct i40e_aqc_switch_resource_alloc_element_resp resp[IXL_SW_RES_SIZE]; + + buf = sbuf_new_for_sysctl(NULL, NULL, 0, req); + if (!buf) { + device_printf(dev, "Could not allocate sbuf for output.\n"); + return (ENOMEM); + } + + bzero(resp, sizeof(resp)); + error = i40e_aq_get_switch_resource_alloc(hw, &num_entries, + resp, + IXL_SW_RES_SIZE, + NULL); + if (error) { + device_printf(dev, + "%s: get_switch_resource_alloc() error %d, aq error %d\n", + __func__, error, hw->aq.asq_last_status); + sbuf_delete(buf); + return error; + } + + /* Sort entries by type for display */ + qsort(resp, num_entries, + sizeof(struct i40e_aqc_switch_resource_alloc_element_resp), + &ixl_res_alloc_cmp); + + sbuf_cat(buf, "\n"); + sbuf_printf(buf, "# of entries: %d\n", num_entries); + sbuf_printf(buf, + "Type | Guaranteed | Total | Used | Un-allocated\n" + " | (this) | (all) | (this) | (all) \n"); + for (int i = 0; i < num_entries; i++) { + sbuf_printf(buf, + "%#4x | %10d %5d %6d %12d", + resp[i].resource_type, + resp[i].guaranteed, + resp[i].total, + resp[i].used, + resp[i].total_unalloced); + if (i < num_entries - 1) + sbuf_cat(buf, "\n"); + } + + error = sbuf_finish(buf); + sbuf_delete(buf); + + return (error); +} + +/* +** Caller must init and delete sbuf; this function will clear and +** finish it for caller. +*/ +static char * +ixl_switch_element_string(struct sbuf *s, u16 seid, bool uplink) +{ + sbuf_clear(s); + + if (seid == 0 && uplink) + sbuf_cat(s, "Network"); + else if (seid == 0) + sbuf_cat(s, "Host"); + else if (seid == 1) + sbuf_cat(s, "EMP"); + else if (seid <= 5) + sbuf_printf(s, "MAC %d", seid - 2); + else if (seid <= 15) + sbuf_cat(s, "Reserved"); + else if (seid <= 31) + sbuf_printf(s, "PF %d", seid - 16); + else if (seid <= 159) + sbuf_printf(s, "VF %d", seid - 32); + else if (seid <= 287) + sbuf_cat(s, "Reserved"); + else if (seid <= 511) + sbuf_cat(s, "Other"); // for other structures + else if (seid <= 895) + sbuf_printf(s, "VSI %d", seid - 512); + else if (seid <= 1023) + sbuf_printf(s, "Reserved"); + else + sbuf_cat(s, "Invalid"); + + sbuf_finish(s); + return sbuf_data(s); +} + +static int +ixl_sysctl_switch_config(SYSCTL_HANDLER_ARGS) +{ + struct ixl_pf *pf = (struct ixl_pf *)arg1; + struct i40e_hw *hw = &pf->hw; + device_t dev = pf->dev; + struct sbuf *buf; + struct sbuf *nmbuf; + int error = 0; + u8 aq_buf[I40E_AQ_LARGE_BUF]; + + u16 next = 0; + struct i40e_aqc_get_switch_config_resp *sw_config; + sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf; + + buf = sbuf_new_for_sysctl(NULL, NULL, 0, req); + if (!buf) { + device_printf(dev, "Could not allocate sbuf for sysctl output.\n"); + return (ENOMEM); + } + + error = i40e_aq_get_switch_config(hw, sw_config, + sizeof(aq_buf), &next, NULL); + if (error) { + device_printf(dev, + "%s: aq_get_switch_config() error %d, aq error %d\n", + __func__, error, hw->aq.asq_last_status); + sbuf_delete(buf); + return error; + } + + nmbuf = sbuf_new_auto(); + if (!nmbuf) { + device_printf(dev, "Could not allocate sbuf for name output.\n"); + return (ENOMEM); + } + + sbuf_cat(buf, "\n"); + // Assuming <= 255 elements in switch + sbuf_printf(buf, "# of elements: %d\n", sw_config->header.num_reported); + /* Exclude: + ** Revision -- all elements are revision 1 for now + */ + sbuf_printf(buf, + "SEID ( Name ) | Uplink | Downlink | Conn Type\n" + " | | | (uplink)\n"); + for (int i = 0; i < sw_config->header.num_reported; i++) { + // "%4d (%8s) | %8s %8s %#8x", + sbuf_printf(buf, "%4d", sw_config->element[i].seid); + sbuf_cat(buf, " "); + sbuf_printf(buf, "(%8s)", ixl_switch_element_string(nmbuf, + sw_config->element[i].seid, false)); + sbuf_cat(buf, " | "); + sbuf_printf(buf, "%8s", ixl_switch_element_string(nmbuf, + sw_config->element[i].uplink_seid, true)); + sbuf_cat(buf, " "); + sbuf_printf(buf, "%8s", ixl_switch_element_string(nmbuf, + sw_config->element[i].downlink_seid, false)); + sbuf_cat(buf, " "); + sbuf_printf(buf, "%#8x", sw_config->element[i].connection_type); + if (i < sw_config->header.num_reported - 1) + sbuf_cat(buf, "\n"); + } + sbuf_delete(nmbuf); + + error = sbuf_finish(buf); + sbuf_delete(buf); + + return (error); +} +#endif /* IXL_DEBUG_SYSCTL */ + + +#ifdef PCI_IOV +static int +ixl_vf_alloc_vsi(struct ixl_pf *pf, struct ixl_vf *vf) +{ + struct i40e_hw *hw; + struct ixl_vsi *vsi; + struct i40e_vsi_context vsi_ctx; + int i; + uint16_t first_queue; + enum i40e_status_code code; + + hw = &pf->hw; + vsi = &pf->vsi; + + vsi_ctx.pf_num = hw->pf_id; + vsi_ctx.uplink_seid = pf->veb_seid; + vsi_ctx.connection_type = IXL_VSI_DATA_PORT; + vsi_ctx.vf_num = hw->func_caps.vf_base_id + vf->vf_num; + vsi_ctx.flags = I40E_AQ_VSI_TYPE_VF; + + bzero(&vsi_ctx.info, sizeof(vsi_ctx.info)); + + vsi_ctx.info.valid_sections = htole16(I40E_AQ_VSI_PROP_SWITCH_VALID); + vsi_ctx.info.switch_id = htole16(0); + + vsi_ctx.info.valid_sections |= htole16(I40E_AQ_VSI_PROP_SECURITY_VALID); + vsi_ctx.info.sec_flags = 0; + if (vf->vf_flags & VF_FLAG_MAC_ANTI_SPOOF) + vsi_ctx.info.sec_flags |= I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK; + + vsi_ctx.info.valid_sections |= htole16(I40E_AQ_VSI_PROP_VLAN_VALID); + vsi_ctx.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | + I40E_AQ_VSI_PVLAN_EMOD_NOTHING; + + vsi_ctx.info.valid_sections |= + htole16(I40E_AQ_VSI_PROP_QUEUE_MAP_VALID); + vsi_ctx.info.mapping_flags = htole16(I40E_AQ_VSI_QUE_MAP_NONCONTIG); + first_queue = vsi->num_queues + vf->vf_num * IXLV_MAX_QUEUES; + for (i = 0; i < IXLV_MAX_QUEUES; i++) + vsi_ctx.info.queue_mapping[i] = htole16(first_queue + i); + for (; i < nitems(vsi_ctx.info.queue_mapping); i++) + vsi_ctx.info.queue_mapping[i] = htole16(I40E_AQ_VSI_QUEUE_MASK); + + vsi_ctx.info.tc_mapping[0] = htole16( + (0 << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | + (1 << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT)); + + code = i40e_aq_add_vsi(hw, &vsi_ctx, NULL); + if (code != I40E_SUCCESS) + return (ixl_adminq_err_to_errno(hw->aq.asq_last_status)); + vf->vsi.seid = vsi_ctx.seid; + vf->vsi.vsi_num = vsi_ctx.vsi_number; + vf->vsi.first_queue = first_queue; + vf->vsi.num_queues = IXLV_MAX_QUEUES; + + code = i40e_aq_get_vsi_params(hw, &vsi_ctx, NULL); + if (code != I40E_SUCCESS) + return (ixl_adminq_err_to_errno(hw->aq.asq_last_status)); + + code = i40e_aq_config_vsi_bw_limit(hw, vf->vsi.seid, 0, 0, NULL); + if (code != I40E_SUCCESS) { + device_printf(pf->dev, "Failed to disable BW limit: %d\n", + ixl_adminq_err_to_errno(hw->aq.asq_last_status)); + return (ixl_adminq_err_to_errno(hw->aq.asq_last_status)); + } + + memcpy(&vf->vsi.info, &vsi_ctx.info, sizeof(vf->vsi.info)); + return (0); +} + +static int +ixl_vf_setup_vsi(struct ixl_pf *pf, struct ixl_vf *vf) +{ + struct i40e_hw *hw; + int error; + + hw = &pf->hw; + + error = ixl_vf_alloc_vsi(pf, vf); + if (error != 0) + return (error); + + vf->vsi.hw_filters_add = 0; + vf->vsi.hw_filters_del = 0; + ixl_add_filter(&vf->vsi, ixl_bcast_addr, IXL_VLAN_ANY); + ixl_reconfigure_filters(&vf->vsi); + + return (0); +} + +static void +ixl_vf_map_vsi_queue(struct i40e_hw *hw, struct ixl_vf *vf, int qnum, + uint32_t val) +{ + uint32_t qtable; + int index, shift; + + /* + * Two queues are mapped in a single register, so we have to do some + * gymnastics to convert the queue number into a register index and + * shift. + */ + index = qnum / 2; + shift = (qnum % 2) * I40E_VSILAN_QTABLE_QINDEX_1_SHIFT; + + qtable = rd32(hw, I40E_VSILAN_QTABLE(index, vf->vsi.vsi_num)); + qtable &= ~(I40E_VSILAN_QTABLE_QINDEX_0_MASK << shift); + qtable |= val << shift; + wr32(hw, I40E_VSILAN_QTABLE(index, vf->vsi.vsi_num), qtable); +} + +static void +ixl_vf_map_queues(struct ixl_pf *pf, struct ixl_vf *vf) +{ + struct i40e_hw *hw; + uint32_t qtable; + int i; + + hw = &pf->hw; + + /* + * Contiguous mappings aren't actually supported by the hardware, + * so we have to use non-contiguous mappings. + */ + wr32(hw, I40E_VSILAN_QBASE(vf->vsi.vsi_num), + I40E_VSILAN_QBASE_VSIQTABLE_ENA_MASK); + + wr32(hw, I40E_VPLAN_MAPENA(vf->vf_num), + I40E_VPLAN_MAPENA_TXRX_ENA_MASK); + + for (i = 0; i < vf->vsi.num_queues; i++) { + qtable = (vf->vsi.first_queue + i) << + I40E_VPLAN_QTABLE_QINDEX_SHIFT; + + wr32(hw, I40E_VPLAN_QTABLE(i, vf->vf_num), qtable); + } + + /* Map queues allocated to VF to its VSI. */ + for (i = 0; i < vf->vsi.num_queues; i++) + ixl_vf_map_vsi_queue(hw, vf, i, vf->vsi.first_queue + i); + + /* Set rest of VSI queues as unused. */ + for (; i < IXL_MAX_VSI_QUEUES; i++) + ixl_vf_map_vsi_queue(hw, vf, i, + I40E_VSILAN_QTABLE_QINDEX_0_MASK); + + ixl_flush(hw); +} + +static void +ixl_vf_vsi_release(struct ixl_pf *pf, struct ixl_vsi *vsi) +{ + struct i40e_hw *hw; + + hw = &pf->hw; + + if (vsi->seid == 0) + return; + + i40e_aq_delete_element(hw, vsi->seid, NULL); +} + +static void +ixl_vf_disable_queue_intr(struct i40e_hw *hw, uint32_t vfint_reg) +{ + + wr32(hw, vfint_reg, I40E_VFINT_DYN_CTLN_CLEARPBA_MASK); + ixl_flush(hw); +} + +static void +ixl_vf_unregister_intr(struct i40e_hw *hw, uint32_t vpint_reg) +{ + + wr32(hw, vpint_reg, I40E_VPINT_LNKLSTN_FIRSTQ_TYPE_MASK | + I40E_VPINT_LNKLSTN_FIRSTQ_INDX_MASK); + ixl_flush(hw); +} + +static void +ixl_vf_release_resources(struct ixl_pf *pf, struct ixl_vf *vf) +{ + struct i40e_hw *hw; + uint32_t vfint_reg, vpint_reg; + int i; + + hw = &pf->hw; + + ixl_vf_vsi_release(pf, &vf->vsi); + + /* Index 0 has a special register. */ + ixl_vf_disable_queue_intr(hw, I40E_VFINT_DYN_CTL0(vf->vf_num)); + + for (i = 1; i < hw->func_caps.num_msix_vectors_vf; i++) { + vfint_reg = IXL_VFINT_DYN_CTLN_REG(hw, i , vf->vf_num); + ixl_vf_disable_queue_intr(hw, vfint_reg); + } + + /* Index 0 has a special register. */ + ixl_vf_unregister_intr(hw, I40E_VPINT_LNKLST0(vf->vf_num)); + + for (i = 1; i < hw->func_caps.num_msix_vectors_vf; i++) { + vpint_reg = IXL_VPINT_LNKLSTN_REG(hw, i, vf->vf_num); + ixl_vf_unregister_intr(hw, vpint_reg); + } + + vf->vsi.num_queues = 0; +} + +static int +ixl_flush_pcie(struct ixl_pf *pf, struct ixl_vf *vf) +{ + struct i40e_hw *hw; + int i; + uint16_t global_vf_num; + uint32_t ciad; + + hw = &pf->hw; + global_vf_num = hw->func_caps.vf_base_id + vf->vf_num; + + wr32(hw, I40E_PF_PCI_CIAA, IXL_PF_PCI_CIAA_VF_DEVICE_STATUS | + (global_vf_num << I40E_PF_PCI_CIAA_VF_NUM_SHIFT)); + for (i = 0; i < IXL_VF_RESET_TIMEOUT; i++) { + ciad = rd32(hw, I40E_PF_PCI_CIAD); + if ((ciad & IXL_PF_PCI_CIAD_VF_TRANS_PENDING_MASK) == 0) + return (0); + DELAY(1); + } + + return (ETIMEDOUT); +} + +static void +ixl_reset_vf(struct ixl_pf *pf, struct ixl_vf *vf) +{ + struct i40e_hw *hw; + uint32_t vfrtrig; + + hw = &pf->hw; + + vfrtrig = rd32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num)); + vfrtrig |= I40E_VPGEN_VFRTRIG_VFSWR_MASK; + wr32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num), vfrtrig); + ixl_flush(hw); + + ixl_reinit_vf(pf, vf); +} + +static void +ixl_reinit_vf(struct ixl_pf *pf, struct ixl_vf *vf) +{ + struct i40e_hw *hw; + uint32_t vfrstat, vfrtrig; + int i, error; + + hw = &pf->hw; + + error = ixl_flush_pcie(pf, vf); + if (error != 0) + device_printf(pf->dev, + "Timed out waiting for PCIe activity to stop on VF-%d\n", + vf->vf_num); + + for (i = 0; i < IXL_VF_RESET_TIMEOUT; i++) { + DELAY(10); + + vfrstat = rd32(hw, I40E_VPGEN_VFRSTAT(vf->vf_num)); + if (vfrstat & I40E_VPGEN_VFRSTAT_VFRD_MASK) + break; + } + + if (i == IXL_VF_RESET_TIMEOUT) + device_printf(pf->dev, "VF %d failed to reset\n", vf->vf_num); + + wr32(hw, I40E_VFGEN_RSTAT1(vf->vf_num), I40E_VFR_COMPLETED); + + vfrtrig = rd32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num)); + vfrtrig &= ~I40E_VPGEN_VFRTRIG_VFSWR_MASK; + wr32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num), vfrtrig); + + if (vf->vsi.seid != 0) + ixl_disable_rings(&vf->vsi); + + ixl_vf_release_resources(pf, vf); + ixl_vf_setup_vsi(pf, vf); + ixl_vf_map_queues(pf, vf); + + wr32(hw, I40E_VFGEN_RSTAT1(vf->vf_num), I40E_VFR_VFACTIVE); + ixl_flush(hw); +} + +static const char * +ixl_vc_opcode_str(uint16_t op) +{ + + switch (op) { + case I40E_VIRTCHNL_OP_VERSION: + return ("VERSION"); + case I40E_VIRTCHNL_OP_RESET_VF: + return ("RESET_VF"); + case I40E_VIRTCHNL_OP_GET_VF_RESOURCES: + return ("GET_VF_RESOURCES"); + case I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE: + return ("CONFIG_TX_QUEUE"); + case I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE: + return ("CONFIG_RX_QUEUE"); + case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES: + return ("CONFIG_VSI_QUEUES"); + case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP: + return ("CONFIG_IRQ_MAP"); + case I40E_VIRTCHNL_OP_ENABLE_QUEUES: + return ("ENABLE_QUEUES"); + case I40E_VIRTCHNL_OP_DISABLE_QUEUES: + return ("DISABLE_QUEUES"); + case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS: + return ("ADD_ETHER_ADDRESS"); + case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS: + return ("DEL_ETHER_ADDRESS"); + case I40E_VIRTCHNL_OP_ADD_VLAN: + return ("ADD_VLAN"); + case I40E_VIRTCHNL_OP_DEL_VLAN: + return ("DEL_VLAN"); + case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE: + return ("CONFIG_PROMISCUOUS_MODE"); + case I40E_VIRTCHNL_OP_GET_STATS: + return ("GET_STATS"); + case I40E_VIRTCHNL_OP_FCOE: + return ("FCOE"); + case I40E_VIRTCHNL_OP_EVENT: + return ("EVENT"); + default: + return ("UNKNOWN"); + } +} + +static int +ixl_vc_opcode_level(uint16_t opcode) +{ + + switch (opcode) { + case I40E_VIRTCHNL_OP_GET_STATS: + return (10); + default: + return (5); + } +} + +static void +ixl_send_vf_msg(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op, + enum i40e_status_code status, void *msg, uint16_t len) +{ + struct i40e_hw *hw; + int global_vf_id; + + hw = &pf->hw; + global_vf_id = hw->func_caps.vf_base_id + vf->vf_num; + + I40E_VC_DEBUG(pf, ixl_vc_opcode_level(op), + "Sending msg (op=%s[%d], status=%d) to VF-%d\n", + ixl_vc_opcode_str(op), op, status, vf->vf_num); + + i40e_aq_send_msg_to_vf(hw, global_vf_id, op, status, msg, len, NULL); +} + +static void +ixl_send_vf_ack(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op) +{ + + ixl_send_vf_msg(pf, vf, op, I40E_SUCCESS, NULL, 0); +} + +static void +ixl_send_vf_nack_msg(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op, + enum i40e_status_code status, const char *file, int line) +{ + + I40E_VC_DEBUG(pf, 1, + "Sending NACK (op=%s[%d], err=%d) to VF-%d from %s:%d\n", + ixl_vc_opcode_str(op), op, status, vf->vf_num, file, line); + ixl_send_vf_msg(pf, vf, op, status, NULL, 0); +} + +static void +ixl_vf_version_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, + uint16_t msg_size) +{ + struct i40e_virtchnl_version_info reply; + + if (msg_size != sizeof(struct i40e_virtchnl_version_info)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_VERSION, + I40E_ERR_PARAM); + return; + } + + reply.major = I40E_VIRTCHNL_VERSION_MAJOR; + reply.minor = I40E_VIRTCHNL_VERSION_MINOR; + ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_VERSION, I40E_SUCCESS, &reply, + sizeof(reply)); +} + +static void +ixl_vf_reset_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, + uint16_t msg_size) +{ + + if (msg_size != 0) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_RESET_VF, + I40E_ERR_PARAM); + return; + } + + ixl_reset_vf(pf, vf); + + /* No response to a reset message. */ +} + +static void +ixl_vf_get_resources_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, + uint16_t msg_size) +{ + struct i40e_virtchnl_vf_resource reply; + + if (msg_size != 0) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_VF_RESOURCES, + I40E_ERR_PARAM); + return; + } + + bzero(&reply, sizeof(reply)); + + reply.vf_offload_flags = I40E_VIRTCHNL_VF_OFFLOAD_L2; + + reply.num_vsis = 1; + reply.num_queue_pairs = vf->vsi.num_queues; + reply.max_vectors = pf->hw.func_caps.num_msix_vectors_vf; + reply.vsi_res[0].vsi_id = vf->vsi.vsi_num; + reply.vsi_res[0].vsi_type = I40E_VSI_SRIOV; + reply.vsi_res[0].num_queue_pairs = vf->vsi.num_queues; + memcpy(reply.vsi_res[0].default_mac_addr, vf->mac, ETHER_ADDR_LEN); + + ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_GET_VF_RESOURCES, + I40E_SUCCESS, &reply, sizeof(reply)); +} + +static int +ixl_vf_config_tx_queue(struct ixl_pf *pf, struct ixl_vf *vf, + struct i40e_virtchnl_txq_info *info) +{ + struct i40e_hw *hw; + struct i40e_hmc_obj_txq txq; + uint16_t global_queue_num, global_vf_num; + enum i40e_status_code status; + uint32_t qtx_ctl; + + hw = &pf->hw; + global_queue_num = vf->vsi.first_queue + info->queue_id; + global_vf_num = hw->func_caps.vf_base_id + vf->vf_num; + bzero(&txq, sizeof(txq)); + + status = i40e_clear_lan_tx_queue_context(hw, global_queue_num); + if (status != I40E_SUCCESS) + return (EINVAL); + + txq.base = info->dma_ring_addr / IXL_TX_CTX_BASE_UNITS; + + txq.head_wb_ena = info->headwb_enabled; + txq.head_wb_addr = info->dma_headwb_addr; + txq.qlen = info->ring_len; + txq.rdylist = le16_to_cpu(vf->vsi.info.qs_handle[0]); + txq.rdylist_act = 0; + + status = i40e_set_lan_tx_queue_context(hw, global_queue_num, &txq); + if (status != I40E_SUCCESS) + return (EINVAL); + + qtx_ctl = I40E_QTX_CTL_VF_QUEUE | + (hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) | + (global_vf_num << I40E_QTX_CTL_VFVM_INDX_SHIFT); + wr32(hw, I40E_QTX_CTL(global_queue_num), qtx_ctl); + ixl_flush(hw); + + return (0); +} + +static int +ixl_vf_config_rx_queue(struct ixl_pf *pf, struct ixl_vf *vf, + struct i40e_virtchnl_rxq_info *info) +{ + struct i40e_hw *hw; + struct i40e_hmc_obj_rxq rxq; + uint16_t global_queue_num; + enum i40e_status_code status; + + hw = &pf->hw; + global_queue_num = vf->vsi.first_queue + info->queue_id; + bzero(&rxq, sizeof(rxq)); + + if (info->databuffer_size > IXL_VF_MAX_BUFFER) + return (EINVAL); + + if (info->max_pkt_size > IXL_VF_MAX_FRAME || + info->max_pkt_size < ETHER_MIN_LEN) + return (EINVAL); + + if (info->splithdr_enabled) { + if (info->hdr_size > IXL_VF_MAX_HDR_BUFFER) + return (EINVAL); + + rxq.hsplit_0 = info->rx_split_pos & + (I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_L2 | + I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_IP | + I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_TCP_UDP | + I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_SCTP); + rxq.hbuff = info->hdr_size >> I40E_RXQ_CTX_HBUFF_SHIFT; + + rxq.dtype = 2; + } + + status = i40e_clear_lan_rx_queue_context(hw, global_queue_num); + if (status != I40E_SUCCESS) + return (EINVAL); + + rxq.base = info->dma_ring_addr / IXL_RX_CTX_BASE_UNITS; + rxq.qlen = info->ring_len; + + rxq.dbuff = info->databuffer_size >> I40E_RXQ_CTX_DBUFF_SHIFT; + + rxq.dsize = 1; + rxq.crcstrip = 1; + rxq.l2tsel = 1; + + rxq.rxmax = info->max_pkt_size; + rxq.tphrdesc_ena = 1; + rxq.tphwdesc_ena = 1; + rxq.tphdata_ena = 1; + rxq.tphhead_ena = 1; + rxq.lrxqthresh = 2; + rxq.prefena = 1; + + status = i40e_set_lan_rx_queue_context(hw, global_queue_num, &rxq); + if (status != I40E_SUCCESS) + return (EINVAL); + + return (0); +} + +static void +ixl_vf_config_vsi_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, + uint16_t msg_size) +{ + struct i40e_virtchnl_vsi_queue_config_info *info; + struct i40e_virtchnl_queue_pair_info *pair; + int i; + + if (msg_size < sizeof(*info)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, + I40E_ERR_PARAM); + return; + } + + info = msg; + if (info->num_queue_pairs == 0) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, + I40E_ERR_PARAM); + return; + } + + if (msg_size != sizeof(*info) + info->num_queue_pairs * sizeof(*pair)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, + I40E_ERR_PARAM); + return; + } + + if (info->vsi_id != vf->vsi.vsi_num) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, + I40E_ERR_PARAM); + return; + } + + for (i = 0; i < info->num_queue_pairs; i++) { + pair = &info->qpair[i]; + + if (pair->txq.vsi_id != vf->vsi.vsi_num || + pair->rxq.vsi_id != vf->vsi.vsi_num || + pair->txq.queue_id != pair->rxq.queue_id || + pair->txq.queue_id >= vf->vsi.num_queues) { + + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM); + return; + } + + if (ixl_vf_config_tx_queue(pf, vf, &pair->txq) != 0) { + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM); + return; + } + + if (ixl_vf_config_rx_queue(pf, vf, &pair->rxq) != 0) { + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM); + return; + } + } + + ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES); +} + +static void +ixl_vf_set_qctl(struct ixl_pf *pf, + const struct i40e_virtchnl_vector_map *vector, + enum i40e_queue_type cur_type, uint16_t cur_queue, + enum i40e_queue_type *last_type, uint16_t *last_queue) +{ + uint32_t offset, qctl; + uint16_t itr_indx; + + if (cur_type == I40E_QUEUE_TYPE_RX) { + offset = I40E_QINT_RQCTL(cur_queue); + itr_indx = vector->rxitr_idx; + } else { + offset = I40E_QINT_TQCTL(cur_queue); + itr_indx = vector->txitr_idx; + } + + qctl = htole32((vector->vector_id << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | + (*last_type << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) | + (*last_queue << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | + I40E_QINT_RQCTL_CAUSE_ENA_MASK | + (itr_indx << I40E_QINT_RQCTL_ITR_INDX_SHIFT)); + + wr32(&pf->hw, offset, qctl); + + *last_type = cur_type; + *last_queue = cur_queue; +} + +static void +ixl_vf_config_vector(struct ixl_pf *pf, struct ixl_vf *vf, + const struct i40e_virtchnl_vector_map *vector) +{ + struct i40e_hw *hw; + u_int qindex; + enum i40e_queue_type type, last_type; + uint32_t lnklst_reg; + uint16_t rxq_map, txq_map, cur_queue, last_queue; + + hw = &pf->hw; + + rxq_map = vector->rxq_map; + txq_map = vector->txq_map; + + last_queue = IXL_END_OF_INTR_LNKLST; + last_type = I40E_QUEUE_TYPE_RX; + + /* + * The datasheet says to optimize performance, RX queues and TX queues + * should be interleaved in the interrupt linked list, so we process + * both at once here. + */ + while ((rxq_map != 0) || (txq_map != 0)) { + if (txq_map != 0) { + qindex = ffs(txq_map) - 1; + type = I40E_QUEUE_TYPE_TX; + cur_queue = vf->vsi.first_queue + qindex; + ixl_vf_set_qctl(pf, vector, type, cur_queue, + &last_type, &last_queue); + txq_map &= ~(1 << qindex); + } + + if (rxq_map != 0) { + qindex = ffs(rxq_map) - 1; + type = I40E_QUEUE_TYPE_RX; + cur_queue = vf->vsi.first_queue + qindex; + ixl_vf_set_qctl(pf, vector, type, cur_queue, + &last_type, &last_queue); + rxq_map &= ~(1 << qindex); + } + } + + if (vector->vector_id == 0) + lnklst_reg = I40E_VPINT_LNKLST0(vf->vf_num); + else + lnklst_reg = IXL_VPINT_LNKLSTN_REG(hw, vector->vector_id, + vf->vf_num); + wr32(hw, lnklst_reg, + (last_queue << I40E_VPINT_LNKLST0_FIRSTQ_INDX_SHIFT) | + (last_type << I40E_VPINT_LNKLST0_FIRSTQ_TYPE_SHIFT)); + + ixl_flush(hw); +} + +static void +ixl_vf_config_irq_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, + uint16_t msg_size) +{ + struct i40e_virtchnl_irq_map_info *map; + struct i40e_virtchnl_vector_map *vector; + struct i40e_hw *hw; + int i, largest_txq, largest_rxq; + + hw = &pf->hw; + + if (msg_size < sizeof(*map)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, + I40E_ERR_PARAM); + return; + } + + map = msg; + if (map->num_vectors == 0) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, + I40E_ERR_PARAM); + return; + } + + if (msg_size != sizeof(*map) + map->num_vectors * sizeof(*vector)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, + I40E_ERR_PARAM); + return; + } + + for (i = 0; i < map->num_vectors; i++) { + vector = &map->vecmap[i]; + + if ((vector->vector_id >= hw->func_caps.num_msix_vectors_vf) || + vector->vsi_id != vf->vsi.vsi_num) { + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, I40E_ERR_PARAM); + return; + } + + if (vector->rxq_map != 0) { + largest_rxq = fls(vector->rxq_map) - 1; + if (largest_rxq >= vf->vsi.num_queues) { + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, + I40E_ERR_PARAM); + return; + } + } + + if (vector->txq_map != 0) { + largest_txq = fls(vector->txq_map) - 1; + if (largest_txq >= vf->vsi.num_queues) { + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, + I40E_ERR_PARAM); + return; + } + } + + if (vector->rxitr_idx > IXL_MAX_ITR_IDX || + vector->txitr_idx > IXL_MAX_ITR_IDX) { + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, + I40E_ERR_PARAM); + return; + } + + ixl_vf_config_vector(pf, vf, vector); + } + + ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP); +} + +static void +ixl_vf_enable_queues_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, + uint16_t msg_size) +{ + struct i40e_virtchnl_queue_select *select; + int error; + + if (msg_size != sizeof(*select)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES, + I40E_ERR_PARAM); + return; + } + + select = msg; + if (select->vsi_id != vf->vsi.vsi_num || + select->rx_queues == 0 || select->tx_queues == 0) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES, + I40E_ERR_PARAM); + return; + } + + error = ixl_enable_rings(&vf->vsi); + if (error) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES, + I40E_ERR_TIMEOUT); + return; + } + + ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES); +} + +static void +ixl_vf_disable_queues_msg(struct ixl_pf *pf, struct ixl_vf *vf, + void *msg, uint16_t msg_size) +{ + struct i40e_virtchnl_queue_select *select; + int error; + + if (msg_size != sizeof(*select)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES, + I40E_ERR_PARAM); + return; + } + + select = msg; + if (select->vsi_id != vf->vsi.vsi_num || + select->rx_queues == 0 || select->tx_queues == 0) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES, + I40E_ERR_PARAM); + return; + } + + error = ixl_disable_rings(&vf->vsi); + if (error) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES, + I40E_ERR_TIMEOUT); + return; + } + + ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES); +} + +static boolean_t +ixl_zero_mac(const uint8_t *addr) +{ + uint8_t zero[ETHER_ADDR_LEN] = {0, 0, 0, 0, 0, 0}; + + return (cmp_etheraddr(addr, zero)); +} + +static boolean_t +ixl_bcast_mac(const uint8_t *addr) +{ + + return (cmp_etheraddr(addr, ixl_bcast_addr)); +} + +static int +ixl_vf_mac_valid(struct ixl_vf *vf, const uint8_t *addr) +{ + + if (ixl_zero_mac(addr) || ixl_bcast_mac(addr)) + return (EINVAL); + + /* + * If the VF is not allowed to change its MAC address, don't let it + * set a MAC filter for an address that is not a multicast address and + * is not its assigned MAC. + */ + if (!(vf->vf_flags & VF_FLAG_SET_MAC_CAP) && + !(ETHER_IS_MULTICAST(addr) || cmp_etheraddr(addr, vf->mac))) + return (EPERM); + + return (0); +} + +static void +ixl_vf_add_mac_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, + uint16_t msg_size) +{ + struct i40e_virtchnl_ether_addr_list *addr_list; + struct i40e_virtchnl_ether_addr *addr; + struct ixl_vsi *vsi; + int i; + size_t expected_size; + + vsi = &vf->vsi; + + if (msg_size < sizeof(*addr_list)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, + I40E_ERR_PARAM); + return; + } + + addr_list = msg; + expected_size = sizeof(*addr_list) + + addr_list->num_elements * sizeof(*addr); + + if (addr_list->num_elements == 0 || + addr_list->vsi_id != vsi->vsi_num || + msg_size != expected_size) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, + I40E_ERR_PARAM); + return; + } + + for (i = 0; i < addr_list->num_elements; i++) { + if (ixl_vf_mac_valid(vf, addr_list->list[i].addr) != 0) { + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, I40E_ERR_PARAM); + return; + } + } + + for (i = 0; i < addr_list->num_elements; i++) { + addr = &addr_list->list[i]; + ixl_add_filter(vsi, addr->addr, IXL_VLAN_ANY); + } + + ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS); +} + +static void +ixl_vf_del_mac_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, + uint16_t msg_size) +{ + struct i40e_virtchnl_ether_addr_list *addr_list; + struct i40e_virtchnl_ether_addr *addr; + size_t expected_size; + int i; + + if (msg_size < sizeof(*addr_list)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, + I40E_ERR_PARAM); + return; + } + + addr_list = msg; + expected_size = sizeof(*addr_list) + + addr_list->num_elements * sizeof(*addr); + + if (addr_list->num_elements == 0 || + addr_list->vsi_id != vf->vsi.vsi_num || + msg_size != expected_size) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, + I40E_ERR_PARAM); + return; + } + + for (i = 0; i < addr_list->num_elements; i++) { + addr = &addr_list->list[i]; + if (ixl_zero_mac(addr->addr) || ixl_bcast_mac(addr->addr)) { + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, I40E_ERR_PARAM); + return; + } + } + + for (i = 0; i < addr_list->num_elements; i++) { + addr = &addr_list->list[i]; + ixl_del_filter(&vf->vsi, addr->addr, IXL_VLAN_ANY); + } + + ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS); +} + +static enum i40e_status_code +ixl_vf_enable_vlan_strip(struct ixl_pf *pf, struct ixl_vf *vf) +{ + struct i40e_vsi_context vsi_ctx; + + vsi_ctx.seid = vf->vsi.seid; + + bzero(&vsi_ctx.info, sizeof(vsi_ctx.info)); + vsi_ctx.info.valid_sections = htole16(I40E_AQ_VSI_PROP_VLAN_VALID); + vsi_ctx.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | + I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH; + return (i40e_aq_update_vsi_params(&pf->hw, &vsi_ctx, NULL)); +} + +static void +ixl_vf_add_vlan_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, + uint16_t msg_size) +{ + struct i40e_virtchnl_vlan_filter_list *filter_list; + enum i40e_status_code code; + size_t expected_size; + int i; + + if (msg_size < sizeof(*filter_list)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, + I40E_ERR_PARAM); + return; + } + + filter_list = msg; + expected_size = sizeof(*filter_list) + + filter_list->num_elements * sizeof(uint16_t); + if (filter_list->num_elements == 0 || + filter_list->vsi_id != vf->vsi.vsi_num || + msg_size != expected_size) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, + I40E_ERR_PARAM); + return; + } + + if (!(vf->vf_flags & VF_FLAG_VLAN_CAP)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, + I40E_ERR_PARAM); + return; + } + + for (i = 0; i < filter_list->num_elements; i++) { + if (filter_list->vlan_id[i] > EVL_VLID_MASK) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, + I40E_ERR_PARAM); + return; + } + } + + code = ixl_vf_enable_vlan_strip(pf, vf); + if (code != I40E_SUCCESS) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, + I40E_ERR_PARAM); + } + + for (i = 0; i < filter_list->num_elements; i++) + ixl_add_filter(&vf->vsi, vf->mac, filter_list->vlan_id[i]); + + ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN); +} + +static void +ixl_vf_del_vlan_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, + uint16_t msg_size) +{ + struct i40e_virtchnl_vlan_filter_list *filter_list; + int i; + size_t expected_size; + + if (msg_size < sizeof(*filter_list)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN, + I40E_ERR_PARAM); + return; + } + + filter_list = msg; + expected_size = sizeof(*filter_list) + + filter_list->num_elements * sizeof(uint16_t); + if (filter_list->num_elements == 0 || + filter_list->vsi_id != vf->vsi.vsi_num || + msg_size != expected_size) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN, + I40E_ERR_PARAM); + return; + } + + for (i = 0; i < filter_list->num_elements; i++) { + if (filter_list->vlan_id[i] > EVL_VLID_MASK) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, + I40E_ERR_PARAM); + return; + } + } + + if (!(vf->vf_flags & VF_FLAG_VLAN_CAP)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, + I40E_ERR_PARAM); + return; + } + + for (i = 0; i < filter_list->num_elements; i++) + ixl_del_filter(&vf->vsi, vf->mac, filter_list->vlan_id[i]); + + ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN); +} + +static void +ixl_vf_config_promisc_msg(struct ixl_pf *pf, struct ixl_vf *vf, + void *msg, uint16_t msg_size) +{ + struct i40e_virtchnl_promisc_info *info; + enum i40e_status_code code; + + if (msg_size != sizeof(*info)) { + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM); + return; + } + + if (!(vf->vf_flags & VF_FLAG_PROMISC_CAP)) { + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM); + return; + } + + info = msg; + if (info->vsi_id != vf->vsi.vsi_num) { + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM); + return; + } + + code = i40e_aq_set_vsi_unicast_promiscuous(&pf->hw, info->vsi_id, + info->flags & I40E_FLAG_VF_UNICAST_PROMISC, NULL); + if (code != I40E_SUCCESS) { + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, code); + return; + } + + code = i40e_aq_set_vsi_multicast_promiscuous(&pf->hw, info->vsi_id, + info->flags & I40E_FLAG_VF_MULTICAST_PROMISC, NULL); + if (code != I40E_SUCCESS) { + i40e_send_vf_nack(pf, vf, + I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, code); + return; + } + + ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE); +} + +static void +ixl_vf_get_stats_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, + uint16_t msg_size) +{ + struct i40e_virtchnl_queue_select *queue; + + if (msg_size != sizeof(*queue)) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_STATS, + I40E_ERR_PARAM); + return; + } + + queue = msg; + if (queue->vsi_id != vf->vsi.vsi_num) { + i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_STATS, + I40E_ERR_PARAM); + return; + } + + ixl_update_eth_stats(&vf->vsi); + + ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_GET_STATS, + I40E_SUCCESS, &vf->vsi.eth_stats, sizeof(vf->vsi.eth_stats)); +} + +static void +ixl_handle_vf_msg(struct ixl_pf *pf, struct i40e_arq_event_info *event) +{ + struct ixl_vf *vf; + void *msg; + uint16_t vf_num, msg_size; + uint32_t opcode; + + vf_num = le16toh(event->desc.retval) - pf->hw.func_caps.vf_base_id; + opcode = le32toh(event->desc.cookie_high); + + if (vf_num >= pf->num_vfs) { + device_printf(pf->dev, "Got msg from illegal VF: %d\n", vf_num); + return; + } + + vf = &pf->vfs[vf_num]; + msg = event->msg_buf; + msg_size = event->msg_len; + + I40E_VC_DEBUG(pf, ixl_vc_opcode_level(opcode), + "Got msg %s(%d) from VF-%d of size %d\n", + ixl_vc_opcode_str(opcode), opcode, vf_num, msg_size); + + switch (opcode) { + case I40E_VIRTCHNL_OP_VERSION: + ixl_vf_version_msg(pf, vf, msg, msg_size); + break; + case I40E_VIRTCHNL_OP_RESET_VF: + ixl_vf_reset_msg(pf, vf, msg, msg_size); + break; + case I40E_VIRTCHNL_OP_GET_VF_RESOURCES: + ixl_vf_get_resources_msg(pf, vf, msg, msg_size); + break; + case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES: + ixl_vf_config_vsi_msg(pf, vf, msg, msg_size); + break; + case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP: + ixl_vf_config_irq_msg(pf, vf, msg, msg_size); + break; + case I40E_VIRTCHNL_OP_ENABLE_QUEUES: + ixl_vf_enable_queues_msg(pf, vf, msg, msg_size); + break; + case I40E_VIRTCHNL_OP_DISABLE_QUEUES: + ixl_vf_disable_queues_msg(pf, vf, msg, msg_size); + break; + case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS: + ixl_vf_add_mac_msg(pf, vf, msg, msg_size); + break; + case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS: + ixl_vf_del_mac_msg(pf, vf, msg, msg_size); + break; + case I40E_VIRTCHNL_OP_ADD_VLAN: + ixl_vf_add_vlan_msg(pf, vf, msg, msg_size); + break; + case I40E_VIRTCHNL_OP_DEL_VLAN: + ixl_vf_del_vlan_msg(pf, vf, msg, msg_size); + break; + case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE: + ixl_vf_config_promisc_msg(pf, vf, msg, msg_size); + break; + case I40E_VIRTCHNL_OP_GET_STATS: + ixl_vf_get_stats_msg(pf, vf, msg, msg_size); + break; + + /* These two opcodes have been superseded by CONFIG_VSI_QUEUES. */ + case I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE: + case I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE: + default: + i40e_send_vf_nack(pf, vf, opcode, I40E_ERR_NOT_IMPLEMENTED); + break; + } +} + +/* Handle any VFs that have reset themselves via a Function Level Reset(FLR). */ +static void +ixl_handle_vflr(void *arg, int pending) +{ + struct ixl_pf *pf; + struct i40e_hw *hw; + uint16_t global_vf_num; + uint32_t vflrstat_index, vflrstat_mask, vflrstat, icr0; + int i; + + pf = arg; + hw = &pf->hw; + + IXL_PF_LOCK(pf); + for (i = 0; i < pf->num_vfs; i++) { + global_vf_num = hw->func_caps.vf_base_id + i; + + vflrstat_index = IXL_GLGEN_VFLRSTAT_INDEX(global_vf_num); + vflrstat_mask = IXL_GLGEN_VFLRSTAT_MASK(global_vf_num); + vflrstat = rd32(hw, I40E_GLGEN_VFLRSTAT(vflrstat_index)); + if (vflrstat & vflrstat_mask) { + wr32(hw, I40E_GLGEN_VFLRSTAT(vflrstat_index), + vflrstat_mask); + + ixl_reinit_vf(pf, &pf->vfs[i]); + } + } + + icr0 = rd32(hw, I40E_PFINT_ICR0_ENA); + icr0 |= I40E_PFINT_ICR0_ENA_VFLR_MASK; + wr32(hw, I40E_PFINT_ICR0_ENA, icr0); + ixl_flush(hw); + + IXL_PF_UNLOCK(pf); +} + +static int +ixl_adminq_err_to_errno(enum i40e_admin_queue_err err) +{ + + switch (err) { + case I40E_AQ_RC_EPERM: + return (EPERM); + case I40E_AQ_RC_ENOENT: + return (ENOENT); + case I40E_AQ_RC_ESRCH: + return (ESRCH); + case I40E_AQ_RC_EINTR: + return (EINTR); + case I40E_AQ_RC_EIO: + return (EIO); + case I40E_AQ_RC_ENXIO: + return (ENXIO); + case I40E_AQ_RC_E2BIG: + return (E2BIG); + case I40E_AQ_RC_EAGAIN: + return (EAGAIN); + case I40E_AQ_RC_ENOMEM: + return (ENOMEM); + case I40E_AQ_RC_EACCES: + return (EACCES); + case I40E_AQ_RC_EFAULT: + return (EFAULT); + case I40E_AQ_RC_EBUSY: + return (EBUSY); + case I40E_AQ_RC_EEXIST: + return (EEXIST); + case I40E_AQ_RC_EINVAL: + return (EINVAL); + case I40E_AQ_RC_ENOTTY: + return (ENOTTY); + case I40E_AQ_RC_ENOSPC: + return (ENOSPC); + case I40E_AQ_RC_ENOSYS: + return (ENOSYS); + case I40E_AQ_RC_ERANGE: + return (ERANGE); + case I40E_AQ_RC_EFLUSHED: + return (EINVAL); /* No exact equivalent in errno.h */ + case I40E_AQ_RC_BAD_ADDR: + return (EFAULT); + case I40E_AQ_RC_EMODE: + return (EPERM); + case I40E_AQ_RC_EFBIG: + return (EFBIG); + default: + return (EINVAL); + } +} + +static int +ixl_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *params) +{ + struct ixl_pf *pf; + struct i40e_hw *hw; + struct ixl_vsi *pf_vsi; + enum i40e_status_code ret; + int i, error; + + pf = device_get_softc(dev); + hw = &pf->hw; + pf_vsi = &pf->vsi; + + IXL_PF_LOCK(pf); + pf->vfs = malloc(sizeof(struct ixl_vf) * num_vfs, M_IXL, M_NOWAIT | + M_ZERO); + + if (pf->vfs == NULL) { + error = ENOMEM; + goto fail; + } + + for (i = 0; i < num_vfs; i++) + sysctl_ctx_init(&pf->vfs[i].ctx); + + ret = i40e_aq_add_veb(hw, pf_vsi->uplink_seid, pf_vsi->seid, + 1, FALSE, FALSE, &pf->veb_seid, NULL); + if (ret != I40E_SUCCESS) { + error = ixl_adminq_err_to_errno(hw->aq.asq_last_status); + device_printf(dev, "add_veb failed; code=%d error=%d", ret, + error); + goto fail; + } + + ixl_configure_msix(pf); + ixl_enable_adminq(hw); + + pf->num_vfs = num_vfs; + IXL_PF_UNLOCK(pf); + return (0); + +fail: + free(pf->vfs, M_IXL); + pf->vfs = NULL; + IXL_PF_UNLOCK(pf); + return (error); +} + +static void +ixl_iov_uninit(device_t dev) +{ + struct ixl_pf *pf; + struct i40e_hw *hw; + struct ixl_vsi *vsi; + struct ifnet *ifp; + struct ixl_vf *vfs; + int i, num_vfs; + + pf = device_get_softc(dev); + hw = &pf->hw; + vsi = &pf->vsi; + ifp = vsi->ifp; + + IXL_PF_LOCK(pf); + for (i = 0; i < pf->num_vfs; i++) { + if (pf->vfs[i].vsi.seid != 0) + i40e_aq_delete_element(hw, pf->vfs[i].vsi.seid, NULL); + } + + if (pf->veb_seid != 0) { + i40e_aq_delete_element(hw, pf->veb_seid, NULL); + pf->veb_seid = 0; + } + +#if __FreeBSD_version > 1100022 + if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) +#else + if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) +#endif + ixl_disable_intr(vsi); + + vfs = pf->vfs; + num_vfs = pf->num_vfs; + + pf->vfs = NULL; + pf->num_vfs = 0; + IXL_PF_UNLOCK(pf); + + /* Do this after the unlock as sysctl_ctx_free might sleep. */ + for (i = 0; i < num_vfs; i++) + sysctl_ctx_free(&vfs[i].ctx); + free(vfs, M_IXL); +} + +static int +ixl_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *params) +{ + char sysctl_name[QUEUE_NAME_LEN]; + struct ixl_pf *pf; + struct ixl_vf *vf; + const void *mac; + size_t size; + int error; + + pf = device_get_softc(dev); + vf = &pf->vfs[vfnum]; + + IXL_PF_LOCK(pf); + vf->vf_num = vfnum; + + vf->vsi.back = pf; + vf->vf_flags = VF_FLAG_ENABLED; + SLIST_INIT(&vf->vsi.ftl); + + error = ixl_vf_setup_vsi(pf, vf); + if (error != 0) + goto out; + + if (nvlist_exists_binary(params, "mac-addr")) { + mac = nvlist_get_binary(params, "mac-addr", &size); + bcopy(mac, vf->mac, ETHER_ADDR_LEN); + + if (nvlist_get_bool(params, "allow-set-mac")) + vf->vf_flags |= VF_FLAG_SET_MAC_CAP; + } else + /* + * If the administrator has not specified a MAC address then + * we must allow the VF to choose one. + */ + vf->vf_flags |= VF_FLAG_SET_MAC_CAP; + + if (nvlist_get_bool(params, "mac-anti-spoof")) + vf->vf_flags |= VF_FLAG_MAC_ANTI_SPOOF; + + if (nvlist_get_bool(params, "allow-promisc")) + vf->vf_flags |= VF_FLAG_PROMISC_CAP; + + vf->vf_flags |= VF_FLAG_VLAN_CAP; + + ixl_reset_vf(pf, vf); +out: + IXL_PF_UNLOCK(pf); + if (error == 0) { + snprintf(sysctl_name, sizeof(sysctl_name), "vf%d", vfnum); + ixl_add_vsi_sysctls(pf, &vf->vsi, &vf->ctx, sysctl_name); + } + + return (error); +} +#endif /* PCI_IOV */ Index: sys/dev/ixl/legacy_if_ixlv.c =================================================================== --- /dev/null +++ sys/dev/ixl/legacy_if_ixlv.c @@ -0,0 +1,2972 @@ +/****************************************************************************** + + Copyright (c) 2013-2015, Intel Corporation + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of the Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +******************************************************************************/ +/*$FreeBSD$*/ + +#ifndef IXL_STANDALONE_BUILD +#include "opt_inet.h" +#include "opt_inet6.h" +#include "opt_rss.h" +#endif + +#include "ixl.h" +#include "ixlv.h" + +#ifdef RSS +#include +#endif + +/********************************************************************* + * Driver version + *********************************************************************/ +char ixlv_driver_version[] = "1.2.6"; + +/********************************************************************* + * PCI Device ID Table + * + * Used by probe to select devices to load on + * Last field stores an index into ixlv_strings + * Last entry must be all 0s + * + * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } + *********************************************************************/ + +static ixl_vendor_info_t ixlv_vendor_info_array[] = +{ + {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_VF, 0, 0, 0}, + {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_VF_HV, 0, 0, 0}, + /* required last entry */ + {0, 0, 0, 0, 0} +}; + +/********************************************************************* + * Table of branding strings + *********************************************************************/ + +static char *ixlv_strings[] = { + "Intel(R) Ethernet Connection XL710 VF Driver" +}; + + +/********************************************************************* + * Function prototypes + *********************************************************************/ +static int ixlv_probe(device_t); +static int ixlv_attach(device_t); +static int ixlv_detach(device_t); +static int ixlv_shutdown(device_t); +static void ixlv_init_locked(struct ixlv_sc *); +static int ixlv_allocate_pci_resources(struct ixlv_sc *); +static void ixlv_free_pci_resources(struct ixlv_sc *); +static int ixlv_assign_msix(struct ixlv_sc *); +static int ixlv_init_msix(struct ixlv_sc *); +static int ixlv_init_taskqueue(struct ixlv_sc *); +static int ixlv_setup_queues(struct ixlv_sc *); +static void ixlv_config_rss(struct ixlv_sc *); +static void ixlv_stop(struct ixlv_sc *); +static void ixlv_add_multi(struct ixl_vsi *); +static void ixlv_del_multi(struct ixl_vsi *); +static void ixlv_free_queues(struct ixl_vsi *); +static int ixlv_setup_interface(device_t, struct ixlv_sc *); + +static int ixlv_media_change(struct ifnet *); +static void ixlv_media_status(struct ifnet *, struct ifmediareq *); + +static void ixlv_local_timer(void *); + +static int ixlv_add_mac_filter(struct ixlv_sc *, u8 *, u16); +static int ixlv_del_mac_filter(struct ixlv_sc *sc, u8 *macaddr); +static void ixlv_init_filters(struct ixlv_sc *); +static void ixlv_free_filters(struct ixlv_sc *); + +static void ixlv_msix_que(void *); +static void ixlv_msix_adminq(void *); +static void ixlv_do_adminq(void *, int); +static void ixlv_do_adminq_locked(struct ixlv_sc *sc); +static void ixlv_handle_que(void *, int); +static int ixlv_reset(struct ixlv_sc *); +static int ixlv_reset_complete(struct i40e_hw *); +static void ixlv_set_queue_rx_itr(struct ixl_queue *); +static void ixlv_set_queue_tx_itr(struct ixl_queue *); +static void ixl_init_cmd_complete(struct ixl_vc_cmd *, void *, + enum i40e_status_code); + +static void ixlv_enable_adminq_irq(struct i40e_hw *); +static void ixlv_disable_adminq_irq(struct i40e_hw *); +static void ixlv_enable_queue_irq(struct i40e_hw *, int); +static void ixlv_disable_queue_irq(struct i40e_hw *, int); + +static void ixlv_setup_vlan_filters(struct ixlv_sc *); +static void ixlv_register_vlan(void *, struct ifnet *, u16); +static void ixlv_unregister_vlan(void *, struct ifnet *, u16); + +static void ixlv_init_hw(struct ixlv_sc *); +static int ixlv_setup_vc(struct ixlv_sc *); +static int ixlv_vf_config(struct ixlv_sc *); + +static void ixlv_cap_txcsum_tso(struct ixl_vsi *, + struct ifnet *, int); + +static void ixlv_add_sysctls(struct ixlv_sc *); +static int ixlv_sysctl_qtx_tail_handler(SYSCTL_HANDLER_ARGS); +static int ixlv_sysctl_qrx_tail_handler(SYSCTL_HANDLER_ARGS); + +/********************************************************************* + * FreeBSD Device Interface Entry Points + *********************************************************************/ + +static device_method_t ixlv_methods[] = { + /* Device interface */ + DEVMETHOD(device_probe, ixlv_probe), + DEVMETHOD(device_attach, ixlv_attach), + DEVMETHOD(device_detach, ixlv_detach), + DEVMETHOD(device_shutdown, ixlv_shutdown), + {0, 0} +}; + +static driver_t ixlv_driver = { + "ixlv", ixlv_methods, sizeof(struct ixlv_sc), +}; + +devclass_t ixlv_devclass; +DRIVER_MODULE(ixlv, pci, ixlv_driver, ixlv_devclass, 0, 0); + +MODULE_DEPEND(ixlv, pci, 1, 1, 1); +MODULE_DEPEND(ixlv, ether, 1, 1, 1); + +/* +** TUNEABLE PARAMETERS: +*/ + +static SYSCTL_NODE(_hw, OID_AUTO, ixlv, CTLFLAG_RD, 0, + "IXLV driver parameters"); + +/* +** Number of descriptors per ring: +** - TX and RX are the same size +*/ +static int ixlv_ringsz = DEFAULT_RING; +TUNABLE_INT("hw.ixlv.ringsz", &ixlv_ringsz); +SYSCTL_INT(_hw_ixlv, OID_AUTO, ring_size, CTLFLAG_RDTUN, + &ixlv_ringsz, 0, "Descriptor Ring Size"); + +/* Set to zero to auto calculate */ +int ixlv_max_queues = 0; +TUNABLE_INT("hw.ixlv.max_queues", &ixlv_max_queues); +SYSCTL_INT(_hw_ixlv, OID_AUTO, max_queues, CTLFLAG_RDTUN, + &ixlv_max_queues, 0, "Number of Queues"); + +/* +** Number of entries in Tx queue buf_ring. +** Increasing this will reduce the number of +** errors when transmitting fragmented UDP +** packets. +*/ +static int ixlv_txbrsz = DEFAULT_TXBRSZ; +TUNABLE_INT("hw.ixlv.txbrsz", &ixlv_txbrsz); +SYSCTL_INT(_hw_ixlv, OID_AUTO, txbr_size, CTLFLAG_RDTUN, + &ixlv_txbrsz, 0, "TX Buf Ring Size"); + +/* +** Controls for Interrupt Throttling +** - true/false for dynamic adjustment +** - default values for static ITR +*/ +int ixlv_dynamic_rx_itr = 0; +TUNABLE_INT("hw.ixlv.dynamic_rx_itr", &ixlv_dynamic_rx_itr); +SYSCTL_INT(_hw_ixlv, OID_AUTO, dynamic_rx_itr, CTLFLAG_RDTUN, + &ixlv_dynamic_rx_itr, 0, "Dynamic RX Interrupt Rate"); + +int ixlv_dynamic_tx_itr = 0; +TUNABLE_INT("hw.ixlv.dynamic_tx_itr", &ixlv_dynamic_tx_itr); +SYSCTL_INT(_hw_ixlv, OID_AUTO, dynamic_tx_itr, CTLFLAG_RDTUN, + &ixlv_dynamic_tx_itr, 0, "Dynamic TX Interrupt Rate"); + +int ixlv_rx_itr = IXL_ITR_8K; +TUNABLE_INT("hw.ixlv.rx_itr", &ixlv_rx_itr); +SYSCTL_INT(_hw_ixlv, OID_AUTO, rx_itr, CTLFLAG_RDTUN, + &ixlv_rx_itr, 0, "RX Interrupt Rate"); + +int ixlv_tx_itr = IXL_ITR_4K; +TUNABLE_INT("hw.ixlv.tx_itr", &ixlv_tx_itr); +SYSCTL_INT(_hw_ixlv, OID_AUTO, tx_itr, CTLFLAG_RDTUN, + &ixlv_tx_itr, 0, "TX Interrupt Rate"); + + +/********************************************************************* + * Device identification routine + * + * ixlv_probe determines if the driver should be loaded on + * the hardware based on PCI vendor/device id of the device. + * + * return BUS_PROBE_DEFAULT on success, positive on failure + *********************************************************************/ + +static int +ixlv_probe(device_t dev) +{ + ixl_vendor_info_t *ent; + + u16 pci_vendor_id, pci_device_id; + u16 pci_subvendor_id, pci_subdevice_id; + char device_name[256]; + + INIT_DEBUGOUT("ixlv_probe: begin"); + + pci_vendor_id = pci_get_vendor(dev); + if (pci_vendor_id != I40E_INTEL_VENDOR_ID) + return (ENXIO); + + pci_device_id = pci_get_device(dev); + pci_subvendor_id = pci_get_subvendor(dev); + pci_subdevice_id = pci_get_subdevice(dev); + + ent = ixlv_vendor_info_array; + while (ent->vendor_id != 0) { + if ((pci_vendor_id == ent->vendor_id) && + (pci_device_id == ent->device_id) && + + ((pci_subvendor_id == ent->subvendor_id) || + (ent->subvendor_id == 0)) && + + ((pci_subdevice_id == ent->subdevice_id) || + (ent->subdevice_id == 0))) { + sprintf(device_name, "%s, Version - %s", + ixlv_strings[ent->index], + ixlv_driver_version); + device_set_desc_copy(dev, device_name); + return (BUS_PROBE_DEFAULT); + } + ent++; + } + return (ENXIO); +} + +/********************************************************************* + * Device initialization routine + * + * The attach entry point is called when the driver is being loaded. + * This routine identifies the type of hardware, allocates all resources + * and initializes the hardware. + * + * return 0 on success, positive on failure + *********************************************************************/ + +static int +ixlv_attach(device_t dev) +{ + struct ixlv_sc *sc; + struct i40e_hw *hw; + struct ixl_vsi *vsi; + int error = 0; + + INIT_DBG_DEV(dev, "begin"); + + /* Allocate, clear, and link in our primary soft structure */ + sc = device_get_softc(dev); + sc->dev = sc->osdep.dev = dev; + hw = &sc->hw; + vsi = &sc->vsi; + vsi->dev = dev; + + /* Initialize hw struct */ + ixlv_init_hw(sc); + + /* Allocate filter lists */ + ixlv_init_filters(sc); + + /* Core Lock Init*/ + mtx_init(&sc->mtx, device_get_nameunit(dev), + "IXL SC Lock", MTX_DEF); + + /* Set up the timer callout */ + callout_init_mtx(&sc->timer, &sc->mtx, 0); + + /* Do PCI setup - map BAR0, etc */ + if (ixlv_allocate_pci_resources(sc)) { + device_printf(dev, "%s: Allocation of PCI resources failed\n", + __func__); + error = ENXIO; + goto err_early; + } + + INIT_DBG_DEV(dev, "Allocated PCI resources and MSIX vectors"); + + error = i40e_set_mac_type(hw); + if (error) { + device_printf(dev, "%s: set_mac_type failed: %d\n", + __func__, error); + goto err_pci_res; + } + + error = ixlv_reset_complete(hw); + if (error) { + device_printf(dev, "%s: Device is still being reset\n", + __func__); + goto err_pci_res; + } + + INIT_DBG_DEV(dev, "VF Device is ready for configuration"); + + error = ixlv_setup_vc(sc); + if (error) { + device_printf(dev, "%s: Error setting up PF comms, %d\n", + __func__, error); + goto err_pci_res; + } + + INIT_DBG_DEV(dev, "PF API version verified"); + + /* TODO: Figure out why MDD events occur when this reset is removed. */ + /* Need API version before sending reset message */ + error = ixlv_reset(sc); + if (error) { + device_printf(dev, "VF reset failed; reload the driver\n"); + goto err_aq; + } + + INIT_DBG_DEV(dev, "VF reset complete"); + + /* Ask for VF config from PF */ + error = ixlv_vf_config(sc); + if (error) { + device_printf(dev, "Error getting configuration from PF: %d\n", + error); + goto err_aq; + } + + INIT_DBG_DEV(dev, "VF config from PF:"); + INIT_DBG_DEV(dev, "VSIs %d, Queues %d, Max Vectors %d, Max MTU %d", + sc->vf_res->num_vsis, + sc->vf_res->num_queue_pairs, + sc->vf_res->max_vectors, + sc->vf_res->max_mtu); + INIT_DBG_DEV(dev, "Offload flags: %#010x", + sc->vf_res->vf_offload_flags); + + // TODO: Move this into ixlv_vf_config? + /* got VF config message back from PF, now we can parse it */ + for (int i = 0; i < sc->vf_res->num_vsis; i++) { + if (sc->vf_res->vsi_res[i].vsi_type == I40E_VSI_SRIOV) + sc->vsi_res = &sc->vf_res->vsi_res[i]; + } + if (!sc->vsi_res) { + device_printf(dev, "%s: no LAN VSI found\n", __func__); + error = EIO; + goto err_res_buf; + } + + INIT_DBG_DEV(dev, "Resource Acquisition complete"); + + /* If no mac address was assigned just make a random one */ + if (!ixlv_check_ether_addr(hw->mac.addr)) { + u8 addr[ETHER_ADDR_LEN]; + arc4rand(&addr, sizeof(addr), 0); + addr[0] &= 0xFE; + addr[0] |= 0x02; + bcopy(addr, hw->mac.addr, sizeof(addr)); + } + + vsi->id = sc->vsi_res->vsi_id; + vsi->back = (void *)sc; + sc->link_up = TRUE; + + /* This allocates the memory and early settings */ + if (ixlv_setup_queues(sc) != 0) { + device_printf(dev, "%s: setup queues failed!\n", + __func__); + error = EIO; + goto out; + } + + /* Setup the stack interface */ + if (ixlv_setup_interface(dev, sc) != 0) { + device_printf(dev, "%s: setup interface failed!\n", + __func__); + error = EIO; + goto out; + } + + INIT_DBG_DEV(dev, "Queue memory and interface setup"); + + /* Do queue interrupt setup */ + ixlv_assign_msix(sc); + + /* Start AdminQ taskqueue */ + ixlv_init_taskqueue(sc); + + /* Initialize stats */ + bzero(&sc->vsi.eth_stats, sizeof(struct i40e_eth_stats)); + ixlv_add_sysctls(sc); + + /* Register for VLAN events */ + vsi->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, + ixlv_register_vlan, vsi, EVENTHANDLER_PRI_FIRST); + vsi->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, + ixlv_unregister_vlan, vsi, EVENTHANDLER_PRI_FIRST); + + /* We want AQ enabled early */ + ixlv_enable_adminq_irq(hw); + + /* Set things up to run init */ + sc->init_state = IXLV_INIT_READY; + + ixl_vc_init_mgr(sc, &sc->vc_mgr); + + INIT_DBG_DEV(dev, "end"); + return (error); + +out: + ixlv_free_queues(vsi); +err_res_buf: + free(sc->vf_res, M_DEVBUF); +err_aq: + i40e_shutdown_adminq(hw); +err_pci_res: + ixlv_free_pci_resources(sc); +err_early: + mtx_destroy(&sc->mtx); + ixlv_free_filters(sc); + INIT_DBG_DEV(dev, "end: error %d", error); + return (error); +} + +/********************************************************************* + * Device removal routine + * + * The detach entry point is called when the driver is being removed. + * This routine stops the adapter and deallocates all the resources + * that were allocated for driver operation. + * + * return 0 on success, positive on failure + *********************************************************************/ + +static int +ixlv_detach(device_t dev) +{ + struct ixlv_sc *sc = device_get_softc(dev); + struct ixl_vsi *vsi = &sc->vsi; + + INIT_DBG_DEV(dev, "begin"); + + /* Make sure VLANS are not using driver */ + if (vsi->ifp->if_vlantrunk != NULL) { + if_printf(vsi->ifp, "Vlan in use, detach first\n"); + INIT_DBG_DEV(dev, "end"); + return (EBUSY); + } + + /* Stop driver */ + ether_ifdetach(vsi->ifp); + if (vsi->ifp->if_drv_flags & IFF_DRV_RUNNING) { + mtx_lock(&sc->mtx); + ixlv_stop(sc); + mtx_unlock(&sc->mtx); + } + + /* Unregister VLAN events */ + if (vsi->vlan_attach != NULL) + EVENTHANDLER_DEREGISTER(vlan_config, vsi->vlan_attach); + if (vsi->vlan_detach != NULL) + EVENTHANDLER_DEREGISTER(vlan_unconfig, vsi->vlan_detach); + + /* Drain VC mgr */ + callout_drain(&sc->vc_mgr.callout); + + i40e_shutdown_adminq(&sc->hw); + taskqueue_free(sc->tq); + if_free(vsi->ifp); + free(sc->vf_res, M_DEVBUF); + ixlv_free_pci_resources(sc); + ixlv_free_queues(vsi); + mtx_destroy(&sc->mtx); + ixlv_free_filters(sc); + + bus_generic_detach(dev); + INIT_DBG_DEV(dev, "end"); + return (0); +} + +/********************************************************************* + * + * Shutdown entry point + * + **********************************************************************/ + +static int +ixlv_shutdown(device_t dev) +{ + struct ixlv_sc *sc = device_get_softc(dev); + + INIT_DBG_DEV(dev, "begin"); + + mtx_lock(&sc->mtx); + ixlv_stop(sc); + mtx_unlock(&sc->mtx); + + INIT_DBG_DEV(dev, "end"); + return (0); +} + +/* + * Configure TXCSUM(IPV6) and TSO(4/6) + * - the hardware handles these together so we + * need to tweak them + */ +static void +ixlv_cap_txcsum_tso(struct ixl_vsi *vsi, struct ifnet *ifp, int mask) +{ + /* Enable/disable TXCSUM/TSO4 */ + if (!(ifp->if_capenable & IFCAP_TXCSUM) + && !(ifp->if_capenable & IFCAP_TSO4)) { + if (mask & IFCAP_TXCSUM) { + ifp->if_capenable |= IFCAP_TXCSUM; + /* enable TXCSUM, restore TSO if previously enabled */ + if (vsi->flags & IXL_FLAGS_KEEP_TSO4) { + vsi->flags &= ~IXL_FLAGS_KEEP_TSO4; + ifp->if_capenable |= IFCAP_TSO4; + } + } + else if (mask & IFCAP_TSO4) { + ifp->if_capenable |= (IFCAP_TXCSUM | IFCAP_TSO4); + vsi->flags &= ~IXL_FLAGS_KEEP_TSO4; + if_printf(ifp, + "TSO4 requires txcsum, enabling both...\n"); + } + } else if((ifp->if_capenable & IFCAP_TXCSUM) + && !(ifp->if_capenable & IFCAP_TSO4)) { + if (mask & IFCAP_TXCSUM) + ifp->if_capenable &= ~IFCAP_TXCSUM; + else if (mask & IFCAP_TSO4) + ifp->if_capenable |= IFCAP_TSO4; + } else if((ifp->if_capenable & IFCAP_TXCSUM) + && (ifp->if_capenable & IFCAP_TSO4)) { + if (mask & IFCAP_TXCSUM) { + vsi->flags |= IXL_FLAGS_KEEP_TSO4; + ifp->if_capenable &= ~(IFCAP_TXCSUM | IFCAP_TSO4); + if_printf(ifp, + "TSO4 requires txcsum, disabling both...\n"); + } else if (mask & IFCAP_TSO4) + ifp->if_capenable &= ~IFCAP_TSO4; + } + + /* Enable/disable TXCSUM_IPV6/TSO6 */ + if (!(ifp->if_capenable & IFCAP_TXCSUM_IPV6) + && !(ifp->if_capenable & IFCAP_TSO6)) { + if (mask & IFCAP_TXCSUM_IPV6) { + ifp->if_capenable |= IFCAP_TXCSUM_IPV6; + if (vsi->flags & IXL_FLAGS_KEEP_TSO6) { + vsi->flags &= ~IXL_FLAGS_KEEP_TSO6; + ifp->if_capenable |= IFCAP_TSO6; + } + } else if (mask & IFCAP_TSO6) { + ifp->if_capenable |= (IFCAP_TXCSUM_IPV6 | IFCAP_TSO6); + vsi->flags &= ~IXL_FLAGS_KEEP_TSO6; + if_printf(ifp, + "TSO6 requires txcsum6, enabling both...\n"); + } + } else if((ifp->if_capenable & IFCAP_TXCSUM_IPV6) + && !(ifp->if_capenable & IFCAP_TSO6)) { + if (mask & IFCAP_TXCSUM_IPV6) + ifp->if_capenable &= ~IFCAP_TXCSUM_IPV6; + else if (mask & IFCAP_TSO6) + ifp->if_capenable |= IFCAP_TSO6; + } else if ((ifp->if_capenable & IFCAP_TXCSUM_IPV6) + && (ifp->if_capenable & IFCAP_TSO6)) { + if (mask & IFCAP_TXCSUM_IPV6) { + vsi->flags |= IXL_FLAGS_KEEP_TSO6; + ifp->if_capenable &= ~(IFCAP_TXCSUM_IPV6 | IFCAP_TSO6); + if_printf(ifp, + "TSO6 requires txcsum6, disabling both...\n"); + } else if (mask & IFCAP_TSO6) + ifp->if_capenable &= ~IFCAP_TSO6; + } +} + +/********************************************************************* + * Ioctl entry point + * + * ixlv_ioctl is called when the user wants to configure the + * interface. + * + * return 0 on success, positive on failure + **********************************************************************/ + +static int +ixlv_ioctl(struct ifnet *ifp, u_long command, caddr_t data) +{ + struct ixl_vsi *vsi = ifp->if_softc; + struct ixlv_sc *sc = vsi->back; + struct ifreq *ifr = (struct ifreq *)data; +#if defined(INET) || defined(INET6) + struct ifaddr *ifa = (struct ifaddr *)data; + bool avoid_reset = FALSE; +#endif + int error = 0; + + + switch (command) { + + case SIOCSIFADDR: +#ifdef INET + if (ifa->ifa_addr->sa_family == AF_INET) + avoid_reset = TRUE; +#endif +#ifdef INET6 + if (ifa->ifa_addr->sa_family == AF_INET6) + avoid_reset = TRUE; +#endif +#if defined(INET) || defined(INET6) + /* + ** Calling init results in link renegotiation, + ** so we avoid doing it when possible. + */ + if (avoid_reset) { + ifp->if_flags |= IFF_UP; + if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) + ixlv_init(vsi); +#ifdef INET + if (!(ifp->if_flags & IFF_NOARP)) + arp_ifinit(ifp, ifa); +#endif + } else + error = ether_ioctl(ifp, command, data); + break; +#endif + case SIOCSIFMTU: + IOCTL_DBG_IF2(ifp, "SIOCSIFMTU (Set Interface MTU)"); + mtx_lock(&sc->mtx); + if (ifr->ifr_mtu > IXL_MAX_FRAME - + ETHER_HDR_LEN - ETHER_CRC_LEN - ETHER_VLAN_ENCAP_LEN) { + error = EINVAL; + IOCTL_DBG_IF(ifp, "mtu too large"); + } else { + IOCTL_DBG_IF2(ifp, "mtu: %lu -> %d", ifp->if_mtu, ifr->ifr_mtu); + // ERJ: Interestingly enough, these types don't match + ifp->if_mtu = (u_long)ifr->ifr_mtu; + vsi->max_frame_size = + ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN + + ETHER_VLAN_ENCAP_LEN; + ixlv_init_locked(sc); + } + mtx_unlock(&sc->mtx); + break; + case SIOCSIFFLAGS: + IOCTL_DBG_IF2(ifp, "SIOCSIFFLAGS (Set Interface Flags)"); + mtx_lock(&sc->mtx); + if (ifp->if_flags & IFF_UP) { + if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) + ixlv_init_locked(sc); + } else + if (ifp->if_drv_flags & IFF_DRV_RUNNING) + ixlv_stop(sc); + sc->if_flags = ifp->if_flags; + mtx_unlock(&sc->mtx); + break; + case SIOCADDMULTI: + IOCTL_DBG_IF2(ifp, "SIOCADDMULTI"); + if (ifp->if_drv_flags & IFF_DRV_RUNNING) { + mtx_lock(&sc->mtx); + ixlv_disable_intr(vsi); + ixlv_add_multi(vsi); + ixlv_enable_intr(vsi); + mtx_unlock(&sc->mtx); + } + break; + case SIOCDELMULTI: + IOCTL_DBG_IF2(ifp, "SIOCDELMULTI"); + if (sc->init_state == IXLV_RUNNING) { + mtx_lock(&sc->mtx); + ixlv_disable_intr(vsi); + ixlv_del_multi(vsi); + ixlv_enable_intr(vsi); + mtx_unlock(&sc->mtx); + } + break; + case SIOCSIFMEDIA: + case SIOCGIFMEDIA: + IOCTL_DBG_IF2(ifp, "SIOCxIFMEDIA (Get/Set Interface Media)"); + error = ifmedia_ioctl(ifp, ifr, &sc->media, command); + break; + case SIOCSIFCAP: + { + int mask = ifr->ifr_reqcap ^ ifp->if_capenable; + IOCTL_DBG_IF2(ifp, "SIOCSIFCAP (Set Capabilities)"); + + ixlv_cap_txcsum_tso(vsi, ifp, mask); + + if (mask & IFCAP_RXCSUM) + ifp->if_capenable ^= IFCAP_RXCSUM; + if (mask & IFCAP_RXCSUM_IPV6) + ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; + if (mask & IFCAP_LRO) + ifp->if_capenable ^= IFCAP_LRO; + if (mask & IFCAP_VLAN_HWTAGGING) + ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; + if (mask & IFCAP_VLAN_HWFILTER) + ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; + if (mask & IFCAP_VLAN_HWTSO) + ifp->if_capenable ^= IFCAP_VLAN_HWTSO; + if (ifp->if_drv_flags & IFF_DRV_RUNNING) { + ixlv_init(vsi); + } + VLAN_CAPABILITIES(ifp); + + break; + } + + default: + IOCTL_DBG_IF2(ifp, "UNKNOWN (0x%X)", (int)command); + error = ether_ioctl(ifp, command, data); + break; + } + + return (error); +} + +/* +** To do a reinit on the VF is unfortunately more complicated +** than a physical device, we must have the PF more or less +** completely recreate our memory, so many things that were +** done only once at attach in traditional drivers now must be +** redone at each reinitialization. This function does that +** 'prelude' so we can then call the normal locked init code. +*/ +int +ixlv_reinit_locked(struct ixlv_sc *sc) +{ + struct i40e_hw *hw = &sc->hw; + struct ixl_vsi *vsi = &sc->vsi; + struct ifnet *ifp = vsi->ifp; + struct ixlv_mac_filter *mf, *mf_temp; + struct ixlv_vlan_filter *vf; + int error = 0; + + INIT_DBG_IF(ifp, "begin"); + + if (ifp->if_drv_flags & IFF_DRV_RUNNING) + ixlv_stop(sc); + + error = ixlv_reset(sc); + + INIT_DBG_IF(ifp, "VF was reset"); + + /* set the state in case we went thru RESET */ + sc->init_state = IXLV_RUNNING; + + /* + ** Resetting the VF drops all filters from hardware; + ** we need to mark them to be re-added in init. + */ + SLIST_FOREACH_SAFE(mf, sc->mac_filters, next, mf_temp) { + if (mf->flags & IXL_FILTER_DEL) { + SLIST_REMOVE(sc->mac_filters, mf, + ixlv_mac_filter, next); + free(mf, M_DEVBUF); + } else + mf->flags |= IXL_FILTER_ADD; + } + if (vsi->num_vlans != 0) + SLIST_FOREACH(vf, sc->vlan_filters, next) + vf->flags = IXL_FILTER_ADD; + else { /* clean any stale filters */ + while (!SLIST_EMPTY(sc->vlan_filters)) { + vf = SLIST_FIRST(sc->vlan_filters); + SLIST_REMOVE_HEAD(sc->vlan_filters, next); + free(vf, M_DEVBUF); + } + } + + ixlv_enable_adminq_irq(hw); + ixl_vc_flush(&sc->vc_mgr); + + INIT_DBG_IF(ifp, "end"); + return (error); +} + +static void +ixl_init_cmd_complete(struct ixl_vc_cmd *cmd, void *arg, + enum i40e_status_code code) +{ + struct ixlv_sc *sc; + + sc = arg; + + /* + * Ignore "Adapter Stopped" message as that happens if an ifconfig down + * happens while a command is in progress, so we don't print an error + * in that case. + */ + if (code != I40E_SUCCESS && code != I40E_ERR_ADAPTER_STOPPED) { + if_printf(sc->vsi.ifp, + "Error %d waiting for PF to complete operation %d\n", + code, cmd->request); + } +} + +static void +ixlv_init_locked(struct ixlv_sc *sc) +{ + struct i40e_hw *hw = &sc->hw; + struct ixl_vsi *vsi = &sc->vsi; + struct ixl_queue *que = vsi->queues; + struct ifnet *ifp = vsi->ifp; + int error = 0; + + INIT_DBG_IF(ifp, "begin"); + + IXLV_CORE_LOCK_ASSERT(sc); + + /* Do a reinit first if an init has already been done */ + if ((sc->init_state == IXLV_RUNNING) || + (sc->init_state == IXLV_RESET_REQUIRED) || + (sc->init_state == IXLV_RESET_PENDING)) + error = ixlv_reinit_locked(sc); + /* Don't bother with init if we failed reinit */ + if (error) + goto init_done; + + /* Remove existing MAC filter if new MAC addr is set */ + if (bcmp(IF_LLADDR(ifp), hw->mac.addr, ETHER_ADDR_LEN) != 0) { + error = ixlv_del_mac_filter(sc, hw->mac.addr); + if (error == 0) + ixl_vc_enqueue(&sc->vc_mgr, &sc->del_mac_cmd, + IXLV_FLAG_AQ_DEL_MAC_FILTER, ixl_init_cmd_complete, + sc); + } + + /* Check for an LAA mac address... */ + bcopy(IF_LLADDR(ifp), hw->mac.addr, ETHER_ADDR_LEN); + + ifp->if_hwassist = 0; + if (ifp->if_capenable & IFCAP_TSO) + ifp->if_hwassist |= CSUM_TSO; + if (ifp->if_capenable & IFCAP_TXCSUM) + ifp->if_hwassist |= (CSUM_OFFLOAD_IPV4 & ~CSUM_IP); + if (ifp->if_capenable & IFCAP_TXCSUM_IPV6) + ifp->if_hwassist |= CSUM_OFFLOAD_IPV6; + + /* Add mac filter for this VF to PF */ + if (i40e_validate_mac_addr(hw->mac.addr) == I40E_SUCCESS) { + error = ixlv_add_mac_filter(sc, hw->mac.addr, 0); + if (!error || error == EEXIST) + ixl_vc_enqueue(&sc->vc_mgr, &sc->add_mac_cmd, + IXLV_FLAG_AQ_ADD_MAC_FILTER, ixl_init_cmd_complete, + sc); + } + + /* Setup vlan's if needed */ + ixlv_setup_vlan_filters(sc); + + /* Prepare the queues for operation */ + for (int i = 0; i < vsi->num_queues; i++, que++) { + struct rx_ring *rxr = &que->rxr; + + ixl_init_tx_ring(que); + + if (vsi->max_frame_size <= MCLBYTES) + rxr->mbuf_sz = MCLBYTES; + else + rxr->mbuf_sz = MJUMPAGESIZE; + ixl_init_rx_ring(que); + } + + /* Configure queues */ + ixl_vc_enqueue(&sc->vc_mgr, &sc->config_queues_cmd, + IXLV_FLAG_AQ_CONFIGURE_QUEUES, ixl_init_cmd_complete, sc); + + /* Set up RSS */ + ixlv_config_rss(sc); + + /* Map vectors */ + ixl_vc_enqueue(&sc->vc_mgr, &sc->map_vectors_cmd, + IXLV_FLAG_AQ_MAP_VECTORS, ixl_init_cmd_complete, sc); + + /* Enable queues */ + ixl_vc_enqueue(&sc->vc_mgr, &sc->enable_queues_cmd, + IXLV_FLAG_AQ_ENABLE_QUEUES, ixl_init_cmd_complete, sc); + + /* Start the local timer */ + callout_reset(&sc->timer, hz, ixlv_local_timer, sc); + + sc->init_state = IXLV_RUNNING; + +init_done: + INIT_DBG_IF(ifp, "end"); + return; +} + +/* +** Init entry point for the stack +*/ +void +ixlv_init(void *arg) +{ + struct ixl_vsi *vsi = (struct ixl_vsi *)arg; + struct ixlv_sc *sc = vsi->back; + int retries = 0; + + mtx_lock(&sc->mtx); + ixlv_init_locked(sc); + mtx_unlock(&sc->mtx); + + /* Wait for init_locked to finish */ + while (!(vsi->ifp->if_drv_flags & IFF_DRV_RUNNING) + && ++retries < 100) { + i40e_msec_delay(10); + } + if (retries >= IXLV_AQ_MAX_ERR) + if_printf(vsi->ifp, + "Init failed to complete in alloted time!\n"); +} + +/* + * ixlv_attach() helper function; gathers information about + * the (virtual) hardware for use elsewhere in the driver. + */ +static void +ixlv_init_hw(struct ixlv_sc *sc) +{ + struct i40e_hw *hw = &sc->hw; + device_t dev = sc->dev; + + /* Save off the information about this board */ + hw->vendor_id = pci_get_vendor(dev); + hw->device_id = pci_get_device(dev); + hw->revision_id = pci_read_config(dev, PCIR_REVID, 1); + hw->subsystem_vendor_id = + pci_read_config(dev, PCIR_SUBVEND_0, 2); + hw->subsystem_device_id = + pci_read_config(dev, PCIR_SUBDEV_0, 2); + + hw->bus.device = pci_get_slot(dev); + hw->bus.func = pci_get_function(dev); +} + +/* + * ixlv_attach() helper function; initalizes the admin queue + * and attempts to establish contact with the PF by + * retrying the initial "API version" message several times + * or until the PF responds. + */ +static int +ixlv_setup_vc(struct ixlv_sc *sc) +{ + struct i40e_hw *hw = &sc->hw; + device_t dev = sc->dev; + int error = 0, ret_error = 0, asq_retries = 0; + bool send_api_ver_retried = 0; + + /* Need to set these AQ paramters before initializing AQ */ + hw->aq.num_arq_entries = IXL_AQ_LEN; + hw->aq.num_asq_entries = IXL_AQ_LEN; + hw->aq.arq_buf_size = IXL_AQ_BUFSZ; + hw->aq.asq_buf_size = IXL_AQ_BUFSZ; + + for (int i = 0; i < IXLV_AQ_MAX_ERR; i++) { + /* Initialize admin queue */ + error = i40e_init_adminq(hw); + if (error) { + device_printf(dev, "%s: init_adminq failed: %d\n", + __func__, error); + ret_error = 1; + continue; + } + + INIT_DBG_DEV(dev, "Initialized Admin Queue, attempt %d", i+1); + +retry_send: + /* Send VF's API version */ + error = ixlv_send_api_ver(sc); + if (error) { + i40e_shutdown_adminq(hw); + ret_error = 2; + device_printf(dev, "%s: unable to send api" + " version to PF on attempt %d, error %d\n", + __func__, i+1, error); + } + + asq_retries = 0; + while (!i40e_asq_done(hw)) { + if (++asq_retries > IXLV_AQ_MAX_ERR) { + i40e_shutdown_adminq(hw); + DDPRINTF(dev, "Admin Queue timeout " + "(waiting for send_api_ver), %d more retries...", + IXLV_AQ_MAX_ERR - (i + 1)); + ret_error = 3; + break; + } + i40e_msec_delay(10); + } + if (asq_retries > IXLV_AQ_MAX_ERR) + continue; + + INIT_DBG_DEV(dev, "Sent API version message to PF"); + + /* Verify that the VF accepts the PF's API version */ + error = ixlv_verify_api_ver(sc); + if (error == ETIMEDOUT) { + if (!send_api_ver_retried) { + /* Resend message, one more time */ + send_api_ver_retried++; + device_printf(dev, + "%s: Timeout while verifying API version on first" + " try!\n", __func__); + goto retry_send; + } else { + device_printf(dev, + "%s: Timeout while verifying API version on second" + " try!\n", __func__); + ret_error = 4; + break; + } + } + if (error) { + device_printf(dev, + "%s: Unable to verify API version," + " error %d\n", __func__, error); + ret_error = 5; + } + break; + } + + if (ret_error >= 4) + i40e_shutdown_adminq(hw); + return (ret_error); +} + +/* + * ixlv_attach() helper function; asks the PF for this VF's + * configuration, and saves the information if it receives it. + */ +static int +ixlv_vf_config(struct ixlv_sc *sc) +{ + struct i40e_hw *hw = &sc->hw; + device_t dev = sc->dev; + int bufsz, error = 0, ret_error = 0; + int asq_retries, retried = 0; + +retry_config: + error = ixlv_send_vf_config_msg(sc); + if (error) { + device_printf(dev, + "%s: Unable to send VF config request, attempt %d," + " error %d\n", __func__, retried + 1, error); + ret_error = 2; + } + + asq_retries = 0; + while (!i40e_asq_done(hw)) { + if (++asq_retries > IXLV_AQ_MAX_ERR) { + device_printf(dev, "%s: Admin Queue timeout " + "(waiting for send_vf_config_msg), attempt %d\n", + __func__, retried + 1); + ret_error = 3; + goto fail; + } + i40e_msec_delay(10); + } + + INIT_DBG_DEV(dev, "Sent VF config message to PF, attempt %d", + retried + 1); + + if (!sc->vf_res) { + bufsz = sizeof(struct i40e_virtchnl_vf_resource) + + (I40E_MAX_VF_VSI * sizeof(struct i40e_virtchnl_vsi_resource)); + sc->vf_res = malloc(bufsz, M_DEVBUF, M_NOWAIT); + if (!sc->vf_res) { + device_printf(dev, + "%s: Unable to allocate memory for VF configuration" + " message from PF on attempt %d\n", __func__, retried + 1); + ret_error = 1; + goto fail; + } + } + + /* Check for VF config response */ + error = ixlv_get_vf_config(sc); + if (error == ETIMEDOUT) { + /* The 1st time we timeout, send the configuration message again */ + if (!retried) { + retried++; + goto retry_config; + } + } + if (error) { + device_printf(dev, + "%s: Unable to get VF configuration from PF after %d tries!\n", + __func__, retried + 1); + ret_error = 4; + } + goto done; + +fail: + free(sc->vf_res, M_DEVBUF); +done: + return (ret_error); +} + +/* + * Allocate MSI/X vectors, setup the AQ vector early + */ +static int +ixlv_init_msix(struct ixlv_sc *sc) +{ + device_t dev = sc->dev; + int rid, want, vectors, queues, available; + + rid = PCIR_BAR(IXL_BAR); + sc->msix_mem = bus_alloc_resource_any(dev, + SYS_RES_MEMORY, &rid, RF_ACTIVE); + if (!sc->msix_mem) { + /* May not be enabled */ + device_printf(sc->dev, + "Unable to map MSIX table \n"); + goto fail; + } + + available = pci_msix_count(dev); + if (available == 0) { /* system has msix disabled */ + bus_release_resource(dev, SYS_RES_MEMORY, + rid, sc->msix_mem); + sc->msix_mem = NULL; + goto fail; + } + + /* Figure out a reasonable auto config value */ + queues = (mp_ncpus > (available - 1)) ? (available - 1) : mp_ncpus; + + /* Override with hardcoded value if sane */ + if ((ixlv_max_queues != 0) && (ixlv_max_queues <= queues)) + queues = ixlv_max_queues; +#ifdef RSS + /* If we're doing RSS, clamp at the number of RSS buckets */ + if (queues > rss_getnumbuckets()) + queues = rss_getnumbuckets(); +#endif + /* Enforce the VF max value */ + if (queues > IXLV_MAX_QUEUES) + queues = IXLV_MAX_QUEUES; + + /* + ** Want one vector (RX/TX pair) per queue + ** plus an additional for the admin queue. + */ + want = queues + 1; + if (want <= available) /* Have enough */ + vectors = want; + else { + device_printf(sc->dev, + "MSIX Configuration Problem, " + "%d vectors available but %d wanted!\n", + available, want); + goto fail; + } + +#ifdef RSS + /* + * If we're doing RSS, the number of queues needs to + * match the number of RSS buckets that are configured. + * + * + If there's more queues than RSS buckets, we'll end + * up with queues that get no traffic. + * + * + If there's more RSS buckets than queues, we'll end + * up having multiple RSS buckets map to the same queue, + * so there'll be some contention. + */ + if (queues != rss_getnumbuckets()) { + device_printf(dev, + "%s: queues (%d) != RSS buckets (%d)" + "; performance will be impacted.\n", + __func__, queues, rss_getnumbuckets()); + } +#endif + + if (pci_alloc_msix(dev, &vectors) == 0) { + device_printf(sc->dev, + "Using MSIX interrupts with %d vectors\n", vectors); + sc->msix = vectors; + sc->vsi.num_queues = queues; + } + + /* + ** Explicitly set the guest PCI BUSMASTER capability + ** and we must rewrite the ENABLE in the MSIX control + ** register again at this point to cause the host to + ** successfully initialize us. + */ + { + u16 pci_cmd_word; + int msix_ctrl; + pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); + pci_cmd_word |= PCIM_CMD_BUSMASTEREN; + pci_write_config(dev, PCIR_COMMAND, pci_cmd_word, 2); + pci_find_cap(dev, PCIY_MSIX, &rid); + rid += PCIR_MSIX_CTRL; + msix_ctrl = pci_read_config(dev, rid, 2); + msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE; + pci_write_config(dev, rid, msix_ctrl, 2); + } + + /* Next we need to setup the vector for the Admin Queue */ + rid = 1; // zero vector + 1 + sc->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, + &rid, RF_SHAREABLE | RF_ACTIVE); + if (sc->res == NULL) { + device_printf(dev,"Unable to allocate" + " bus resource: AQ interrupt \n"); + goto fail; + } + if (bus_setup_intr(dev, sc->res, + INTR_TYPE_NET | INTR_MPSAFE, NULL, + ixlv_msix_adminq, sc, &sc->tag)) { + sc->res = NULL; + device_printf(dev, "Failed to register AQ handler"); + goto fail; + } + bus_describe_intr(dev, sc->res, sc->tag, "adminq"); + + return (vectors); + +fail: + /* The VF driver MUST use MSIX */ + return (0); +} + +static int +ixlv_allocate_pci_resources(struct ixlv_sc *sc) +{ + int rid; + device_t dev = sc->dev; + + rid = PCIR_BAR(0); + sc->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, + &rid, RF_ACTIVE); + + if (!(sc->pci_mem)) { + device_printf(dev,"Unable to allocate bus resource: memory\n"); + return (ENXIO); + } + + sc->osdep.mem_bus_space_tag = + rman_get_bustag(sc->pci_mem); + sc->osdep.mem_bus_space_handle = + rman_get_bushandle(sc->pci_mem); + sc->osdep.mem_bus_space_size = rman_get_size(sc->pci_mem); + sc->osdep.flush_reg = I40E_VFGEN_RSTAT; + sc->hw.hw_addr = (u8 *) &sc->osdep.mem_bus_space_handle; + + sc->hw.back = &sc->osdep; + + /* Disable adminq interrupts */ + ixlv_disable_adminq_irq(&sc->hw); + + /* + ** Now setup MSI/X, it will return + ** us the number of supported vectors + */ + sc->msix = ixlv_init_msix(sc); + + /* We fail without MSIX support */ + if (sc->msix == 0) + return (ENXIO); + + return (0); +} + +static void +ixlv_free_pci_resources(struct ixlv_sc *sc) +{ + struct ixl_vsi *vsi = &sc->vsi; + struct ixl_queue *que = vsi->queues; + device_t dev = sc->dev; + + /* We may get here before stations are setup */ + if (que == NULL) + goto early; + + /* + ** Release all msix queue resources: + */ + for (int i = 0; i < vsi->num_queues; i++, que++) { + int rid = que->msix + 1; + if (que->tag != NULL) { + bus_teardown_intr(dev, que->res, que->tag); + que->tag = NULL; + } + if (que->res != NULL) + bus_release_resource(dev, SYS_RES_IRQ, rid, que->res); + } + +early: + /* Clean the AdminQ interrupt */ + if (sc->tag != NULL) { + bus_teardown_intr(dev, sc->res, sc->tag); + sc->tag = NULL; + } + if (sc->res != NULL) + bus_release_resource(dev, SYS_RES_IRQ, 1, sc->res); + + pci_release_msi(dev); + + if (sc->msix_mem != NULL) + bus_release_resource(dev, SYS_RES_MEMORY, + PCIR_BAR(IXL_BAR), sc->msix_mem); + + if (sc->pci_mem != NULL) + bus_release_resource(dev, SYS_RES_MEMORY, + PCIR_BAR(0), sc->pci_mem); + + return; +} + +/* + * Create taskqueue and tasklet for Admin Queue interrupts. + */ +static int +ixlv_init_taskqueue(struct ixlv_sc *sc) +{ + int error = 0; + + TASK_INIT(&sc->aq_irq, 0, ixlv_do_adminq, sc); + + sc->tq = taskqueue_create_fast("ixl_adm", M_NOWAIT, + taskqueue_thread_enqueue, &sc->tq); + taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s sc->tq", + device_get_nameunit(sc->dev)); + + return (error); +} + +/********************************************************************* + * + * Setup MSIX Interrupt resources and handlers for the VSI queues + * + **********************************************************************/ +static int +ixlv_assign_msix(struct ixlv_sc *sc) +{ + device_t dev = sc->dev; + struct ixl_vsi *vsi = &sc->vsi; + struct ixl_queue *que = vsi->queues; + struct tx_ring *txr; + int error, rid, vector = 1; +#ifdef RSS + cpuset_t cpu_mask; +#endif + + for (int i = 0; i < vsi->num_queues; i++, vector++, que++) { + int cpu_id = i; + rid = vector + 1; + txr = &que->txr; + que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, + RF_SHAREABLE | RF_ACTIVE); + if (que->res == NULL) { + device_printf(dev,"Unable to allocate" + " bus resource: que interrupt [%d]\n", vector); + return (ENXIO); + } + /* Set the handler function */ + error = bus_setup_intr(dev, que->res, + INTR_TYPE_NET | INTR_MPSAFE, NULL, + ixlv_msix_que, que, &que->tag); + if (error) { + que->res = NULL; + device_printf(dev, "Failed to register que handler"); + return (error); + } + bus_describe_intr(dev, que->res, que->tag, "que %d", i); + /* Bind the vector to a CPU */ +#ifdef RSS + cpu_id = rss_getcpu(i % rss_getnumbuckets()); +#endif + bus_bind_intr(dev, que->res, cpu_id); + que->msix = vector; + vsi->que_mask |= (u64)(1 << que->msix); + TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que); + TASK_INIT(&que->task, 0, ixlv_handle_que, que); + que->tq = taskqueue_create_fast("ixlv_que", M_NOWAIT, + taskqueue_thread_enqueue, &que->tq); +#ifdef RSS + CPU_SETOF(cpu_id, &cpu_mask); + taskqueue_start_threads_cpuset(&que->tq, 1, PI_NET, + &cpu_mask, "%s (bucket %d)", + device_get_nameunit(dev), cpu_id); +#else + taskqueue_start_threads(&que->tq, 1, PI_NET, + "%s que", device_get_nameunit(dev)); +#endif + + } + + return (0); +} + +/* +** Requests a VF reset from the PF. +** +** Requires the VF's Admin Queue to be initialized. +*/ +static int +ixlv_reset(struct ixlv_sc *sc) +{ + struct i40e_hw *hw = &sc->hw; + device_t dev = sc->dev; + int error = 0; + + /* Ask the PF to reset us if we are initiating */ + if (sc->init_state != IXLV_RESET_PENDING) + ixlv_request_reset(sc); + + i40e_msec_delay(100); + error = ixlv_reset_complete(hw); + if (error) { + device_printf(dev, "%s: VF reset failed\n", + __func__); + return (error); + } + + error = i40e_shutdown_adminq(hw); + if (error) { + device_printf(dev, "%s: shutdown_adminq failed: %d\n", + __func__, error); + return (error); + } + + error = i40e_init_adminq(hw); + if (error) { + device_printf(dev, "%s: init_adminq failed: %d\n", + __func__, error); + return(error); + } + + return (0); +} + +static int +ixlv_reset_complete(struct i40e_hw *hw) +{ + u32 reg; + + for (int i = 0; i < 100; i++) { + reg = rd32(hw, I40E_VFGEN_RSTAT) & + I40E_VFGEN_RSTAT_VFR_STATE_MASK; + + if ((reg == I40E_VFR_VFACTIVE) || + (reg == I40E_VFR_COMPLETED)) + return (0); + i40e_msec_delay(100); + } + + return (EBUSY); +} + + +/********************************************************************* + * + * Setup networking device structure and register an interface. + * + **********************************************************************/ +static int +ixlv_setup_interface(device_t dev, struct ixlv_sc *sc) +{ + struct ifnet *ifp; + struct ixl_vsi *vsi = &sc->vsi; + struct ixl_queue *que = vsi->queues; + + INIT_DBG_DEV(dev, "begin"); + + ifp = vsi->ifp = if_alloc(IFT_ETHER); + if (ifp == NULL) { + device_printf(dev, "%s: could not allocate ifnet" + " structure!\n", __func__); + return (-1); + } + + if_initname(ifp, device_get_name(dev), device_get_unit(dev)); + + ifp->if_mtu = ETHERMTU; + ifp->if_baudrate = 4000000000; // ?? + ifp->if_init = ixlv_init; + ifp->if_softc = vsi; + ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; + ifp->if_ioctl = ixlv_ioctl; + +#if __FreeBSD_version >= 1100000 + if_setgetcounterfn(ifp, ixl_get_counter); +#endif + + ifp->if_transmit = ixl_mq_start; + + ifp->if_qflush = ixl_qflush; + ifp->if_snd.ifq_maxlen = que->num_desc - 2; + + ether_ifattach(ifp, sc->hw.mac.addr); + + vsi->max_frame_size = + ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN + + ETHER_VLAN_ENCAP_LEN; + + /* + * Tell the upper layer(s) we support long frames. + */ + ifp->if_hdrlen = sizeof(struct ether_vlan_header); + + ifp->if_capabilities |= IFCAP_HWCSUM; + ifp->if_capabilities |= IFCAP_HWCSUM_IPV6; + ifp->if_capabilities |= IFCAP_TSO; + ifp->if_capabilities |= IFCAP_JUMBO_MTU; + + ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING + | IFCAP_VLAN_HWTSO + | IFCAP_VLAN_MTU + | IFCAP_VLAN_HWCSUM + | IFCAP_LRO; + ifp->if_capenable = ifp->if_capabilities; + + /* + ** Don't turn this on by default, if vlans are + ** created on another pseudo device (eg. lagg) + ** then vlan events are not passed thru, breaking + ** operation, but with HW FILTER off it works. If + ** using vlans directly on the ixl driver you can + ** enable this and get full hardware tag filtering. + */ + ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; + + /* + * Specify the media types supported by this adapter and register + * callbacks to update media and link information + */ + ifmedia_init(&sc->media, IFM_IMASK, ixlv_media_change, + ixlv_media_status); + + // JFV Add media types later? + + ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); + ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); + + INIT_DBG_DEV(dev, "end"); + return (0); +} + +/* +** Allocate and setup the interface queues +*/ +static int +ixlv_setup_queues(struct ixlv_sc *sc) +{ + device_t dev = sc->dev; + struct ixl_vsi *vsi; + struct ixl_queue *que; + struct tx_ring *txr; + struct rx_ring *rxr; + int rsize, tsize; + int error = I40E_SUCCESS; + + vsi = &sc->vsi; + vsi->back = (void *)sc; + vsi->hw = &sc->hw; + vsi->num_vlans = 0; + + /* Get memory for the station queues */ + if (!(vsi->queues = + (struct ixl_queue *) malloc(sizeof(struct ixl_queue) * + vsi->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { + device_printf(dev, "Unable to allocate queue memory\n"); + error = ENOMEM; + goto early; + } + + for (int i = 0; i < vsi->num_queues; i++) { + que = &vsi->queues[i]; + que->num_desc = ixlv_ringsz; + que->me = i; + que->vsi = vsi; + /* mark the queue as active */ + vsi->active_queues |= (u64)1 << que->me; + + txr = &que->txr; + txr->que = que; + txr->tail = I40E_QTX_TAIL1(que->me); + /* Initialize the TX lock */ + snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", + device_get_nameunit(dev), que->me); + mtx_init(&txr->mtx, txr->mtx_name, NULL, MTX_DEF); + /* + ** Create the TX descriptor ring, the extra int is + ** added as the location for HEAD WB. + */ + tsize = roundup2((que->num_desc * + sizeof(struct i40e_tx_desc)) + + sizeof(u32), DBA_ALIGN); + if (i40e_allocate_dma_mem(&sc->hw, + &txr->dma, i40e_mem_reserved, tsize, DBA_ALIGN)) { + device_printf(dev, + "Unable to allocate TX Descriptor memory\n"); + error = ENOMEM; + goto fail; + } + txr->base = (struct i40e_tx_desc *)txr->dma.va; + bzero((void *)txr->base, tsize); + /* Now allocate transmit soft structs for the ring */ + if (ixl_allocate_tx_data(que)) { + device_printf(dev, + "Critical Failure setting up TX structures\n"); + error = ENOMEM; + goto fail; + } + /* Allocate a buf ring */ + txr->br = buf_ring_alloc(ixlv_txbrsz, M_DEVBUF, + M_WAITOK, &txr->mtx); + if (txr->br == NULL) { + device_printf(dev, + "Critical Failure setting up TX buf ring\n"); + error = ENOMEM; + goto fail; + } + + /* + * Next the RX queues... + */ + rsize = roundup2(que->num_desc * + sizeof(union i40e_rx_desc), DBA_ALIGN); + rxr = &que->rxr; + rxr->que = que; + rxr->tail = I40E_QRX_TAIL1(que->me); + + /* Initialize the RX side lock */ + snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", + device_get_nameunit(dev), que->me); + mtx_init(&rxr->mtx, rxr->mtx_name, NULL, MTX_DEF); + + if (i40e_allocate_dma_mem(&sc->hw, + &rxr->dma, i40e_mem_reserved, rsize, 4096)) { //JFV - should this be DBA? + device_printf(dev, + "Unable to allocate RX Descriptor memory\n"); + error = ENOMEM; + goto fail; + } + rxr->base = (union i40e_rx_desc *)rxr->dma.va; + bzero((void *)rxr->base, rsize); + + /* Allocate receive soft structs for the ring*/ + if (ixl_allocate_rx_data(que)) { + device_printf(dev, + "Critical Failure setting up receive structs\n"); + error = ENOMEM; + goto fail; + } + } + + return (0); + +fail: + for (int i = 0; i < vsi->num_queues; i++) { + que = &vsi->queues[i]; + rxr = &que->rxr; + txr = &que->txr; + if (rxr->base) + i40e_free_dma_mem(&sc->hw, &rxr->dma); + if (txr->base) + i40e_free_dma_mem(&sc->hw, &txr->dma); + } + free(vsi->queues, M_DEVBUF); + +early: + return (error); +} + +/* +** This routine is run via an vlan config EVENT, +** it enables us to use the HW Filter table since +** we can get the vlan id. This just creates the +** entry in the soft version of the VFTA, init will +** repopulate the real table. +*/ +static void +ixlv_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) +{ + struct ixl_vsi *vsi = arg; + struct ixlv_sc *sc = vsi->back; + struct ixlv_vlan_filter *v; + + + if (ifp->if_softc != arg) /* Not our event */ + return; + + if ((vtag == 0) || (vtag > 4095)) /* Invalid */ + return; + + /* Sanity check - make sure it doesn't already exist */ + SLIST_FOREACH(v, sc->vlan_filters, next) { + if (v->vlan == vtag) + return; + } + + mtx_lock(&sc->mtx); + ++vsi->num_vlans; + v = malloc(sizeof(struct ixlv_vlan_filter), M_DEVBUF, M_NOWAIT | M_ZERO); + SLIST_INSERT_HEAD(sc->vlan_filters, v, next); + v->vlan = vtag; + v->flags = IXL_FILTER_ADD; + ixl_vc_enqueue(&sc->vc_mgr, &sc->add_vlan_cmd, + IXLV_FLAG_AQ_ADD_VLAN_FILTER, ixl_init_cmd_complete, sc); + mtx_unlock(&sc->mtx); + return; +} + +/* +** This routine is run via an vlan +** unconfig EVENT, remove our entry +** in the soft vfta. +*/ +static void +ixlv_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) +{ + struct ixl_vsi *vsi = arg; + struct ixlv_sc *sc = vsi->back; + struct ixlv_vlan_filter *v; + int i = 0; + + if (ifp->if_softc != arg) + return; + + if ((vtag == 0) || (vtag > 4095)) /* Invalid */ + return; + + mtx_lock(&sc->mtx); + SLIST_FOREACH(v, sc->vlan_filters, next) { + if (v->vlan == vtag) { + v->flags = IXL_FILTER_DEL; + ++i; + --vsi->num_vlans; + } + } + if (i) + ixl_vc_enqueue(&sc->vc_mgr, &sc->del_vlan_cmd, + IXLV_FLAG_AQ_DEL_VLAN_FILTER, ixl_init_cmd_complete, sc); + mtx_unlock(&sc->mtx); + return; +} + +/* +** Get a new filter and add it to the mac filter list. +*/ +static struct ixlv_mac_filter * +ixlv_get_mac_filter(struct ixlv_sc *sc) +{ + struct ixlv_mac_filter *f; + + f = malloc(sizeof(struct ixlv_mac_filter), + M_DEVBUF, M_NOWAIT | M_ZERO); + if (f) + SLIST_INSERT_HEAD(sc->mac_filters, f, next); + + return (f); +} + +/* +** Find the filter with matching MAC address +*/ +static struct ixlv_mac_filter * +ixlv_find_mac_filter(struct ixlv_sc *sc, u8 *macaddr) +{ + struct ixlv_mac_filter *f; + bool match = FALSE; + + SLIST_FOREACH(f, sc->mac_filters, next) { + if (cmp_etheraddr(f->macaddr, macaddr)) { + match = TRUE; + break; + } + } + + if (!match) + f = NULL; + return (f); +} + +/* +** Admin Queue interrupt handler +*/ +static void +ixlv_msix_adminq(void *arg) +{ + struct ixlv_sc *sc = arg; + struct i40e_hw *hw = &sc->hw; + u32 reg, mask; + + reg = rd32(hw, I40E_VFINT_ICR01); + mask = rd32(hw, I40E_VFINT_ICR0_ENA1); + + reg = rd32(hw, I40E_VFINT_DYN_CTL01); + reg |= I40E_VFINT_DYN_CTL01_CLEARPBA_MASK; + wr32(hw, I40E_VFINT_DYN_CTL01, reg); + + /* schedule task */ + taskqueue_enqueue(sc->tq, &sc->aq_irq); + return; +} + +void +ixlv_enable_intr(struct ixl_vsi *vsi) +{ + struct i40e_hw *hw = vsi->hw; + struct ixl_queue *que = vsi->queues; + + ixlv_enable_adminq_irq(hw); + for (int i = 0; i < vsi->num_queues; i++, que++) + ixlv_enable_queue_irq(hw, que->me); +} + +void +ixlv_disable_intr(struct ixl_vsi *vsi) +{ + struct i40e_hw *hw = vsi->hw; + struct ixl_queue *que = vsi->queues; + + ixlv_disable_adminq_irq(hw); + for (int i = 0; i < vsi->num_queues; i++, que++) + ixlv_disable_queue_irq(hw, que->me); +} + + +static void +ixlv_disable_adminq_irq(struct i40e_hw *hw) +{ + wr32(hw, I40E_VFINT_DYN_CTL01, 0); + wr32(hw, I40E_VFINT_ICR0_ENA1, 0); + /* flush */ + rd32(hw, I40E_VFGEN_RSTAT); + return; +} + +static void +ixlv_enable_adminq_irq(struct i40e_hw *hw) +{ + wr32(hw, I40E_VFINT_DYN_CTL01, + I40E_VFINT_DYN_CTL01_INTENA_MASK | + I40E_VFINT_DYN_CTL01_ITR_INDX_MASK); + wr32(hw, I40E_VFINT_ICR0_ENA1, I40E_VFINT_ICR0_ENA1_ADMINQ_MASK); + /* flush */ + rd32(hw, I40E_VFGEN_RSTAT); + return; +} + +static void +ixlv_enable_queue_irq(struct i40e_hw *hw, int id) +{ + u32 reg; + + reg = I40E_VFINT_DYN_CTLN1_INTENA_MASK | + I40E_VFINT_DYN_CTLN1_CLEARPBA_MASK; + wr32(hw, I40E_VFINT_DYN_CTLN1(id), reg); +} + +static void +ixlv_disable_queue_irq(struct i40e_hw *hw, int id) +{ + wr32(hw, I40E_VFINT_DYN_CTLN1(id), 0); + rd32(hw, I40E_VFGEN_RSTAT); + return; +} + + +/* +** Provide a update to the queue RX +** interrupt moderation value. +*/ +static void +ixlv_set_queue_rx_itr(struct ixl_queue *que) +{ + struct ixl_vsi *vsi = que->vsi; + struct i40e_hw *hw = vsi->hw; + struct rx_ring *rxr = &que->rxr; + u16 rx_itr; + u16 rx_latency = 0; + int rx_bytes; + + + /* Idle, do nothing */ + if (rxr->bytes == 0) + return; + + if (ixlv_dynamic_rx_itr) { + rx_bytes = rxr->bytes/rxr->itr; + rx_itr = rxr->itr; + + /* Adjust latency range */ + switch (rxr->latency) { + case IXL_LOW_LATENCY: + if (rx_bytes > 10) { + rx_latency = IXL_AVE_LATENCY; + rx_itr = IXL_ITR_20K; + } + break; + case IXL_AVE_LATENCY: + if (rx_bytes > 20) { + rx_latency = IXL_BULK_LATENCY; + rx_itr = IXL_ITR_8K; + } else if (rx_bytes <= 10) { + rx_latency = IXL_LOW_LATENCY; + rx_itr = IXL_ITR_100K; + } + break; + case IXL_BULK_LATENCY: + if (rx_bytes <= 20) { + rx_latency = IXL_AVE_LATENCY; + rx_itr = IXL_ITR_20K; + } + break; + } + + rxr->latency = rx_latency; + + if (rx_itr != rxr->itr) { + /* do an exponential smoothing */ + rx_itr = (10 * rx_itr * rxr->itr) / + ((9 * rx_itr) + rxr->itr); + rxr->itr = rx_itr & IXL_MAX_ITR; + wr32(hw, I40E_VFINT_ITRN1(IXL_RX_ITR, + que->me), rxr->itr); + } + } else { /* We may have have toggled to non-dynamic */ + if (vsi->rx_itr_setting & IXL_ITR_DYNAMIC) + vsi->rx_itr_setting = ixlv_rx_itr; + /* Update the hardware if needed */ + if (rxr->itr != vsi->rx_itr_setting) { + rxr->itr = vsi->rx_itr_setting; + wr32(hw, I40E_VFINT_ITRN1(IXL_RX_ITR, + que->me), rxr->itr); + } + } + rxr->bytes = 0; + rxr->packets = 0; + return; +} + + +/* +** Provide a update to the queue TX +** interrupt moderation value. +*/ +static void +ixlv_set_queue_tx_itr(struct ixl_queue *que) +{ + struct ixl_vsi *vsi = que->vsi; + struct i40e_hw *hw = vsi->hw; + struct tx_ring *txr = &que->txr; + u16 tx_itr; + u16 tx_latency = 0; + int tx_bytes; + + + /* Idle, do nothing */ + if (txr->bytes == 0) + return; + + if (ixlv_dynamic_tx_itr) { + tx_bytes = txr->bytes/txr->itr; + tx_itr = txr->itr; + + switch (txr->latency) { + case IXL_LOW_LATENCY: + if (tx_bytes > 10) { + tx_latency = IXL_AVE_LATENCY; + tx_itr = IXL_ITR_20K; + } + break; + case IXL_AVE_LATENCY: + if (tx_bytes > 20) { + tx_latency = IXL_BULK_LATENCY; + tx_itr = IXL_ITR_8K; + } else if (tx_bytes <= 10) { + tx_latency = IXL_LOW_LATENCY; + tx_itr = IXL_ITR_100K; + } + break; + case IXL_BULK_LATENCY: + if (tx_bytes <= 20) { + tx_latency = IXL_AVE_LATENCY; + tx_itr = IXL_ITR_20K; + } + break; + } + + txr->latency = tx_latency; + + if (tx_itr != txr->itr) { + /* do an exponential smoothing */ + tx_itr = (10 * tx_itr * txr->itr) / + ((9 * tx_itr) + txr->itr); + txr->itr = tx_itr & IXL_MAX_ITR; + wr32(hw, I40E_VFINT_ITRN1(IXL_TX_ITR, + que->me), txr->itr); + } + + } else { /* We may have have toggled to non-dynamic */ + if (vsi->tx_itr_setting & IXL_ITR_DYNAMIC) + vsi->tx_itr_setting = ixlv_tx_itr; + /* Update the hardware if needed */ + if (txr->itr != vsi->tx_itr_setting) { + txr->itr = vsi->tx_itr_setting; + wr32(hw, I40E_VFINT_ITRN1(IXL_TX_ITR, + que->me), txr->itr); + } + } + txr->bytes = 0; + txr->packets = 0; + return; +} + + +/* +** +** MSIX Interrupt Handlers and Tasklets +** +*/ +static void +ixlv_handle_que(void *context, int pending) +{ + struct ixl_queue *que = context; + struct ixl_vsi *vsi = que->vsi; + struct i40e_hw *hw = vsi->hw; + struct tx_ring *txr = &que->txr; + struct ifnet *ifp = vsi->ifp; + bool more; + + if (ifp->if_drv_flags & IFF_DRV_RUNNING) { + more = ixl_rxeof(que, IXL_RX_LIMIT); + mtx_lock(&txr->mtx); + ixl_txeof(que); + if (!drbr_empty(ifp, txr->br)) + ixl_mq_start_locked(ifp, txr); + mtx_unlock(&txr->mtx); + if (more) { + taskqueue_enqueue(que->tq, &que->task); + return; + } + } + + /* Reenable this interrupt - hmmm */ + ixlv_enable_queue_irq(hw, que->me); + return; +} + + +/********************************************************************* + * + * MSIX Queue Interrupt Service routine + * + **********************************************************************/ +static void +ixlv_msix_que(void *arg) +{ + struct ixl_queue *que = arg; + struct ixl_vsi *vsi = que->vsi; + struct i40e_hw *hw = vsi->hw; + struct tx_ring *txr = &que->txr; + bool more_tx, more_rx; + + /* Spurious interrupts are ignored */ + if (!(vsi->ifp->if_drv_flags & IFF_DRV_RUNNING)) + return; + + ++que->irqs; + + more_rx = ixl_rxeof(que, IXL_RX_LIMIT); + + mtx_lock(&txr->mtx); + more_tx = ixl_txeof(que); + /* + ** Make certain that if the stack + ** has anything queued the task gets + ** scheduled to handle it. + */ + if (!drbr_empty(vsi->ifp, txr->br)) + more_tx = 1; + mtx_unlock(&txr->mtx); + + ixlv_set_queue_rx_itr(que); + ixlv_set_queue_tx_itr(que); + + if (more_tx || more_rx) + taskqueue_enqueue(que->tq, &que->task); + else + ixlv_enable_queue_irq(hw, que->me); + + return; +} + + +/********************************************************************* + * + * Media Ioctl callback + * + * This routine is called whenever the user queries the status of + * the interface using ifconfig. + * + **********************************************************************/ +static void +ixlv_media_status(struct ifnet * ifp, struct ifmediareq * ifmr) +{ + struct ixl_vsi *vsi = ifp->if_softc; + struct ixlv_sc *sc = vsi->back; + + INIT_DBG_IF(ifp, "begin"); + + mtx_lock(&sc->mtx); + + ixlv_update_link_status(sc); + + ifmr->ifm_status = IFM_AVALID; + ifmr->ifm_active = IFM_ETHER; + + if (!sc->link_up) { + mtx_unlock(&sc->mtx); + INIT_DBG_IF(ifp, "end: link not up"); + return; + } + + ifmr->ifm_status |= IFM_ACTIVE; + /* Hardware is always full-duplex */ + ifmr->ifm_active |= IFM_FDX; + mtx_unlock(&sc->mtx); + INIT_DBG_IF(ifp, "end"); + return; +} + +/********************************************************************* + * + * Media Ioctl callback + * + * This routine is called when the user changes speed/duplex using + * media/mediopt option with ifconfig. + * + **********************************************************************/ +static int +ixlv_media_change(struct ifnet * ifp) +{ + struct ixl_vsi *vsi = ifp->if_softc; + struct ifmedia *ifm = &vsi->media; + + INIT_DBG_IF(ifp, "begin"); + + if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) + return (EINVAL); + + INIT_DBG_IF(ifp, "end"); + return (0); +} + + +/********************************************************************* + * Multicast Initialization + * + * This routine is called by init to reset a fresh state. + * + **********************************************************************/ + +static void +ixlv_init_multi(struct ixl_vsi *vsi) +{ + struct ixlv_mac_filter *f; + struct ixlv_sc *sc = vsi->back; + int mcnt = 0; + + IOCTL_DBG_IF(vsi->ifp, "begin"); + + /* First clear any multicast filters */ + SLIST_FOREACH(f, sc->mac_filters, next) { + if ((f->flags & IXL_FILTER_USED) + && (f->flags & IXL_FILTER_MC)) { + f->flags |= IXL_FILTER_DEL; + mcnt++; + } + } + if (mcnt > 0) + ixl_vc_enqueue(&sc->vc_mgr, &sc->del_multi_cmd, + IXLV_FLAG_AQ_DEL_MAC_FILTER, ixl_init_cmd_complete, + sc); + + IOCTL_DBG_IF(vsi->ifp, "end"); +} + +static void +ixlv_add_multi(struct ixl_vsi *vsi) +{ + struct ifmultiaddr *ifma; + struct ifnet *ifp = vsi->ifp; + struct ixlv_sc *sc = vsi->back; + int mcnt = 0; + + IOCTL_DBG_IF(ifp, "begin"); + + if_maddr_rlock(ifp); + /* + ** Get a count, to decide if we + ** simply use multicast promiscuous. + */ + TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + if (ifma->ifma_addr->sa_family != AF_LINK) + continue; + mcnt++; + } + if_maddr_runlock(ifp); + + // TODO: Remove -- cannot set promiscuous mode in a VF + if (__predict_false(mcnt >= MAX_MULTICAST_ADDR)) { + /* delete all multicast filters */ + ixlv_init_multi(vsi); + sc->promiscuous_flags |= I40E_FLAG_VF_MULTICAST_PROMISC; + ixl_vc_enqueue(&sc->vc_mgr, &sc->add_multi_cmd, + IXLV_FLAG_AQ_CONFIGURE_PROMISC, ixl_init_cmd_complete, + sc); + IOCTL_DEBUGOUT("%s: end: too many filters", __func__); + return; + } + + mcnt = 0; + if_maddr_rlock(ifp); + TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + if (ifma->ifma_addr->sa_family != AF_LINK) + continue; + if (!ixlv_add_mac_filter(sc, + (u8*)LLADDR((struct sockaddr_dl *) ifma->ifma_addr), + IXL_FILTER_MC)) + mcnt++; + } + if_maddr_runlock(ifp); + /* + ** Notify AQ task that sw filters need to be + ** added to hw list + */ + if (mcnt > 0) + ixl_vc_enqueue(&sc->vc_mgr, &sc->add_multi_cmd, + IXLV_FLAG_AQ_ADD_MAC_FILTER, ixl_init_cmd_complete, + sc); + + IOCTL_DBG_IF(ifp, "end"); +} + +static void +ixlv_del_multi(struct ixl_vsi *vsi) +{ + struct ixlv_mac_filter *f; + struct ifmultiaddr *ifma; + struct ifnet *ifp = vsi->ifp; + struct ixlv_sc *sc = vsi->back; + int mcnt = 0; + bool match = FALSE; + + IOCTL_DBG_IF(ifp, "begin"); + + /* Search for removed multicast addresses */ + if_maddr_rlock(ifp); + SLIST_FOREACH(f, sc->mac_filters, next) { + if ((f->flags & IXL_FILTER_USED) + && (f->flags & IXL_FILTER_MC)) { + /* check if mac address in filter is in sc's list */ + match = FALSE; + TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { + if (ifma->ifma_addr->sa_family != AF_LINK) + continue; + u8 *mc_addr = + (u8 *)LLADDR((struct sockaddr_dl *)ifma->ifma_addr); + if (cmp_etheraddr(f->macaddr, mc_addr)) { + match = TRUE; + break; + } + } + /* if this filter is not in the sc's list, remove it */ + if (match == FALSE && !(f->flags & IXL_FILTER_DEL)) { + f->flags |= IXL_FILTER_DEL; + mcnt++; + IOCTL_DBG_IF(ifp, "marked: " MAC_FORMAT, + MAC_FORMAT_ARGS(f->macaddr)); + } + else if (match == FALSE) + IOCTL_DBG_IF(ifp, "exists: " MAC_FORMAT, + MAC_FORMAT_ARGS(f->macaddr)); + } + } + if_maddr_runlock(ifp); + + if (mcnt > 0) + ixl_vc_enqueue(&sc->vc_mgr, &sc->del_multi_cmd, + IXLV_FLAG_AQ_DEL_MAC_FILTER, ixl_init_cmd_complete, + sc); + + IOCTL_DBG_IF(ifp, "end"); +} + +/********************************************************************* + * Timer routine + * + * This routine checks for link status,updates statistics, + * and runs the watchdog check. + * + **********************************************************************/ + +static void +ixlv_local_timer(void *arg) +{ + struct ixlv_sc *sc = arg; + struct i40e_hw *hw = &sc->hw; + struct ixl_vsi *vsi = &sc->vsi; + struct ixl_queue *que = vsi->queues; + device_t dev = sc->dev; + int hung = 0; + u32 mask, val; + + IXLV_CORE_LOCK_ASSERT(sc); + + /* If Reset is in progress just bail */ + if (sc->init_state == IXLV_RESET_PENDING) + return; + + /* Check for when PF triggers a VF reset */ + val = rd32(hw, I40E_VFGEN_RSTAT) & + I40E_VFGEN_RSTAT_VFR_STATE_MASK; + + if (val != I40E_VFR_VFACTIVE + && val != I40E_VFR_COMPLETED) { + DDPRINTF(dev, "reset in progress! (%d)", val); + return; + } + + ixlv_request_stats(sc); + + /* clean and process any events */ + taskqueue_enqueue(sc->tq, &sc->aq_irq); + + /* + ** Check status on the queues for a hang + */ + mask = (I40E_VFINT_DYN_CTLN1_INTENA_MASK | + I40E_VFINT_DYN_CTLN1_SWINT_TRIG_MASK); + + for (int i = 0; i < vsi->num_queues; i++,que++) { + /* Any queues with outstanding work get a sw irq */ + if (que->busy) + wr32(hw, I40E_VFINT_DYN_CTLN1(que->me), mask); + /* + ** Each time txeof runs without cleaning, but there + ** are uncleaned descriptors it increments busy. If + ** we get to 5 we declare it hung. + */ + if (que->busy == IXL_QUEUE_HUNG) { + ++hung; + /* Mark the queue as inactive */ + vsi->active_queues &= ~((u64)1 << que->me); + continue; + } else { + /* Check if we've come back from hung */ + if ((vsi->active_queues & ((u64)1 << que->me)) == 0) + vsi->active_queues |= ((u64)1 << que->me); + } + if (que->busy >= IXL_MAX_TX_BUSY) { + device_printf(dev,"Warning queue %d " + "appears to be hung!\n", i); + que->busy = IXL_QUEUE_HUNG; + ++hung; + } + } + /* Only reset when all queues show hung */ + if (hung == vsi->num_queues) + goto hung; + callout_reset(&sc->timer, hz, ixlv_local_timer, sc); + return; + +hung: + device_printf(dev, "Local Timer: TX HANG DETECTED - Resetting!!\n"); + sc->init_state = IXLV_RESET_REQUIRED; + ixlv_init_locked(sc); +} + +/* +** Note: this routine updates the OS on the link state +** the real check of the hardware only happens with +** a link interrupt. +*/ +void +ixlv_update_link_status(struct ixlv_sc *sc) +{ + struct ixl_vsi *vsi = &sc->vsi; + struct ifnet *ifp = vsi->ifp; + + if (sc->link_up){ + if (vsi->link_active == FALSE) { + if (bootverbose) + if_printf(ifp,"Link is Up, %d Gbps\n", + (sc->link_speed == I40E_LINK_SPEED_40GB) ? 40:10); + vsi->link_active = TRUE; + if_link_state_change(ifp, LINK_STATE_UP); + } + } else { /* Link down */ + if (vsi->link_active == TRUE) { + if (bootverbose) + if_printf(ifp,"Link is Down\n"); + if_link_state_change(ifp, LINK_STATE_DOWN); + vsi->link_active = FALSE; + } + } + + return; +} + +/********************************************************************* + * + * This routine disables all traffic on the adapter by issuing a + * global reset on the MAC and deallocates TX/RX buffers. + * + **********************************************************************/ + +static void +ixlv_stop(struct ixlv_sc *sc) +{ + struct ifnet *ifp; + int start; + + ifp = sc->vsi.ifp; + INIT_DBG_IF(ifp, "begin"); + + IXLV_CORE_LOCK_ASSERT(sc); + + ixl_vc_flush(&sc->vc_mgr); + ixlv_disable_queues(sc); + + start = ticks; + while ((ifp->if_drv_flags & IFF_DRV_RUNNING) && + ((ticks - start) < hz/10)) + ixlv_do_adminq_locked(sc); + + /* Stop the local timer */ + callout_stop(&sc->timer); + + INIT_DBG_IF(ifp, "end"); +} + + +/********************************************************************* + * + * Free all station queue structs. + * + **********************************************************************/ +static void +ixlv_free_queues(struct ixl_vsi *vsi) +{ + struct ixlv_sc *sc = (struct ixlv_sc *)vsi->back; + struct ixl_queue *que = vsi->queues; + + for (int i = 0; i < vsi->num_queues; i++, que++) { + struct tx_ring *txr = &que->txr; + struct rx_ring *rxr = &que->rxr; + + if (!mtx_initialized(&txr->mtx)) /* uninitialized */ + continue; + IXL_TX_LOCK(txr); + ixl_free_que_tx(que); + if (txr->base) + i40e_free_dma_mem(&sc->hw, &txr->dma); + IXL_TX_UNLOCK(txr); + IXL_TX_LOCK_DESTROY(txr); + + if (!mtx_initialized(&rxr->mtx)) /* uninitialized */ + continue; + IXL_RX_LOCK(rxr); + ixl_free_que_rx(que); + if (rxr->base) + i40e_free_dma_mem(&sc->hw, &rxr->dma); + IXL_RX_UNLOCK(rxr); + IXL_RX_LOCK_DESTROY(rxr); + + } + free(vsi->queues, M_DEVBUF); +} + + +/* +** ixlv_config_rss - setup RSS +** +** RSS keys and table are cleared on VF reset. +*/ +static void +ixlv_config_rss(struct ixlv_sc *sc) +{ + struct i40e_hw *hw = &sc->hw; + struct ixl_vsi *vsi = &sc->vsi; + u32 lut = 0; + u64 set_hena = 0, hena; + int i, j, que_id; +#ifdef RSS + u32 rss_hash_config; + u32 rss_seed[IXL_KEYSZ]; +#else + u32 rss_seed[IXL_KEYSZ] = {0x41b01687, + 0x183cfd8c, 0xce880440, 0x580cbc3c, + 0x35897377, 0x328b25e1, 0x4fa98922, + 0xb7d90c14, 0xd5bad70d, 0xcd15a2c1}; +#endif + + /* Don't set up RSS if using a single queue */ + if (vsi->num_queues == 1) { + wr32(hw, I40E_VFQF_HENA(0), 0); + wr32(hw, I40E_VFQF_HENA(1), 0); + ixl_flush(hw); + return; + } + +#ifdef RSS + /* Fetch the configured RSS key */ + rss_getkey((uint8_t *) &rss_seed); +#endif + /* Fill out hash function seed */ + for (i = 0; i <= IXL_KEYSZ; i++) + wr32(hw, I40E_VFQF_HKEY(i), rss_seed[i]); + + /* Enable PCTYPES for RSS: */ +#ifdef RSS + rss_hash_config = rss_gethashconfig(); + if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4) + set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER); + if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4) + set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP); + if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4) + set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP); + if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6) + set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER); + if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX) + set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6); + if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6) + set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP); + if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6) + set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP); +#else + set_hena = + ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP) | + ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP) | + ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) | + ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) | + ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV4) | + ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP) | + ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP) | + ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) | + ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) | + ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6) | + ((u64)1 << I40E_FILTER_PCTYPE_L2_PAYLOAD); +#endif + hena = (u64)rd32(hw, I40E_VFQF_HENA(0)) | + ((u64)rd32(hw, I40E_VFQF_HENA(1)) << 32); + hena |= set_hena; + wr32(hw, I40E_VFQF_HENA(0), (u32)hena); + wr32(hw, I40E_VFQF_HENA(1), (u32)(hena >> 32)); + + /* Populate the LUT with max no. of queues in round robin fashion */ + for (i = 0, j = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++, j++) { + if (j == vsi->num_queues) + j = 0; +#ifdef RSS + /* + * Fetch the RSS bucket id for the given indirection entry. + * Cap it at the number of configured buckets (which is + * num_queues.) + */ + que_id = rss_get_indirection_to_bucket(i); + que_id = que_id % vsi->num_queues; +#else + que_id = j; +#endif + /* lut = 4-byte sliding window of 4 lut entries */ + lut = (lut << 8) | (que_id & 0xF); + /* On i = 3, we have 4 entries in lut; write to the register */ + if ((i & 3) == 3) { + wr32(hw, I40E_VFQF_HLUT(i), lut); + DDPRINTF(sc->dev, "HLUT(%2d): %#010x", i, lut); + } + } + ixl_flush(hw); +} + + +/* +** This routine refreshes vlan filters, called by init +** it scans the filter table and then updates the AQ +*/ +static void +ixlv_setup_vlan_filters(struct ixlv_sc *sc) +{ + struct ixl_vsi *vsi = &sc->vsi; + struct ixlv_vlan_filter *f; + int cnt = 0; + + if (vsi->num_vlans == 0) + return; + /* + ** Scan the filter table for vlan entries, + ** and if found call for the AQ update. + */ + SLIST_FOREACH(f, sc->vlan_filters, next) + if (f->flags & IXL_FILTER_ADD) + cnt++; + if (cnt > 0) + ixl_vc_enqueue(&sc->vc_mgr, &sc->add_vlan_cmd, + IXLV_FLAG_AQ_ADD_VLAN_FILTER, ixl_init_cmd_complete, sc); +} + + +/* +** This routine adds new MAC filters to the sc's list; +** these are later added in hardware by sending a virtual +** channel message. +*/ +static int +ixlv_add_mac_filter(struct ixlv_sc *sc, u8 *macaddr, u16 flags) +{ + struct ixlv_mac_filter *f; + + /* Does one already exist? */ + f = ixlv_find_mac_filter(sc, macaddr); + if (f != NULL) { + IDPRINTF(sc->vsi.ifp, "exists: " MAC_FORMAT, + MAC_FORMAT_ARGS(macaddr)); + return (EEXIST); + } + + /* If not, get a new empty filter */ + f = ixlv_get_mac_filter(sc); + if (f == NULL) { + if_printf(sc->vsi.ifp, "%s: no filters available!!\n", + __func__); + return (ENOMEM); + } + + IDPRINTF(sc->vsi.ifp, "marked: " MAC_FORMAT, + MAC_FORMAT_ARGS(macaddr)); + + bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN); + f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED); + f->flags |= flags; + return (0); +} + +/* +** Marks a MAC filter for deletion. +*/ +static int +ixlv_del_mac_filter(struct ixlv_sc *sc, u8 *macaddr) +{ + struct ixlv_mac_filter *f; + + f = ixlv_find_mac_filter(sc, macaddr); + if (f == NULL) + return (ENOENT); + + f->flags |= IXL_FILTER_DEL; + return (0); +} + +/* +** Tasklet handler for MSIX Adminq interrupts +** - done outside interrupt context since it might sleep +*/ +static void +ixlv_do_adminq(void *context, int pending) +{ + struct ixlv_sc *sc = context; + + mtx_lock(&sc->mtx); + ixlv_do_adminq_locked(sc); + mtx_unlock(&sc->mtx); + return; +} + +static void +ixlv_do_adminq_locked(struct ixlv_sc *sc) +{ + struct i40e_hw *hw = &sc->hw; + struct i40e_arq_event_info event; + struct i40e_virtchnl_msg *v_msg; + device_t dev = sc->dev; + u16 result = 0; + u32 reg, oldreg; + i40e_status ret; + + IXLV_CORE_LOCK_ASSERT(sc); + + event.buf_len = IXL_AQ_BUF_SZ; + event.msg_buf = sc->aq_buffer; + v_msg = (struct i40e_virtchnl_msg *)&event.desc; + + do { + ret = i40e_clean_arq_element(hw, &event, &result); + if (ret) + break; + ixlv_vc_completion(sc, v_msg->v_opcode, + v_msg->v_retval, event.msg_buf, event.msg_len); + if (result != 0) + bzero(event.msg_buf, IXL_AQ_BUF_SZ); + } while (result); + + /* check for Admin queue errors */ + oldreg = reg = rd32(hw, hw->aq.arq.len); + if (reg & I40E_VF_ARQLEN1_ARQVFE_MASK) { + device_printf(dev, "ARQ VF Error detected\n"); + reg &= ~I40E_VF_ARQLEN1_ARQVFE_MASK; + } + if (reg & I40E_VF_ARQLEN1_ARQOVFL_MASK) { + device_printf(dev, "ARQ Overflow Error detected\n"); + reg &= ~I40E_VF_ARQLEN1_ARQOVFL_MASK; + } + if (reg & I40E_VF_ARQLEN1_ARQCRIT_MASK) { + device_printf(dev, "ARQ Critical Error detected\n"); + reg &= ~I40E_VF_ARQLEN1_ARQCRIT_MASK; + } + if (oldreg != reg) + wr32(hw, hw->aq.arq.len, reg); + + oldreg = reg = rd32(hw, hw->aq.asq.len); + if (reg & I40E_VF_ATQLEN1_ATQVFE_MASK) { + device_printf(dev, "ASQ VF Error detected\n"); + reg &= ~I40E_VF_ATQLEN1_ATQVFE_MASK; + } + if (reg & I40E_VF_ATQLEN1_ATQOVFL_MASK) { + device_printf(dev, "ASQ Overflow Error detected\n"); + reg &= ~I40E_VF_ATQLEN1_ATQOVFL_MASK; + } + if (reg & I40E_VF_ATQLEN1_ATQCRIT_MASK) { + device_printf(dev, "ASQ Critical Error detected\n"); + reg &= ~I40E_VF_ATQLEN1_ATQCRIT_MASK; + } + if (oldreg != reg) + wr32(hw, hw->aq.asq.len, reg); + + ixlv_enable_adminq_irq(hw); +} + +static void +ixlv_add_sysctls(struct ixlv_sc *sc) +{ + device_t dev = sc->dev; + struct ixl_vsi *vsi = &sc->vsi; + struct i40e_eth_stats *es = &vsi->eth_stats; + + struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); + struct sysctl_oid *tree = device_get_sysctl_tree(dev); + struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree); + + struct sysctl_oid *vsi_node, *queue_node; + struct sysctl_oid_list *vsi_list, *queue_list; + +#define QUEUE_NAME_LEN 32 + char queue_namebuf[QUEUE_NAME_LEN]; + + struct ixl_queue *queues = vsi->queues; + struct tx_ring *txr; + struct rx_ring *rxr; + + /* Driver statistics sysctls */ + SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events", + CTLFLAG_RD, &sc->watchdog_events, + "Watchdog timeouts"); + SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "admin_irq", + CTLFLAG_RD, &sc->admin_irq, + "Admin Queue IRQ Handled"); + + /* VSI statistics sysctls */ + vsi_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "vsi", + CTLFLAG_RD, NULL, "VSI-specific statistics"); + vsi_list = SYSCTL_CHILDREN(vsi_node); + + struct ixl_sysctl_info ctls[] = + { + {&es->rx_bytes, "good_octets_rcvd", "Good Octets Received"}, + {&es->rx_unicast, "ucast_pkts_rcvd", + "Unicast Packets Received"}, + {&es->rx_multicast, "mcast_pkts_rcvd", + "Multicast Packets Received"}, + {&es->rx_broadcast, "bcast_pkts_rcvd", + "Broadcast Packets Received"}, + {&es->rx_discards, "rx_discards", "Discarded RX packets"}, + {&es->rx_unknown_protocol, "rx_unknown_proto", "RX unknown protocol packets"}, + {&es->tx_bytes, "good_octets_txd", "Good Octets Transmitted"}, + {&es->tx_unicast, "ucast_pkts_txd", "Unicast Packets Transmitted"}, + {&es->tx_multicast, "mcast_pkts_txd", + "Multicast Packets Transmitted"}, + {&es->tx_broadcast, "bcast_pkts_txd", + "Broadcast Packets Transmitted"}, + {&es->tx_errors, "tx_errors", "TX packet errors"}, + // end + {0,0,0} + }; + struct ixl_sysctl_info *entry = ctls; + while (entry->stat != NULL) + { + SYSCTL_ADD_QUAD(ctx, child, OID_AUTO, entry->name, + CTLFLAG_RD, entry->stat, + entry->description); + entry++; + } + + /* Queue sysctls */ + for (int q = 0; q < vsi->num_queues; q++) { + snprintf(queue_namebuf, QUEUE_NAME_LEN, "que%d", q); + queue_node = SYSCTL_ADD_NODE(ctx, vsi_list, OID_AUTO, queue_namebuf, + CTLFLAG_RD, NULL, "Queue Name"); + queue_list = SYSCTL_CHILDREN(queue_node); + + txr = &(queues[q].txr); + rxr = &(queues[q].rxr); + + SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "mbuf_defrag_failed", + CTLFLAG_RD, &(queues[q].mbuf_defrag_failed), + "m_defrag() failed"); + SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "dropped", + CTLFLAG_RD, &(queues[q].dropped_pkts), + "Driver dropped packets"); + SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "irqs", + CTLFLAG_RD, &(queues[q].irqs), + "irqs on this queue"); + SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "tso_tx", + CTLFLAG_RD, &(queues[q].tso), + "TSO"); + SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "tx_dma_setup", + CTLFLAG_RD, &(queues[q].tx_dma_setup), + "Driver tx dma failure in xmit"); + SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "no_desc_avail", + CTLFLAG_RD, &(txr->no_desc), + "Queue No Descriptor Available"); + SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "tx_packets", + CTLFLAG_RD, &(txr->total_packets), + "Queue Packets Transmitted"); + SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "tx_bytes", + CTLFLAG_RD, &(txr->tx_bytes), + "Queue Bytes Transmitted"); + SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "rx_packets", + CTLFLAG_RD, &(rxr->rx_packets), + "Queue Packets Received"); + SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "rx_bytes", + CTLFLAG_RD, &(rxr->rx_bytes), + "Queue Bytes Received"); + + /* Examine queue state */ + SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "qtx_head", + CTLTYPE_UINT | CTLFLAG_RD, &queues[q], + sizeof(struct ixl_queue), + ixlv_sysctl_qtx_tail_handler, "IU", + "Queue Transmit Descriptor Tail"); + SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "qrx_head", + CTLTYPE_UINT | CTLFLAG_RD, &queues[q], + sizeof(struct ixl_queue), + ixlv_sysctl_qrx_tail_handler, "IU", + "Queue Receive Descriptor Tail"); + } +} + +static void +ixlv_init_filters(struct ixlv_sc *sc) +{ + sc->mac_filters = malloc(sizeof(struct ixlv_mac_filter), + M_DEVBUF, M_NOWAIT | M_ZERO); + SLIST_INIT(sc->mac_filters); + sc->vlan_filters = malloc(sizeof(struct ixlv_vlan_filter), + M_DEVBUF, M_NOWAIT | M_ZERO); + SLIST_INIT(sc->vlan_filters); + return; +} + +static void +ixlv_free_filters(struct ixlv_sc *sc) +{ + struct ixlv_mac_filter *f; + struct ixlv_vlan_filter *v; + + while (!SLIST_EMPTY(sc->mac_filters)) { + f = SLIST_FIRST(sc->mac_filters); + SLIST_REMOVE_HEAD(sc->mac_filters, next); + free(f, M_DEVBUF); + } + while (!SLIST_EMPTY(sc->vlan_filters)) { + v = SLIST_FIRST(sc->vlan_filters); + SLIST_REMOVE_HEAD(sc->vlan_filters, next); + free(v, M_DEVBUF); + } + return; +} + +/** + * ixlv_sysctl_qtx_tail_handler + * Retrieves I40E_QTX_TAIL1 value from hardware + * for a sysctl. + */ +static int +ixlv_sysctl_qtx_tail_handler(SYSCTL_HANDLER_ARGS) +{ + struct ixl_queue *que; + int error; + u32 val; + + que = ((struct ixl_queue *)oidp->oid_arg1); + if (!que) return 0; + + val = rd32(que->vsi->hw, que->txr.tail); + error = sysctl_handle_int(oidp, &val, 0, req); + if (error || !req->newptr) + return error; + return (0); +} + +/** + * ixlv_sysctl_qrx_tail_handler + * Retrieves I40E_QRX_TAIL1 value from hardware + * for a sysctl. + */ +static int +ixlv_sysctl_qrx_tail_handler(SYSCTL_HANDLER_ARGS) +{ + struct ixl_queue *que; + int error; + u32 val; + + que = ((struct ixl_queue *)oidp->oid_arg1); + if (!que) return 0; + + val = rd32(que->vsi->hw, que->rxr.tail); + error = sysctl_handle_int(oidp, &val, 0, req); + if (error || !req->newptr) + return error; + return (0); +} + Index: sys/dev/ixl/legacy_ixl.h =================================================================== --- /dev/null +++ sys/dev/ixl/legacy_ixl.h @@ -0,0 +1,655 @@ +/****************************************************************************** + + Copyright (c) 2013-2015, Intel Corporation + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of the Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +******************************************************************************/ +/*$FreeBSD$*/ + + +#ifndef _LEGACY_IXL_H_ +#define _LEGACY_IXL_H_ + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef PCI_IOV +#include +#include +#include +#endif + +#include "i40e_type.h" +#include "i40e_prototype.h" + +#if defined(IXL_DEBUG) || defined(IXL_DEBUG_SYSCTL) +#include + +#define MAC_FORMAT "%02x:%02x:%02x:%02x:%02x:%02x" +#define MAC_FORMAT_ARGS(mac_addr) \ + (mac_addr)[0], (mac_addr)[1], (mac_addr)[2], (mac_addr)[3], \ + (mac_addr)[4], (mac_addr)[5] +#define ON_OFF_STR(is_set) ((is_set) ? "On" : "Off") +#endif /* IXL_DEBUG || IXL_DEBUG_SYSCTL */ + +#ifdef IXL_DEBUG +/* Enable debug sysctls */ +#ifndef IXL_DEBUG_SYSCTL +#define IXL_DEBUG_SYSCTL 1 +#endif + +#define _DBG_PRINTF(S, ...) printf("%s: " S "\n", __func__, ##__VA_ARGS__) +#define _DEV_DBG_PRINTF(dev, S, ...) device_printf(dev, "%s: " S "\n", __func__, ##__VA_ARGS__) +#define _IF_DBG_PRINTF(ifp, S, ...) if_printf(ifp, "%s: " S "\n", __func__, ##__VA_ARGS__) + +/* Defines for printing generic debug information */ +#define DPRINTF(...) _DBG_PRINTF(__VA_ARGS__) +#define DDPRINTF(...) _DEV_DBG_PRINTF(__VA_ARGS__) +#define IDPRINTF(...) _IF_DBG_PRINTF(__VA_ARGS__) + +/* Defines for printing specific debug information */ +#define DEBUG_INIT 1 +#define DEBUG_IOCTL 1 +#define DEBUG_HW 1 + +#define INIT_DEBUGOUT(...) if (DEBUG_INIT) _DBG_PRINTF(__VA_ARGS__) +#define INIT_DBG_DEV(...) if (DEBUG_INIT) _DEV_DBG_PRINTF(__VA_ARGS__) +#define INIT_DBG_IF(...) if (DEBUG_INIT) _IF_DBG_PRINTF(__VA_ARGS__) + +#define IOCTL_DEBUGOUT(...) if (DEBUG_IOCTL) _DBG_PRINTF(__VA_ARGS__) +#define IOCTL_DBG_IF2(ifp, S, ...) if (DEBUG_IOCTL) \ + if_printf(ifp, S "\n", ##__VA_ARGS__) +#define IOCTL_DBG_IF(...) if (DEBUG_IOCTL) _IF_DBG_PRINTF(__VA_ARGS__) + +#define HW_DEBUGOUT(...) if (DEBUG_HW) _DBG_PRINTF(__VA_ARGS__) + +#else /* no IXL_DEBUG */ +#define DEBUG_INIT 0 +#define DEBUG_IOCTL 0 +#define DEBUG_HW 0 + +#define DPRINTF(...) +#define DDPRINTF(...) +#define IDPRINTF(...) + +#define INIT_DEBUGOUT(...) +#define INIT_DBG_DEV(...) +#define INIT_DBG_IF(...) +#define IOCTL_DEBUGOUT(...) +#define IOCTL_DBG_IF2(...) +#define IOCTL_DBG_IF(...) +#define HW_DEBUGOUT(...) +#endif /* IXL_DEBUG */ + +/* Tunables */ + +/* + * Ring Descriptors Valid Range: 32-4096 Default Value: 1024 This value is the + * number of tx/rx descriptors allocated by the driver. Increasing this + * value allows the driver to queue more operations. Each descriptor is 16 + * or 32 bytes (configurable in FVL) + */ +#define DEFAULT_RING 1024 +#define PERFORM_RING 2048 +#define MAX_RING 4096 +#define MIN_RING 32 + +/* +** Default number of entries in Tx queue buf_ring. +*/ +#define SMALL_TXBRSZ 4096 +/* This may require mbuf cluster tuning */ +#define DEFAULT_TXBRSZ (SMALL_TXBRSZ * SMALL_TXBRSZ) + +/* Alignment for rings */ +#define DBA_ALIGN 128 + +/* + * This parameter controls the maximum no of times the driver will loop in + * the isr. Minimum Value = 1 + */ +#define MAX_LOOP 10 + +/* + * This is the max watchdog interval, ie. the time that can + * pass between any two TX clean operations, such only happening + * when the TX hardware is functioning. + */ +#define IXL_WATCHDOG (10 * hz) + +/* + * This parameters control when the driver calls the routine to reclaim + * transmit descriptors. + */ +#define IXL_TX_CLEANUP_THRESHOLD (que->num_desc / 8) +#define IXL_TX_OP_THRESHOLD (que->num_desc / 32) + +/* Flow control constants */ +#define IXL_FC_PAUSE 0xFFFF +#define IXL_FC_HI 0x20000 +#define IXL_FC_LO 0x10000 + +#define MAX_MULTICAST_ADDR 128 + +#define IXL_BAR 3 +#define IXL_ADM_LIMIT 2 +#define IXL_TSO_SIZE 65535 +#define IXL_TX_BUF_SZ ((u32) 1514) +#define IXL_AQ_BUF_SZ ((u32) 4096) +#define IXL_RX_HDR 128 +/* Controls the length of the Admin Queue */ +#define IXL_AQ_LEN 256 +#define IXL_AQ_LEN_MAX 1024 +#define IXL_AQ_BUFSZ 4096 +#define IXL_RX_LIMIT 512 +#define IXL_RX_ITR 0 +#define IXL_TX_ITR 1 +#define IXL_ITR_NONE 3 +#define IXL_QUEUE_EOL 0x7FF +#define IXL_MAX_FRAME 0x2600 +#define IXL_MAX_TX_SEGS 8 +#define IXL_MAX_TSO_SEGS 66 +#define IXL_SPARSE_CHAIN 6 +#define IXL_QUEUE_HUNG 0x80000000 +#define IXL_KEYSZ 10 + +#define IXL_VF_MAX_BUFFER 0x3F80 +#define IXL_VF_MAX_HDR_BUFFER 0x840 +#define IXL_VF_MAX_FRAME 0x3FFF + +/* ERJ: hardware can support ~1.5k filters between all functions */ +#define IXL_MAX_FILTERS 256 +#define IXL_MAX_TX_BUSY 10 + +#define IXL_NVM_VERSION_LO_SHIFT 0 +#define IXL_NVM_VERSION_LO_MASK (0xff << IXL_NVM_VERSION_LO_SHIFT) +#define IXL_NVM_VERSION_HI_SHIFT 12 +#define IXL_NVM_VERSION_HI_MASK (0xf << IXL_NVM_VERSION_HI_SHIFT) + + +/* + * Interrupt Moderation parameters + */ +#define IXL_MAX_ITR 0x07FF +#define IXL_ITR_100K 0x0005 +#define IXL_ITR_20K 0x0019 +#define IXL_ITR_8K 0x003E +#define IXL_ITR_4K 0x007A +#define IXL_ITR_DYNAMIC 0x8000 +#define IXL_LOW_LATENCY 0 +#define IXL_AVE_LATENCY 1 +#define IXL_BULK_LATENCY 2 + +/* MacVlan Flags */ +#define IXL_FILTER_USED (u16)(1 << 0) +#define IXL_FILTER_VLAN (u16)(1 << 1) +#define IXL_FILTER_ADD (u16)(1 << 2) +#define IXL_FILTER_DEL (u16)(1 << 3) +#define IXL_FILTER_MC (u16)(1 << 4) + +/* used in the vlan field of the filter when not a vlan */ +#define IXL_VLAN_ANY -1 + +#define CSUM_OFFLOAD_IPV4 (CSUM_IP|CSUM_TCP|CSUM_UDP|CSUM_SCTP) +#define CSUM_OFFLOAD_IPV6 (CSUM_TCP_IPV6|CSUM_UDP_IPV6|CSUM_SCTP_IPV6) +#define CSUM_OFFLOAD (CSUM_OFFLOAD_IPV4|CSUM_OFFLOAD_IPV6|CSUM_TSO) + +/* Misc flags for ixl_vsi.flags */ +#define IXL_FLAGS_KEEP_TSO4 (1 << 0) +#define IXL_FLAGS_KEEP_TSO6 (1 << 1) + +#define IXL_VF_RESET_TIMEOUT 100 + +#define IXL_VSI_DATA_PORT 0x01 + +#define IXLV_MAX_QUEUES 16 +#define IXL_MAX_VSI_QUEUES (2 * (I40E_VSILAN_QTABLE_MAX_INDEX + 1)) + +#define IXL_RX_CTX_BASE_UNITS 128 +#define IXL_TX_CTX_BASE_UNITS 128 + +#define IXL_VPINT_LNKLSTN_REG(hw, vector, vf_num) \ + I40E_VPINT_LNKLSTN(((vector) - 1) + \ + (((hw)->func_caps.num_msix_vectors_vf - 1) * (vf_num))) + +#define IXL_VFINT_DYN_CTLN_REG(hw, vector, vf_num) \ + I40E_VFINT_DYN_CTLN(((vector) - 1) + \ + (((hw)->func_caps.num_msix_vectors_vf - 1) * (vf_num))) + +#define IXL_PF_PCI_CIAA_VF_DEVICE_STATUS 0xAA + +#define IXL_PF_PCI_CIAD_VF_TRANS_PENDING_MASK 0x20 + +#define IXL_GLGEN_VFLRSTAT_INDEX(glb_vf) ((glb_vf) / 32) +#define IXL_GLGEN_VFLRSTAT_MASK(glb_vf) (1 << ((glb_vf) % 32)) + +#define IXL_MAX_ITR_IDX 3 + +#define IXL_END_OF_INTR_LNKLST 0x7FF + +#define IXL_TX_LOCK(_sc) mtx_lock(&(_sc)->mtx) +#define IXL_TX_UNLOCK(_sc) mtx_unlock(&(_sc)->mtx) +#define IXL_TX_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->mtx) +#define IXL_TX_TRYLOCK(_sc) mtx_trylock(&(_sc)->mtx) +#define IXL_TX_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->mtx, MA_OWNED) + +#define IXL_RX_LOCK(_sc) mtx_lock(&(_sc)->mtx) +#define IXL_RX_UNLOCK(_sc) mtx_unlock(&(_sc)->mtx) +#define IXL_RX_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->mtx) + +#if __FreeBSD_version >= 1100036 +#define IXL_SET_IPACKETS(vsi, count) (vsi)->ipackets = (count) +#define IXL_SET_IERRORS(vsi, count) (vsi)->ierrors = (count) +#define IXL_SET_OPACKETS(vsi, count) (vsi)->opackets = (count) +#define IXL_SET_OERRORS(vsi, count) (vsi)->oerrors = (count) +#define IXL_SET_COLLISIONS(vsi, count) /* Do nothing; collisions is always 0. */ +#define IXL_SET_IBYTES(vsi, count) (vsi)->ibytes = (count) +#define IXL_SET_OBYTES(vsi, count) (vsi)->obytes = (count) +#define IXL_SET_IMCASTS(vsi, count) (vsi)->imcasts = (count) +#define IXL_SET_OMCASTS(vsi, count) (vsi)->omcasts = (count) +#define IXL_SET_IQDROPS(vsi, count) (vsi)->iqdrops = (count) +#define IXL_SET_OQDROPS(vsi, count) (vsi)->oqdrops = (count) +#define IXL_SET_NOPROTO(vsi, count) (vsi)->noproto = (count) +#else +#define IXL_SET_IPACKETS(vsi, count) (vsi)->ifp->if_ipackets = (count) +#define IXL_SET_IERRORS(vsi, count) (vsi)->ifp->if_ierrors = (count) +#define IXL_SET_OPACKETS(vsi, count) (vsi)->ifp->if_opackets = (count) +#define IXL_SET_OERRORS(vsi, count) (vsi)->ifp->if_oerrors = (count) +#define IXL_SET_COLLISIONS(vsi, count) (vsi)->ifp->if_collisions = (count) +#define IXL_SET_IBYTES(vsi, count) (vsi)->ifp->if_ibytes = (count) +#define IXL_SET_OBYTES(vsi, count) (vsi)->ifp->if_obytes = (count) +#define IXL_SET_IMCASTS(vsi, count) (vsi)->ifp->if_imcasts = (count) +#define IXL_SET_OMCASTS(vsi, count) (vsi)->ifp->if_omcasts = (count) +#define IXL_SET_IQDROPS(vsi, count) (vsi)->ifp->if_iqdrops = (count) +#define IXL_SET_OQDROPS(vsi, odrops) (vsi)->ifp->if_snd.ifq_drops = (odrops) +#define IXL_SET_NOPROTO(vsi, count) (vsi)->noproto = (count) +#endif + +/* + ***************************************************************************** + * vendor_info_array + * + * This array contains the list of Subvendor/Subdevice IDs on which the driver + * should load. + * + ***************************************************************************** + */ +typedef struct _ixl_vendor_info_t { + unsigned int vendor_id; + unsigned int device_id; + unsigned int subvendor_id; + unsigned int subdevice_id; + unsigned int index; +} ixl_vendor_info_t; + + +struct ixl_tx_buf { + u32 eop_index; + struct mbuf *m_head; + bus_dmamap_t map; + bus_dma_tag_t tag; +}; + +struct ixl_rx_buf { + struct mbuf *m_head; + struct mbuf *m_pack; + struct mbuf *fmp; + bus_dmamap_t hmap; + bus_dmamap_t pmap; +}; + +/* +** This struct has multiple uses, multicast +** addresses, vlans, and mac filters all use it. +*/ +struct ixl_mac_filter { + SLIST_ENTRY(ixl_mac_filter) next; + u8 macaddr[ETHER_ADDR_LEN]; + s16 vlan; + u16 flags; +}; + + +/* + * The Transmit ring control struct + */ +struct tx_ring { + struct ixl_queue *que; + struct mtx mtx; + u32 tail; + struct i40e_tx_desc *base; + struct i40e_dma_mem dma; + u16 next_avail; + u16 next_to_clean; + u16 atr_rate; + u16 atr_count; + u16 itr; + u16 latency; + struct ixl_tx_buf *buffers; + volatile u16 avail; + u32 cmd; + bus_dma_tag_t tx_tag; + bus_dma_tag_t tso_tag; + char mtx_name[16]; + struct buf_ring *br; + + /* Used for Dynamic ITR calculation */ + u32 packets; + u32 bytes; + + /* Soft Stats */ + u64 tx_bytes; + u64 no_desc; + u64 total_packets; +}; + + +/* + * The Receive ring control struct + */ +struct rx_ring { + struct ixl_queue *que; + struct mtx mtx; + union i40e_rx_desc *base; + struct i40e_dma_mem dma; + struct lro_ctrl lro; + bool lro_enabled; + bool hdr_split; + bool discard; + u16 next_refresh; + u16 next_check; + u16 itr; + u16 latency; + char mtx_name[16]; + struct ixl_rx_buf *buffers; + u32 mbuf_sz; + u32 tail; + bus_dma_tag_t htag; + bus_dma_tag_t ptag; + + /* Used for Dynamic ITR calculation */ + u32 packets; + u32 bytes; + + /* Soft stats */ + u64 split; + u64 rx_packets; + u64 rx_bytes; + u64 discarded; + u64 not_done; +}; + +/* +** Driver queue struct: this is the interrupt container +** for the associated tx and rx ring pair. +*/ +struct ixl_queue { + struct ixl_vsi *vsi; + u32 me; + u32 msix; /* This queue's MSIX vector */ + u32 eims; /* This queue's EIMS bit */ + struct resource *res; + void *tag; + int num_desc; /* both tx and rx */ + int busy; + struct tx_ring txr; + struct rx_ring rxr; + struct task task; + struct task tx_task; + struct taskqueue *tq; + + /* Queue stats */ + u64 irqs; + u64 tso; + u64 mbuf_defrag_failed; + u64 mbuf_hdr_failed; + u64 mbuf_pkt_failed; + u64 tx_map_avail; + u64 tx_dma_setup; + u64 dropped_pkts; +}; + +/* +** Virtual Station interface: +** there would be one of these per traffic class/type +** for now just one, and its embedded in the pf +*/ +SLIST_HEAD(ixl_ftl_head, ixl_mac_filter); +struct ixl_vsi { + void *back; + struct ifnet *ifp; + struct device *dev; + struct i40e_hw *hw; + struct ifmedia media; + u64 que_mask; + int id; + u16 vsi_num; + u16 msix_base; /* station base MSIX vector */ + u16 first_queue; + u16 num_queues; + u16 rx_itr_setting; + u16 tx_itr_setting; + struct ixl_queue *queues; /* head of queues */ + bool link_active; + u16 seid; + u16 uplink_seid; + u16 downlink_seid; + u16 max_frame_size; + + /* MAC/VLAN Filter list */ + struct ixl_ftl_head ftl; + u16 num_macs; + + struct i40e_aqc_vsi_properties_data info; + + eventhandler_tag vlan_attach; + eventhandler_tag vlan_detach; + u16 num_vlans; + + /* Per-VSI stats from hardware */ + struct i40e_eth_stats eth_stats; + struct i40e_eth_stats eth_stats_offsets; + bool stat_offsets_loaded; + /* VSI stat counters */ + u64 ipackets; + u64 ierrors; + u64 opackets; + u64 oerrors; + u64 ibytes; + u64 obytes; + u64 imcasts; + u64 omcasts; + u64 iqdrops; + u64 oqdrops; + u64 noproto; + + /* Driver statistics */ + u64 hw_filters_del; + u64 hw_filters_add; + + /* Misc. */ + u64 active_queues; + u64 flags; + struct sysctl_oid *vsi_node; +}; + +/* +** Find the number of unrefreshed RX descriptors +*/ +static inline u16 +ixl_rx_unrefreshed(struct ixl_queue *que) +{ + struct rx_ring *rxr = &que->rxr; + + if (rxr->next_check > rxr->next_refresh) + return (rxr->next_check - rxr->next_refresh - 1); + else + return ((que->num_desc + rxr->next_check) - + rxr->next_refresh - 1); +} + +/* +** Find the next available unused filter +*/ +static inline struct ixl_mac_filter * +ixl_get_filter(struct ixl_vsi *vsi) +{ + struct ixl_mac_filter *f; + + /* create a new empty filter */ + f = malloc(sizeof(struct ixl_mac_filter), + M_DEVBUF, M_NOWAIT | M_ZERO); + if (f) + SLIST_INSERT_HEAD(&vsi->ftl, f, next); + + return (f); +} + +/* +** Compare two ethernet addresses +*/ +static inline bool +cmp_etheraddr(const u8 *ea1, const u8 *ea2) +{ + bool cmp = FALSE; + + if ((ea1[0] == ea2[0]) && (ea1[1] == ea2[1]) && + (ea1[2] == ea2[2]) && (ea1[3] == ea2[3]) && + (ea1[4] == ea2[4]) && (ea1[5] == ea2[5])) + cmp = TRUE; + + return (cmp); +} + +/* + * Info for stats sysctls + */ +struct ixl_sysctl_info { + u64 *stat; + char *name; + char *description; +}; + +extern int ixl_atr_rate; + +/* +** ixl_fw_version_str - format the FW and NVM version strings +*/ +static inline char * +ixl_fw_version_str(struct i40e_hw *hw) +{ + static char buf[32]; + + snprintf(buf, sizeof(buf), + "f%d.%d a%d.%d n%02x.%02x e%08x", + hw->aq.fw_maj_ver, hw->aq.fw_min_ver, + hw->aq.api_maj_ver, hw->aq.api_min_ver, + (hw->nvm.version & IXL_NVM_VERSION_HI_MASK) >> + IXL_NVM_VERSION_HI_SHIFT, + (hw->nvm.version & IXL_NVM_VERSION_LO_MASK) >> + IXL_NVM_VERSION_LO_SHIFT, + hw->nvm.eetrack); + return buf; +} + +/********************************************************************* + * TXRX Function prototypes + *********************************************************************/ +int ixl_allocate_tx_data(struct ixl_queue *); +int ixl_allocate_rx_data(struct ixl_queue *); +void ixl_init_tx_ring(struct ixl_queue *); +int ixl_init_rx_ring(struct ixl_queue *); +bool ixl_rxeof(struct ixl_queue *, int); +bool ixl_txeof(struct ixl_queue *); +int ixl_mq_start(struct ifnet *, struct mbuf *); +int ixl_mq_start_locked(struct ifnet *, struct tx_ring *); +void ixl_deferred_mq_start(void *, int); +void ixl_qflush(struct ifnet *); +void ixl_free_vsi(struct ixl_vsi *); +void ixl_free_que_tx(struct ixl_queue *); +void ixl_free_que_rx(struct ixl_queue *); +#ifdef IXL_FDIR +void ixl_atr(struct ixl_queue *, struct tcphdr *, int); +#endif +#if __FreeBSD_version >= 1100000 +uint64_t ixl_get_counter(if_t ifp, ift_counter cnt); +#endif + +#endif /* _IXL_H_ */ Index: sys/dev/ixl/legacy_ixl_pf.h =================================================================== --- /dev/null +++ sys/dev/ixl/legacy_ixl_pf.h @@ -0,0 +1,139 @@ +/****************************************************************************** + + Copyright (c) 2013-2015, Intel Corporation + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of the Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +******************************************************************************/ +/*$FreeBSD$*/ + + +#ifndef _LEGACY_IXL_PF_H_ +#define _LEGACY_IXL_PF_H_ + +#define VF_FLAG_ENABLED 0x01 +#define VF_FLAG_SET_MAC_CAP 0x02 +#define VF_FLAG_VLAN_CAP 0x04 +#define VF_FLAG_PROMISC_CAP 0x08 +#define VF_FLAG_MAC_ANTI_SPOOF 0x10 + +struct ixl_vf { + struct ixl_vsi vsi; + uint32_t vf_flags; + + uint8_t mac[ETHER_ADDR_LEN]; + uint16_t vf_num; + + struct sysctl_ctx_list ctx; +}; + +/* Physical controller structure */ +struct ixl_pf { + struct i40e_hw hw; + struct i40e_osdep osdep; + struct device *dev; + + struct resource *pci_mem; + struct resource *msix_mem; + + /* + * Interrupt resources: this set is + * either used for legacy, or for Link + * when doing MSIX + */ + void *tag; + struct resource *res; + + struct callout timer; + int msix; + int if_flags; + + struct mtx pf_mtx; + + u32 qbase; + u32 admvec; + struct task adminq; + struct taskqueue *tq; + + bool link_up; + u32 link_speed; + int advertised_speed; + int fc; /* local flow ctrl setting */ + + /* + ** Network interfaces + ** These are the traffic class holders, and + ** will have a stack interface and queues + ** associated with them. + ** NOTE: The PF has only a single interface, + ** so it is embedded in the PF struct. + */ + struct ixl_vsi vsi; + + /* Misc stats maintained by the driver */ + u64 watchdog_events; + u64 admin_irq; + + /* Statistics from hw */ + struct i40e_hw_port_stats stats; + struct i40e_hw_port_stats stats_offsets; + bool stat_offsets_loaded; + + struct ixl_vf *vfs; + int num_vfs; + uint16_t veb_seid; + struct task vflr_task; + int vc_debug_lvl; +}; + +#define IXL_SET_ADVERTISE_HELP \ +"Control link advertise speed:\n" \ +"\tFlags:\n" \ +"\t\t0x1 - advertise 100 Mb\n" \ +"\t\t0x2 - advertise 1G\n" \ +"\t\t0x4 - advertise 10G\n" \ +"\t\t0x8 - advertise 20G\n\n" \ +"\tDoes not work on 40G devices." + +#define I40E_VC_DEBUG(pf, level, ...) \ + do { \ + if ((pf)->vc_debug_lvl >= (level)) \ + device_printf((pf)->dev, __VA_ARGS__); \ + } while (0) + +#define i40e_send_vf_nack(pf, vf, op, st) \ + ixl_send_vf_nack_msg((pf), (vf), (op), (st), __FILE__, __LINE__) + +#define IXL_PF_LOCK_INIT(_sc, _name) \ + mtx_init(&(_sc)->pf_mtx, _name, "IXL PF Lock", MTX_DEF) +#define IXL_PF_LOCK(_sc) mtx_lock(&(_sc)->pf_mtx) +#define IXL_PF_UNLOCK(_sc) mtx_unlock(&(_sc)->pf_mtx) +#define IXL_PF_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->pf_mtx) +#define IXL_PF_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->pf_mtx, MA_OWNED) + +#endif /* _IXL_PF_H_ */ Index: sys/dev/ixl/legacy_ixl_txrx.c =================================================================== --- /dev/null +++ sys/dev/ixl/legacy_ixl_txrx.c @@ -0,0 +1,1827 @@ +/****************************************************************************** + + Copyright (c) 2013-2015, Intel Corporation + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of the Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +******************************************************************************/ +/*$FreeBSD$*/ + +/* +** IXL driver TX/RX Routines: +** This was seperated to allow usage by +** both the BASE and the VF drivers. +*/ + +#ifndef IXL_STANDALONE_BUILD +#include "opt_inet.h" +#include "opt_inet6.h" +#include "opt_rss.h" +#endif + +#include "ixl.h" + +#ifdef RSS +#include +#endif + +/* Local Prototypes */ +static void ixl_rx_checksum(struct mbuf *, u32, u32, u8); +static void ixl_refresh_mbufs(struct ixl_queue *, int); +static int ixl_xmit(struct ixl_queue *, struct mbuf **); +static int ixl_tx_setup_offload(struct ixl_queue *, + struct mbuf *, u32 *, u32 *); +static bool ixl_tso_setup(struct ixl_queue *, struct mbuf *); + +static __inline void ixl_rx_discard(struct rx_ring *, int); +static __inline void ixl_rx_input(struct rx_ring *, struct ifnet *, + struct mbuf *, u8); + +#ifdef DEV_NETMAP +#include +#endif /* DEV_NETMAP */ + +/* +** Multiqueue Transmit driver +*/ +int +ixl_mq_start(struct ifnet *ifp, struct mbuf *m) +{ + struct ixl_vsi *vsi = ifp->if_softc; + struct ixl_queue *que; + struct tx_ring *txr; + int err, i; +#ifdef RSS + u32 bucket_id; +#endif + + /* + ** Which queue to use: + ** + ** When doing RSS, map it to the same outbound + ** queue as the incoming flow would be mapped to. + ** If everything is setup correctly, it should be + ** the same bucket that the current CPU we're on is. + */ + if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) { +#ifdef RSS + if (rss_hash2bucket(m->m_pkthdr.flowid, + M_HASHTYPE_GET(m), &bucket_id) == 0) { + i = bucket_id % vsi->num_queues; + } else +#endif + i = m->m_pkthdr.flowid % vsi->num_queues; + } else + i = curcpu % vsi->num_queues; + /* + ** This may not be perfect, but until something + ** better comes along it will keep from scheduling + ** on stalled queues. + */ + if (((1 << i) & vsi->active_queues) == 0) + i = ffsl(vsi->active_queues); + + que = &vsi->queues[i]; + txr = &que->txr; + + err = drbr_enqueue(ifp, txr->br, m); + if (err) + return (err); + if (IXL_TX_TRYLOCK(txr)) { + ixl_mq_start_locked(ifp, txr); + IXL_TX_UNLOCK(txr); + } else + taskqueue_enqueue(que->tq, &que->tx_task); + + return (0); +} + +int +ixl_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr) +{ + struct ixl_queue *que = txr->que; + struct ixl_vsi *vsi = que->vsi; + struct mbuf *next; + int err = 0; + + + if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) || + vsi->link_active == 0) + return (ENETDOWN); + + /* Process the transmit queue */ + while ((next = drbr_peek(ifp, txr->br)) != NULL) { + if ((err = ixl_xmit(que, &next)) != 0) { + if (next == NULL) + drbr_advance(ifp, txr->br); + else + drbr_putback(ifp, txr->br, next); + break; + } + drbr_advance(ifp, txr->br); + /* Send a copy of the frame to the BPF listener */ + ETHER_BPF_MTAP(ifp, next); + if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) + break; + } + + if (txr->avail < IXL_TX_CLEANUP_THRESHOLD) + ixl_txeof(que); + + return (err); +} + +/* + * Called from a taskqueue to drain queued transmit packets. + */ +void +ixl_deferred_mq_start(void *arg, int pending) +{ + struct ixl_queue *que = arg; + struct tx_ring *txr = &que->txr; + struct ixl_vsi *vsi = que->vsi; + struct ifnet *ifp = vsi->ifp; + + IXL_TX_LOCK(txr); + if (!drbr_empty(ifp, txr->br)) + ixl_mq_start_locked(ifp, txr); + IXL_TX_UNLOCK(txr); +} + +/* +** Flush all queue ring buffers +*/ +void +ixl_qflush(struct ifnet *ifp) +{ + struct ixl_vsi *vsi = ifp->if_softc; + + for (int i = 0; i < vsi->num_queues; i++) { + struct ixl_queue *que = &vsi->queues[i]; + struct tx_ring *txr = &que->txr; + struct mbuf *m; + IXL_TX_LOCK(txr); + while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) + m_freem(m); + IXL_TX_UNLOCK(txr); + } + if_qflush(ifp); +} + +/* +** Find mbuf chains passed to the driver +** that are 'sparse', using more than 8 +** mbufs to deliver an mss-size chunk of data +*/ +static inline bool +ixl_tso_detect_sparse(struct mbuf *mp) +{ + struct mbuf *m; + int num = 0, mss; + bool ret = FALSE; + + mss = mp->m_pkthdr.tso_segsz; + for (m = mp->m_next; m != NULL; m = m->m_next) { + num++; + mss -= m->m_len; + if (mss < 1) + break; + if (m->m_next == NULL) + break; + } + if (num > IXL_SPARSE_CHAIN) + ret = TRUE; + + return (ret); +} + + +/********************************************************************* + * + * This routine maps the mbufs to tx descriptors, allowing the + * TX engine to transmit the packets. + * - return 0 on success, positive on failure + * + **********************************************************************/ +#define IXL_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS) + +static int +ixl_xmit(struct ixl_queue *que, struct mbuf **m_headp) +{ + struct ixl_vsi *vsi = que->vsi; + struct i40e_hw *hw = vsi->hw; + struct tx_ring *txr = &que->txr; + struct ixl_tx_buf *buf; + struct i40e_tx_desc *txd = NULL; + struct mbuf *m_head, *m; + int i, j, error, nsegs, maxsegs; + int first, last = 0; + u16 vtag = 0; + u32 cmd, off; + bus_dmamap_t map; + bus_dma_tag_t tag; + bus_dma_segment_t segs[IXL_MAX_TSO_SEGS]; + + + cmd = off = 0; + m_head = *m_headp; + + /* + * Important to capture the first descriptor + * used because it will contain the index of + * the one we tell the hardware to report back + */ + first = txr->next_avail; + buf = &txr->buffers[first]; + map = buf->map; + tag = txr->tx_tag; + maxsegs = IXL_MAX_TX_SEGS; + + if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { + /* Use larger mapping for TSO */ + tag = txr->tso_tag; + maxsegs = IXL_MAX_TSO_SEGS; + if (ixl_tso_detect_sparse(m_head)) { + m = m_defrag(m_head, M_NOWAIT); + if (m == NULL) { + m_freem(*m_headp); + *m_headp = NULL; + return (ENOBUFS); + } + *m_headp = m; + } + } + + /* + * Map the packet for DMA. + */ + error = bus_dmamap_load_mbuf_sg(tag, map, + *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); + + if (error == EFBIG) { + struct mbuf *m; + + m = m_defrag(*m_headp, M_NOWAIT); + if (m == NULL) { + que->mbuf_defrag_failed++; + m_freem(*m_headp); + *m_headp = NULL; + return (ENOBUFS); + } + *m_headp = m; + + /* Try it again */ + error = bus_dmamap_load_mbuf_sg(tag, map, + *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); + + if (error == ENOMEM) { + que->tx_dma_setup++; + return (error); + } else if (error != 0) { + que->tx_dma_setup++; + m_freem(*m_headp); + *m_headp = NULL; + return (error); + } + } else if (error == ENOMEM) { + que->tx_dma_setup++; + return (error); + } else if (error != 0) { + que->tx_dma_setup++; + m_freem(*m_headp); + *m_headp = NULL; + return (error); + } + + /* Make certain there are enough descriptors */ + if (nsegs > txr->avail - 2) { + txr->no_desc++; + error = ENOBUFS; + goto xmit_fail; + } + m_head = *m_headp; + + /* Set up the TSO/CSUM offload */ + if (m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD) { + error = ixl_tx_setup_offload(que, m_head, &cmd, &off); + if (error) + goto xmit_fail; + } + + cmd |= I40E_TX_DESC_CMD_ICRC; + /* Grab the VLAN tag */ + if (m_head->m_flags & M_VLANTAG) { + cmd |= I40E_TX_DESC_CMD_IL2TAG1; + vtag = htole16(m_head->m_pkthdr.ether_vtag); + } + + i = txr->next_avail; + for (j = 0; j < nsegs; j++) { + bus_size_t seglen; + + buf = &txr->buffers[i]; + buf->tag = tag; /* Keep track of the type tag */ + txd = &txr->base[i]; + seglen = segs[j].ds_len; + + txd->buffer_addr = htole64(segs[j].ds_addr); + txd->cmd_type_offset_bsz = + htole64(I40E_TX_DESC_DTYPE_DATA + | ((u64)cmd << I40E_TXD_QW1_CMD_SHIFT) + | ((u64)off << I40E_TXD_QW1_OFFSET_SHIFT) + | ((u64)seglen << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) + | ((u64)vtag << I40E_TXD_QW1_L2TAG1_SHIFT)); + + last = i; /* descriptor that will get completion IRQ */ + + if (++i == que->num_desc) + i = 0; + + buf->m_head = NULL; + buf->eop_index = -1; + } + /* Set the last descriptor for report */ + txd->cmd_type_offset_bsz |= + htole64(((u64)IXL_TXD_CMD << I40E_TXD_QW1_CMD_SHIFT)); + txr->avail -= nsegs; + txr->next_avail = i; + + buf->m_head = m_head; + /* Swap the dma map between the first and last descriptor */ + txr->buffers[first].map = buf->map; + buf->map = map; + bus_dmamap_sync(tag, map, BUS_DMASYNC_PREWRITE); + + /* Set the index of the descriptor that will be marked done */ + buf = &txr->buffers[first]; + buf->eop_index = last; + + bus_dmamap_sync(txr->dma.tag, txr->dma.map, + BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); + /* + * Advance the Transmit Descriptor Tail (Tdt), this tells the + * hardware that this frame is available to transmit. + */ + ++txr->total_packets; + wr32(hw, txr->tail, i); + + /* Mark outstanding work */ + if (que->busy == 0) + que->busy = 1; + return (0); + +xmit_fail: + bus_dmamap_unload(tag, buf->map); + return (error); +} + + +/********************************************************************* + * + * Allocate memory for tx_buffer structures. The tx_buffer stores all + * the information needed to transmit a packet on the wire. This is + * called only once at attach, setup is done every reset. + * + **********************************************************************/ +int +ixl_allocate_tx_data(struct ixl_queue *que) +{ + struct tx_ring *txr = &que->txr; + struct ixl_vsi *vsi = que->vsi; + device_t dev = vsi->dev; + struct ixl_tx_buf *buf; + int error = 0; + + /* + * Setup DMA descriptor areas. + */ + if ((error = bus_dma_tag_create(NULL, /* parent */ + 1, 0, /* alignment, bounds */ + BUS_SPACE_MAXADDR, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter, filterarg */ + IXL_TSO_SIZE, /* maxsize */ + IXL_MAX_TX_SEGS, /* nsegments */ + PAGE_SIZE, /* maxsegsize */ + 0, /* flags */ + NULL, /* lockfunc */ + NULL, /* lockfuncarg */ + &txr->tx_tag))) { + device_printf(dev,"Unable to allocate TX DMA tag\n"); + goto fail; + } + + /* Make a special tag for TSO */ + if ((error = bus_dma_tag_create(NULL, /* parent */ + 1, 0, /* alignment, bounds */ + BUS_SPACE_MAXADDR, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter, filterarg */ + IXL_TSO_SIZE, /* maxsize */ + IXL_MAX_TSO_SEGS, /* nsegments */ + PAGE_SIZE, /* maxsegsize */ + 0, /* flags */ + NULL, /* lockfunc */ + NULL, /* lockfuncarg */ + &txr->tso_tag))) { + device_printf(dev,"Unable to allocate TX TSO DMA tag\n"); + goto fail; + } + + if (!(txr->buffers = + (struct ixl_tx_buf *) malloc(sizeof(struct ixl_tx_buf) * + que->num_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { + device_printf(dev, "Unable to allocate tx_buffer memory\n"); + error = ENOMEM; + goto fail; + } + + /* Create the descriptor buffer default dma maps */ + buf = txr->buffers; + for (int i = 0; i < que->num_desc; i++, buf++) { + buf->tag = txr->tx_tag; + error = bus_dmamap_create(buf->tag, 0, &buf->map); + if (error != 0) { + device_printf(dev, "Unable to create TX DMA map\n"); + goto fail; + } + } +fail: + return (error); +} + + +/********************************************************************* + * + * (Re)Initialize a queue transmit ring. + * - called by init, it clears the descriptor ring, + * and frees any stale mbufs + * + **********************************************************************/ +void +ixl_init_tx_ring(struct ixl_queue *que) +{ +#ifdef DEV_NETMAP + struct netmap_adapter *na = NA(que->vsi->ifp); + struct netmap_slot *slot; +#endif /* DEV_NETMAP */ + struct tx_ring *txr = &que->txr; + struct ixl_tx_buf *buf; + + /* Clear the old ring contents */ + IXL_TX_LOCK(txr); + +#ifdef DEV_NETMAP + /* + * (under lock): if in netmap mode, do some consistency + * checks and set slot to entry 0 of the netmap ring. + */ + slot = netmap_reset(na, NR_TX, que->me, 0); +#endif /* DEV_NETMAP */ + + bzero((void *)txr->base, + (sizeof(struct i40e_tx_desc)) * que->num_desc); + + /* Reset indices */ + txr->next_avail = 0; + txr->next_to_clean = 0; + +#ifdef IXL_FDIR + /* Initialize flow director */ + txr->atr_rate = ixl_atr_rate; + txr->atr_count = 0; +#endif + + /* Free any existing tx mbufs. */ + buf = txr->buffers; + for (int i = 0; i < que->num_desc; i++, buf++) { + if (buf->m_head != NULL) { + bus_dmamap_sync(buf->tag, buf->map, + BUS_DMASYNC_POSTWRITE); + bus_dmamap_unload(buf->tag, buf->map); + m_freem(buf->m_head); + buf->m_head = NULL; + } +#ifdef DEV_NETMAP + /* + * In netmap mode, set the map for the packet buffer. + * NOTE: Some drivers (not this one) also need to set + * the physical buffer address in the NIC ring. + * netmap_idx_n2k() maps a nic index, i, into the corresponding + * netmap slot index, si + */ + if (slot) { + int si = netmap_idx_n2k(&na->tx_rings[que->me], i); + netmap_load_map(na, buf->tag, buf->map, NMB(na, slot + si)); + } +#endif /* DEV_NETMAP */ + /* Clear the EOP index */ + buf->eop_index = -1; + } + + /* Set number of descriptors available */ + txr->avail = que->num_desc; + + bus_dmamap_sync(txr->dma.tag, txr->dma.map, + BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); + IXL_TX_UNLOCK(txr); +} + + +/********************************************************************* + * + * Free transmit ring related data structures. + * + **********************************************************************/ +void +ixl_free_que_tx(struct ixl_queue *que) +{ + struct tx_ring *txr = &que->txr; + struct ixl_tx_buf *buf; + + INIT_DBG_IF(que->vsi->ifp, "queue %d: begin", que->me); + + for (int i = 0; i < que->num_desc; i++) { + buf = &txr->buffers[i]; + if (buf->m_head != NULL) { + bus_dmamap_sync(buf->tag, buf->map, + BUS_DMASYNC_POSTWRITE); + bus_dmamap_unload(buf->tag, + buf->map); + m_freem(buf->m_head); + buf->m_head = NULL; + if (buf->map != NULL) { + bus_dmamap_destroy(buf->tag, + buf->map); + buf->map = NULL; + } + } else if (buf->map != NULL) { + bus_dmamap_unload(buf->tag, + buf->map); + bus_dmamap_destroy(buf->tag, + buf->map); + buf->map = NULL; + } + } + if (txr->br != NULL) + buf_ring_free(txr->br, M_DEVBUF); + if (txr->buffers != NULL) { + free(txr->buffers, M_DEVBUF); + txr->buffers = NULL; + } + if (txr->tx_tag != NULL) { + bus_dma_tag_destroy(txr->tx_tag); + txr->tx_tag = NULL; + } + if (txr->tso_tag != NULL) { + bus_dma_tag_destroy(txr->tso_tag); + txr->tso_tag = NULL; + } + + INIT_DBG_IF(que->vsi->ifp, "queue %d: end", que->me); + return; +} + +/********************************************************************* + * + * Setup descriptor for hw offloads + * + **********************************************************************/ + +static int +ixl_tx_setup_offload(struct ixl_queue *que, + struct mbuf *mp, u32 *cmd, u32 *off) +{ + struct ether_vlan_header *eh; +#ifdef INET + struct ip *ip = NULL; +#endif + struct tcphdr *th = NULL; +#ifdef INET6 + struct ip6_hdr *ip6; +#endif + int elen, ip_hlen = 0, tcp_hlen; + u16 etype; + u8 ipproto = 0; + bool tso = FALSE; + + + /* Set up the TSO context descriptor if required */ + if (mp->m_pkthdr.csum_flags & CSUM_TSO) { + tso = ixl_tso_setup(que, mp); + if (tso) + ++que->tso; + else + return (ENXIO); + } + + /* + * Determine where frame payload starts. + * Jump over vlan headers if already present, + * helpful for QinQ too. + */ + eh = mtod(mp, struct ether_vlan_header *); + if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { + etype = ntohs(eh->evl_proto); + elen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; + } else { + etype = ntohs(eh->evl_encap_proto); + elen = ETHER_HDR_LEN; + } + + switch (etype) { +#ifdef INET + case ETHERTYPE_IP: + ip = (struct ip *)(mp->m_data + elen); + ip_hlen = ip->ip_hl << 2; + ipproto = ip->ip_p; + th = (struct tcphdr *)((caddr_t)ip + ip_hlen); + /* The IP checksum must be recalculated with TSO */ + if (tso) + *cmd |= I40E_TX_DESC_CMD_IIPT_IPV4_CSUM; + else + *cmd |= I40E_TX_DESC_CMD_IIPT_IPV4; + break; +#endif +#ifdef INET6 + case ETHERTYPE_IPV6: + ip6 = (struct ip6_hdr *)(mp->m_data + elen); + ip_hlen = sizeof(struct ip6_hdr); + ipproto = ip6->ip6_nxt; + th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen); + *cmd |= I40E_TX_DESC_CMD_IIPT_IPV6; + break; +#endif + default: + break; + } + + *off |= (elen >> 1) << I40E_TX_DESC_LENGTH_MACLEN_SHIFT; + *off |= (ip_hlen >> 2) << I40E_TX_DESC_LENGTH_IPLEN_SHIFT; + + switch (ipproto) { + case IPPROTO_TCP: + tcp_hlen = th->th_off << 2; + if (mp->m_pkthdr.csum_flags & (CSUM_TCP|CSUM_TCP_IPV6)) { + *cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP; + *off |= (tcp_hlen >> 2) << + I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; + } +#ifdef IXL_FDIR + ixl_atr(que, th, etype); +#endif + break; + case IPPROTO_UDP: + if (mp->m_pkthdr.csum_flags & (CSUM_UDP|CSUM_UDP_IPV6)) { + *cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP; + *off |= (sizeof(struct udphdr) >> 2) << + I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; + } + break; + + case IPPROTO_SCTP: + if (mp->m_pkthdr.csum_flags & (CSUM_SCTP|CSUM_SCTP_IPV6)) { + *cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP; + *off |= (sizeof(struct sctphdr) >> 2) << + I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; + } + /* Fall Thru */ + default: + break; + } + + return (0); +} + + +/********************************************************************** + * + * Setup context for hardware segmentation offload (TSO) + * + **********************************************************************/ +static bool +ixl_tso_setup(struct ixl_queue *que, struct mbuf *mp) +{ + struct tx_ring *txr = &que->txr; + struct i40e_tx_context_desc *TXD; + struct ixl_tx_buf *buf; + u32 cmd, mss, type, tsolen; + u16 etype; + int idx, elen, ip_hlen, tcp_hlen; + struct ether_vlan_header *eh; +#ifdef INET + struct ip *ip; +#endif +#ifdef INET6 + struct ip6_hdr *ip6; +#endif +#if defined(INET6) || defined(INET) + struct tcphdr *th; +#endif + u64 type_cmd_tso_mss; + + /* + * Determine where frame payload starts. + * Jump over vlan headers if already present + */ + eh = mtod(mp, struct ether_vlan_header *); + if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { + elen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; + etype = eh->evl_proto; + } else { + elen = ETHER_HDR_LEN; + etype = eh->evl_encap_proto; + } + + switch (ntohs(etype)) { +#ifdef INET6 + case ETHERTYPE_IPV6: + ip6 = (struct ip6_hdr *)(mp->m_data + elen); + if (ip6->ip6_nxt != IPPROTO_TCP) + return (ENXIO); + ip_hlen = sizeof(struct ip6_hdr); + th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen); + th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0); + tcp_hlen = th->th_off << 2; + break; +#endif +#ifdef INET + case ETHERTYPE_IP: + ip = (struct ip *)(mp->m_data + elen); + if (ip->ip_p != IPPROTO_TCP) + return (ENXIO); + ip->ip_sum = 0; + ip_hlen = ip->ip_hl << 2; + th = (struct tcphdr *)((caddr_t)ip + ip_hlen); + th->th_sum = in_pseudo(ip->ip_src.s_addr, + ip->ip_dst.s_addr, htons(IPPROTO_TCP)); + tcp_hlen = th->th_off << 2; + break; +#endif + default: + printf("%s: CSUM_TSO but no supported IP version (0x%04x)", + __func__, ntohs(etype)); + return FALSE; + } + + /* Ensure we have at least the IP+TCP header in the first mbuf. */ + if (mp->m_len < elen + ip_hlen + sizeof(struct tcphdr)) + return FALSE; + + idx = txr->next_avail; + buf = &txr->buffers[idx]; + TXD = (struct i40e_tx_context_desc *) &txr->base[idx]; + tsolen = mp->m_pkthdr.len - (elen + ip_hlen + tcp_hlen); + + type = I40E_TX_DESC_DTYPE_CONTEXT; + cmd = I40E_TX_CTX_DESC_TSO; + mss = mp->m_pkthdr.tso_segsz; + + type_cmd_tso_mss = ((u64)type << I40E_TXD_CTX_QW1_DTYPE_SHIFT) | + ((u64)cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) | + ((u64)tsolen << I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) | + ((u64)mss << I40E_TXD_CTX_QW1_MSS_SHIFT); + TXD->type_cmd_tso_mss = htole64(type_cmd_tso_mss); + + TXD->tunneling_params = htole32(0); + buf->m_head = NULL; + buf->eop_index = -1; + + if (++idx == que->num_desc) + idx = 0; + + txr->avail--; + txr->next_avail = idx; + + return TRUE; +} + +/* +** ixl_get_tx_head - Retrieve the value from the +** location the HW records its HEAD index +*/ +static inline u32 +ixl_get_tx_head(struct ixl_queue *que) +{ + struct tx_ring *txr = &que->txr; + void *head = &txr->base[que->num_desc]; + return LE32_TO_CPU(*(volatile __le32 *)head); +} + +/********************************************************************** + * + * Examine each tx_buffer in the used queue. If the hardware is done + * processing the packet then free associated resources. The + * tx_buffer is put back on the free queue. + * + **********************************************************************/ +bool +ixl_txeof(struct ixl_queue *que) +{ + struct tx_ring *txr = &que->txr; + u32 first, last, head, done, processed; + struct ixl_tx_buf *buf; + struct i40e_tx_desc *tx_desc, *eop_desc; + + + mtx_assert(&txr->mtx, MA_OWNED); + +#ifdef DEV_NETMAP + // XXX todo: implement moderation + if (netmap_tx_irq(que->vsi->ifp, que->me)) + return FALSE; +#endif /* DEF_NETMAP */ + + /* These are not the descriptors you seek, move along :) */ + if (txr->avail == que->num_desc) { + que->busy = 0; + return FALSE; + } + + processed = 0; + first = txr->next_to_clean; + buf = &txr->buffers[first]; + tx_desc = (struct i40e_tx_desc *)&txr->base[first]; + last = buf->eop_index; + if (last == -1) + return FALSE; + eop_desc = (struct i40e_tx_desc *)&txr->base[last]; + + /* Get the Head WB value */ + head = ixl_get_tx_head(que); + + /* + ** Get the index of the first descriptor + ** BEYOND the EOP and call that 'done'. + ** I do this so the comparison in the + ** inner while loop below can be simple + */ + if (++last == que->num_desc) last = 0; + done = last; + + bus_dmamap_sync(txr->dma.tag, txr->dma.map, + BUS_DMASYNC_POSTREAD); + /* + ** The HEAD index of the ring is written in a + ** defined location, this rather than a done bit + ** is what is used to keep track of what must be + ** 'cleaned'. + */ + while (first != head) { + /* We clean the range of the packet */ + while (first != done) { + ++txr->avail; + ++processed; + + if (buf->m_head) { + txr->bytes += /* for ITR adjustment */ + buf->m_head->m_pkthdr.len; + txr->tx_bytes += /* for TX stats */ + buf->m_head->m_pkthdr.len; + bus_dmamap_sync(buf->tag, + buf->map, + BUS_DMASYNC_POSTWRITE); + bus_dmamap_unload(buf->tag, + buf->map); + m_freem(buf->m_head); + buf->m_head = NULL; + buf->map = NULL; + } + buf->eop_index = -1; + + if (++first == que->num_desc) + first = 0; + + buf = &txr->buffers[first]; + tx_desc = &txr->base[first]; + } + ++txr->packets; + /* See if there is more work now */ + last = buf->eop_index; + if (last != -1) { + eop_desc = &txr->base[last]; + /* Get next done point */ + if (++last == que->num_desc) last = 0; + done = last; + } else + break; + } + bus_dmamap_sync(txr->dma.tag, txr->dma.map, + BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); + + txr->next_to_clean = first; + + + /* + ** Hang detection, we know there's + ** work outstanding or the first return + ** would have been taken, so indicate an + ** unsuccessful pass, in local_timer if + ** the value is too great the queue will + ** be considered hung. If anything has been + ** cleaned then reset the state. + */ + if ((processed == 0) && (que->busy != IXL_QUEUE_HUNG)) + ++que->busy; + + if (processed) + que->busy = 1; /* Note this turns off HUNG */ + + /* + * If there are no pending descriptors, clear the timeout. + */ + if (txr->avail == que->num_desc) { + que->busy = 0; + return FALSE; + } + + return TRUE; +} + +/********************************************************************* + * + * Refresh mbuf buffers for RX descriptor rings + * - now keeps its own state so discards due to resource + * exhaustion are unnecessary, if an mbuf cannot be obtained + * it just returns, keeping its placeholder, thus it can simply + * be recalled to try again. + * + **********************************************************************/ +static void +ixl_refresh_mbufs(struct ixl_queue *que, int limit) +{ + struct ixl_vsi *vsi = que->vsi; + struct rx_ring *rxr = &que->rxr; + bus_dma_segment_t hseg[1]; + bus_dma_segment_t pseg[1]; + struct ixl_rx_buf *buf; + struct mbuf *mh, *mp; + int i, j, nsegs, error; + bool refreshed = FALSE; + + i = j = rxr->next_refresh; + /* Control the loop with one beyond */ + if (++j == que->num_desc) + j = 0; + + while (j != limit) { + buf = &rxr->buffers[i]; + if (rxr->hdr_split == FALSE) + goto no_split; + + if (buf->m_head == NULL) { + mh = m_gethdr(M_NOWAIT, MT_DATA); + if (mh == NULL) + goto update; + } else + mh = buf->m_head; + + mh->m_pkthdr.len = mh->m_len = MHLEN; + mh->m_len = MHLEN; + mh->m_flags |= M_PKTHDR; + /* Get the memory mapping */ + error = bus_dmamap_load_mbuf_sg(rxr->htag, + buf->hmap, mh, hseg, &nsegs, BUS_DMA_NOWAIT); + if (error != 0) { + printf("Refresh mbufs: hdr dmamap load" + " failure - %d\n", error); + m_free(mh); + buf->m_head = NULL; + goto update; + } + buf->m_head = mh; + bus_dmamap_sync(rxr->htag, buf->hmap, + BUS_DMASYNC_PREREAD); + rxr->base[i].read.hdr_addr = + htole64(hseg[0].ds_addr); + +no_split: + if (buf->m_pack == NULL) { + mp = m_getjcl(M_NOWAIT, MT_DATA, + M_PKTHDR, rxr->mbuf_sz); + if (mp == NULL) + goto update; + } else + mp = buf->m_pack; + + mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz; + /* Get the memory mapping */ + error = bus_dmamap_load_mbuf_sg(rxr->ptag, + buf->pmap, mp, pseg, &nsegs, BUS_DMA_NOWAIT); + if (error != 0) { + printf("Refresh mbufs: payload dmamap load" + " failure - %d\n", error); + m_free(mp); + buf->m_pack = NULL; + goto update; + } + buf->m_pack = mp; + bus_dmamap_sync(rxr->ptag, buf->pmap, + BUS_DMASYNC_PREREAD); + rxr->base[i].read.pkt_addr = + htole64(pseg[0].ds_addr); + /* Used only when doing header split */ + rxr->base[i].read.hdr_addr = 0; + + refreshed = TRUE; + /* Next is precalculated */ + i = j; + rxr->next_refresh = i; + if (++j == que->num_desc) + j = 0; + } +update: + if (refreshed) /* Update hardware tail index */ + wr32(vsi->hw, rxr->tail, rxr->next_refresh); + return; +} + + +/********************************************************************* + * + * Allocate memory for rx_buffer structures. Since we use one + * rx_buffer per descriptor, the maximum number of rx_buffer's + * that we'll need is equal to the number of receive descriptors + * that we've defined. + * + **********************************************************************/ +int +ixl_allocate_rx_data(struct ixl_queue *que) +{ + struct rx_ring *rxr = &que->rxr; + struct ixl_vsi *vsi = que->vsi; + device_t dev = vsi->dev; + struct ixl_rx_buf *buf; + int i, bsize, error; + + bsize = sizeof(struct ixl_rx_buf) * que->num_desc; + if (!(rxr->buffers = + (struct ixl_rx_buf *) malloc(bsize, + M_DEVBUF, M_NOWAIT | M_ZERO))) { + device_printf(dev, "Unable to allocate rx_buffer memory\n"); + error = ENOMEM; + return (error); + } + + if ((error = bus_dma_tag_create(NULL, /* parent */ + 1, 0, /* alignment, bounds */ + BUS_SPACE_MAXADDR, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter, filterarg */ + MSIZE, /* maxsize */ + 1, /* nsegments */ + MSIZE, /* maxsegsize */ + 0, /* flags */ + NULL, /* lockfunc */ + NULL, /* lockfuncarg */ + &rxr->htag))) { + device_printf(dev, "Unable to create RX DMA htag\n"); + return (error); + } + + if ((error = bus_dma_tag_create(NULL, /* parent */ + 1, 0, /* alignment, bounds */ + BUS_SPACE_MAXADDR, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter, filterarg */ + MJUM16BYTES, /* maxsize */ + 1, /* nsegments */ + MJUM16BYTES, /* maxsegsize */ + 0, /* flags */ + NULL, /* lockfunc */ + NULL, /* lockfuncarg */ + &rxr->ptag))) { + device_printf(dev, "Unable to create RX DMA ptag\n"); + return (error); + } + + for (i = 0; i < que->num_desc; i++) { + buf = &rxr->buffers[i]; + error = bus_dmamap_create(rxr->htag, + BUS_DMA_NOWAIT, &buf->hmap); + if (error) { + device_printf(dev, "Unable to create RX head map\n"); + break; + } + error = bus_dmamap_create(rxr->ptag, + BUS_DMA_NOWAIT, &buf->pmap); + if (error) { + device_printf(dev, "Unable to create RX pkt map\n"); + break; + } + } + + return (error); +} + + +/********************************************************************* + * + * (Re)Initialize the queue receive ring and its buffers. + * + **********************************************************************/ +int +ixl_init_rx_ring(struct ixl_queue *que) +{ + struct rx_ring *rxr = &que->rxr; + struct ixl_vsi *vsi = que->vsi; +#if defined(INET6) || defined(INET) + struct ifnet *ifp = vsi->ifp; + struct lro_ctrl *lro = &rxr->lro; +#endif + struct ixl_rx_buf *buf; + bus_dma_segment_t pseg[1], hseg[1]; + int rsize, nsegs, error = 0; +#ifdef DEV_NETMAP + struct netmap_adapter *na = NA(que->vsi->ifp); + struct netmap_slot *slot; +#endif /* DEV_NETMAP */ + + IXL_RX_LOCK(rxr); +#ifdef DEV_NETMAP + /* same as in ixl_init_tx_ring() */ + slot = netmap_reset(na, NR_RX, que->me, 0); +#endif /* DEV_NETMAP */ + /* Clear the ring contents */ + rsize = roundup2(que->num_desc * + sizeof(union i40e_rx_desc), DBA_ALIGN); + bzero((void *)rxr->base, rsize); + /* Cleanup any existing buffers */ + for (int i = 0; i < que->num_desc; i++) { + buf = &rxr->buffers[i]; + if (buf->m_head != NULL) { + bus_dmamap_sync(rxr->htag, buf->hmap, + BUS_DMASYNC_POSTREAD); + bus_dmamap_unload(rxr->htag, buf->hmap); + buf->m_head->m_flags |= M_PKTHDR; + m_freem(buf->m_head); + } + if (buf->m_pack != NULL) { + bus_dmamap_sync(rxr->ptag, buf->pmap, + BUS_DMASYNC_POSTREAD); + bus_dmamap_unload(rxr->ptag, buf->pmap); + buf->m_pack->m_flags |= M_PKTHDR; + m_freem(buf->m_pack); + } + buf->m_head = NULL; + buf->m_pack = NULL; + } + + /* header split is off */ + rxr->hdr_split = FALSE; + + /* Now replenish the mbufs */ + for (int j = 0; j != que->num_desc; ++j) { + struct mbuf *mh, *mp; + + buf = &rxr->buffers[j]; +#ifdef DEV_NETMAP + /* + * In netmap mode, fill the map and set the buffer + * address in the NIC ring, considering the offset + * between the netmap and NIC rings (see comment in + * ixgbe_setup_transmit_ring() ). No need to allocate + * an mbuf, so end the block with a continue; + */ + if (slot) { + int sj = netmap_idx_n2k(&na->rx_rings[que->me], j); + uint64_t paddr; + void *addr; + + addr = PNMB(na, slot + sj, &paddr); + netmap_load_map(na, rxr->dma.tag, buf->pmap, addr); + /* Update descriptor and the cached value */ + rxr->base[j].read.pkt_addr = htole64(paddr); + rxr->base[j].read.hdr_addr = 0; + continue; + } +#endif /* DEV_NETMAP */ + /* + ** Don't allocate mbufs if not + ** doing header split, its wasteful + */ + if (rxr->hdr_split == FALSE) + goto skip_head; + + /* First the header */ + buf->m_head = m_gethdr(M_NOWAIT, MT_DATA); + if (buf->m_head == NULL) { + error = ENOBUFS; + goto fail; + } + m_adj(buf->m_head, ETHER_ALIGN); + mh = buf->m_head; + mh->m_len = mh->m_pkthdr.len = MHLEN; + mh->m_flags |= M_PKTHDR; + /* Get the memory mapping */ + error = bus_dmamap_load_mbuf_sg(rxr->htag, + buf->hmap, buf->m_head, hseg, + &nsegs, BUS_DMA_NOWAIT); + if (error != 0) /* Nothing elegant to do here */ + goto fail; + bus_dmamap_sync(rxr->htag, + buf->hmap, BUS_DMASYNC_PREREAD); + /* Update descriptor */ + rxr->base[j].read.hdr_addr = htole64(hseg[0].ds_addr); + +skip_head: + /* Now the payload cluster */ + buf->m_pack = m_getjcl(M_NOWAIT, MT_DATA, + M_PKTHDR, rxr->mbuf_sz); + if (buf->m_pack == NULL) { + error = ENOBUFS; + goto fail; + } + mp = buf->m_pack; + mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz; + /* Get the memory mapping */ + error = bus_dmamap_load_mbuf_sg(rxr->ptag, + buf->pmap, mp, pseg, + &nsegs, BUS_DMA_NOWAIT); + if (error != 0) + goto fail; + bus_dmamap_sync(rxr->ptag, + buf->pmap, BUS_DMASYNC_PREREAD); + /* Update descriptor */ + rxr->base[j].read.pkt_addr = htole64(pseg[0].ds_addr); + rxr->base[j].read.hdr_addr = 0; + } + + + /* Setup our descriptor indices */ + rxr->next_check = 0; + rxr->next_refresh = 0; + rxr->lro_enabled = FALSE; + rxr->split = 0; + rxr->bytes = 0; + rxr->discard = FALSE; + + wr32(vsi->hw, rxr->tail, que->num_desc - 1); + ixl_flush(vsi->hw); + +#if defined(INET6) || defined(INET) + /* + ** Now set up the LRO interface: + */ + if (ifp->if_capenable & IFCAP_LRO) { + int err = tcp_lro_init(lro); + if (err) { + if_printf(ifp, "queue %d: LRO Initialization failed!\n", que->me); + goto fail; + } + INIT_DBG_IF(ifp, "queue %d: RX Soft LRO Initialized", que->me); + rxr->lro_enabled = TRUE; + lro->ifp = vsi->ifp; + } +#endif + + bus_dmamap_sync(rxr->dma.tag, rxr->dma.map, + BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); + +fail: + IXL_RX_UNLOCK(rxr); + return (error); +} + + +/********************************************************************* + * + * Free station receive ring data structures + * + **********************************************************************/ +void +ixl_free_que_rx(struct ixl_queue *que) +{ + struct rx_ring *rxr = &que->rxr; + struct ixl_rx_buf *buf; + + INIT_DBG_IF(que->vsi->ifp, "queue %d: begin", que->me); + + /* Cleanup any existing buffers */ + if (rxr->buffers != NULL) { + for (int i = 0; i < que->num_desc; i++) { + buf = &rxr->buffers[i]; + if (buf->m_head != NULL) { + bus_dmamap_sync(rxr->htag, buf->hmap, + BUS_DMASYNC_POSTREAD); + bus_dmamap_unload(rxr->htag, buf->hmap); + buf->m_head->m_flags |= M_PKTHDR; + m_freem(buf->m_head); + } + if (buf->m_pack != NULL) { + bus_dmamap_sync(rxr->ptag, buf->pmap, + BUS_DMASYNC_POSTREAD); + bus_dmamap_unload(rxr->ptag, buf->pmap); + buf->m_pack->m_flags |= M_PKTHDR; + m_freem(buf->m_pack); + } + buf->m_head = NULL; + buf->m_pack = NULL; + if (buf->hmap != NULL) { + bus_dmamap_destroy(rxr->htag, buf->hmap); + buf->hmap = NULL; + } + if (buf->pmap != NULL) { + bus_dmamap_destroy(rxr->ptag, buf->pmap); + buf->pmap = NULL; + } + } + if (rxr->buffers != NULL) { + free(rxr->buffers, M_DEVBUF); + rxr->buffers = NULL; + } + } + + if (rxr->htag != NULL) { + bus_dma_tag_destroy(rxr->htag); + rxr->htag = NULL; + } + if (rxr->ptag != NULL) { + bus_dma_tag_destroy(rxr->ptag); + rxr->ptag = NULL; + } + + INIT_DBG_IF(que->vsi->ifp, "queue %d: end", que->me); + return; +} + +static __inline void +ixl_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u8 ptype) +{ + +#if defined(INET6) || defined(INET) + /* + * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet + * should be computed by hardware. Also it should not have VLAN tag in + * ethernet header. + */ + if (rxr->lro_enabled && + (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 && + (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) == + (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) { + /* + * Send to the stack if: + ** - LRO not enabled, or + ** - no LRO resources, or + ** - lro enqueue fails + */ + if (rxr->lro.lro_cnt != 0) + if (tcp_lro_rx(&rxr->lro, m, 0) == 0) + return; + } +#endif + IXL_RX_UNLOCK(rxr); + (*ifp->if_input)(ifp, m); + IXL_RX_LOCK(rxr); +} + + +static __inline void +ixl_rx_discard(struct rx_ring *rxr, int i) +{ + struct ixl_rx_buf *rbuf; + + rbuf = &rxr->buffers[i]; + + if (rbuf->fmp != NULL) {/* Partial chain ? */ + rbuf->fmp->m_flags |= M_PKTHDR; + m_freem(rbuf->fmp); + rbuf->fmp = NULL; + } + + /* + ** With advanced descriptors the writeback + ** clobbers the buffer addrs, so its easier + ** to just free the existing mbufs and take + ** the normal refresh path to get new buffers + ** and mapping. + */ + if (rbuf->m_head) { + m_free(rbuf->m_head); + rbuf->m_head = NULL; + } + + if (rbuf->m_pack) { + m_free(rbuf->m_pack); + rbuf->m_pack = NULL; + } + + return; +} + +#ifdef RSS +/* +** i40e_ptype_to_hash: parse the packet type +** to determine the appropriate hash. +*/ +static inline int +ixl_ptype_to_hash(u8 ptype) +{ + struct i40e_rx_ptype_decoded decoded; + u8 ex = 0; + + decoded = decode_rx_desc_ptype(ptype); + ex = decoded.outer_frag; + + if (!decoded.known) + return M_HASHTYPE_OPAQUE; + + if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_L2) + return M_HASHTYPE_OPAQUE; + + /* Note: anything that gets to this point is IP */ + if (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6) { + switch (decoded.inner_prot) { + case I40E_RX_PTYPE_INNER_PROT_TCP: + if (ex) + return M_HASHTYPE_RSS_TCP_IPV6_EX; + else + return M_HASHTYPE_RSS_TCP_IPV6; + case I40E_RX_PTYPE_INNER_PROT_UDP: + if (ex) + return M_HASHTYPE_RSS_UDP_IPV6_EX; + else + return M_HASHTYPE_RSS_UDP_IPV6; + default: + if (ex) + return M_HASHTYPE_RSS_IPV6_EX; + else + return M_HASHTYPE_RSS_IPV6; + } + } + if (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV4) { + switch (decoded.inner_prot) { + case I40E_RX_PTYPE_INNER_PROT_TCP: + return M_HASHTYPE_RSS_TCP_IPV4; + case I40E_RX_PTYPE_INNER_PROT_UDP: + if (ex) + return M_HASHTYPE_RSS_UDP_IPV4_EX; + else + return M_HASHTYPE_RSS_UDP_IPV4; + default: + return M_HASHTYPE_RSS_IPV4; + } + } + /* We should never get here!! */ + return M_HASHTYPE_OPAQUE; +} +#endif /* RSS */ + +/********************************************************************* + * + * This routine executes in interrupt context. It replenishes + * the mbufs in the descriptor and sends data which has been + * dma'ed into host memory to upper layer. + * + * We loop at most count times if count is > 0, or until done if + * count < 0. + * + * Return TRUE for more work, FALSE for all clean. + *********************************************************************/ +bool +ixl_rxeof(struct ixl_queue *que, int count) +{ + struct ixl_vsi *vsi = que->vsi; + struct rx_ring *rxr = &que->rxr; + struct ifnet *ifp = vsi->ifp; +#if defined(INET6) || defined(INET) + struct lro_ctrl *lro = &rxr->lro; +#endif + int i, nextp, processed = 0; + union i40e_rx_desc *cur; + struct ixl_rx_buf *rbuf, *nbuf; + + + IXL_RX_LOCK(rxr); + +#ifdef DEV_NETMAP + if (netmap_rx_irq(ifp, que->me, &count)) { + IXL_RX_UNLOCK(rxr); + return (FALSE); + } +#endif /* DEV_NETMAP */ + + for (i = rxr->next_check; count != 0;) { + struct mbuf *sendmp, *mh, *mp; + u32 rsc, status, error; + u16 hlen, plen, vtag; + u64 qword; + u8 ptype; + bool eop; + + /* Sync the ring. */ + bus_dmamap_sync(rxr->dma.tag, rxr->dma.map, + BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); + + cur = &rxr->base[i]; + qword = le64toh(cur->wb.qword1.status_error_len); + status = (qword & I40E_RXD_QW1_STATUS_MASK) + >> I40E_RXD_QW1_STATUS_SHIFT; + error = (qword & I40E_RXD_QW1_ERROR_MASK) + >> I40E_RXD_QW1_ERROR_SHIFT; + plen = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) + >> I40E_RXD_QW1_LENGTH_PBUF_SHIFT; + hlen = (qword & I40E_RXD_QW1_LENGTH_HBUF_MASK) + >> I40E_RXD_QW1_LENGTH_HBUF_SHIFT; + ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) + >> I40E_RXD_QW1_PTYPE_SHIFT; + + if ((status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) == 0) { + ++rxr->not_done; + break; + } + if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) + break; + + count--; + sendmp = NULL; + nbuf = NULL; + rsc = 0; + cur->wb.qword1.status_error_len = 0; + rbuf = &rxr->buffers[i]; + mh = rbuf->m_head; + mp = rbuf->m_pack; + eop = (status & (1 << I40E_RX_DESC_STATUS_EOF_SHIFT)); + if (status & (1 << I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)) + vtag = le16toh(cur->wb.qword0.lo_dword.l2tag1); + else + vtag = 0; + + /* + ** Make sure bad packets are discarded, + ** note that only EOP descriptor has valid + ** error results. + */ + if (eop && (error & (1 << I40E_RX_DESC_ERROR_RXE_SHIFT))) { + rxr->discarded++; + ixl_rx_discard(rxr, i); + goto next_desc; + } + + /* Prefetch the next buffer */ + if (!eop) { + nextp = i + 1; + if (nextp == que->num_desc) + nextp = 0; + nbuf = &rxr->buffers[nextp]; + prefetch(nbuf); + } + + /* + ** The header mbuf is ONLY used when header + ** split is enabled, otherwise we get normal + ** behavior, ie, both header and payload + ** are DMA'd into the payload buffer. + ** + ** Rather than using the fmp/lmp global pointers + ** we now keep the head of a packet chain in the + ** buffer struct and pass this along from one + ** descriptor to the next, until we get EOP. + */ + if (rxr->hdr_split && (rbuf->fmp == NULL)) { + if (hlen > IXL_RX_HDR) + hlen = IXL_RX_HDR; + mh->m_len = hlen; + mh->m_flags |= M_PKTHDR; + mh->m_next = NULL; + mh->m_pkthdr.len = mh->m_len; + /* Null buf pointer so it is refreshed */ + rbuf->m_head = NULL; + /* + ** Check the payload length, this + ** could be zero if its a small + ** packet. + */ + if (plen > 0) { + mp->m_len = plen; + mp->m_next = NULL; + mp->m_flags &= ~M_PKTHDR; + mh->m_next = mp; + mh->m_pkthdr.len += mp->m_len; + /* Null buf pointer so it is refreshed */ + rbuf->m_pack = NULL; + rxr->split++; + } + /* + ** Now create the forward + ** chain so when complete + ** we wont have to. + */ + if (eop == 0) { + /* stash the chain head */ + nbuf->fmp = mh; + /* Make forward chain */ + if (plen) + mp->m_next = nbuf->m_pack; + else + mh->m_next = nbuf->m_pack; + } else { + /* Singlet, prepare to send */ + sendmp = mh; + if (vtag) { + sendmp->m_pkthdr.ether_vtag = vtag; + sendmp->m_flags |= M_VLANTAG; + } + } + } else { + /* + ** Either no header split, or a + ** secondary piece of a fragmented + ** split packet. + */ + mp->m_len = plen; + /* + ** See if there is a stored head + ** that determines what we are + */ + sendmp = rbuf->fmp; + rbuf->m_pack = rbuf->fmp = NULL; + + if (sendmp != NULL) /* secondary frag */ + sendmp->m_pkthdr.len += mp->m_len; + else { + /* first desc of a non-ps chain */ + sendmp = mp; + sendmp->m_flags |= M_PKTHDR; + sendmp->m_pkthdr.len = mp->m_len; + if (vtag) { + sendmp->m_pkthdr.ether_vtag = vtag; + sendmp->m_flags |= M_VLANTAG; + } + } + /* Pass the head pointer on */ + if (eop == 0) { + nbuf->fmp = sendmp; + sendmp = NULL; + mp->m_next = nbuf->m_pack; + } + } + ++processed; + /* Sending this frame? */ + if (eop) { + sendmp->m_pkthdr.rcvif = ifp; + /* gather stats */ + rxr->rx_packets++; + rxr->rx_bytes += sendmp->m_pkthdr.len; + /* capture data for dynamic ITR adjustment */ + rxr->packets++; + rxr->bytes += sendmp->m_pkthdr.len; + if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) + ixl_rx_checksum(sendmp, status, error, ptype); +#ifdef RSS + sendmp->m_pkthdr.flowid = + le32toh(cur->wb.qword0.hi_dword.rss); + M_HASHTYPE_SET(sendmp, ixl_ptype_to_hash(ptype)); +#else + sendmp->m_pkthdr.flowid = que->msix; + M_HASHTYPE_SET(sendmp, M_HASHTYPE_OPAQUE); +#endif + } +next_desc: + bus_dmamap_sync(rxr->dma.tag, rxr->dma.map, + BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); + + /* Advance our pointers to the next descriptor. */ + if (++i == que->num_desc) + i = 0; + + /* Now send to the stack or do LRO */ + if (sendmp != NULL) { + rxr->next_check = i; + ixl_rx_input(rxr, ifp, sendmp, ptype); + i = rxr->next_check; + } + + /* Every 8 descriptors we go to refresh mbufs */ + if (processed == 8) { + ixl_refresh_mbufs(que, i); + processed = 0; + } + } + + /* Refresh any remaining buf structs */ + if (ixl_rx_unrefreshed(que)) + ixl_refresh_mbufs(que, i); + + rxr->next_check = i; + +#if defined(INET6) || defined(INET) + /* + * Flush any outstanding LRO work + */ + tcp_lro_flush_all(lro); +#endif + + IXL_RX_UNLOCK(rxr); + return (FALSE); +} + + +/********************************************************************* + * + * Verify that the hardware indicated that the checksum is valid. + * Inform the stack about the status of checksum so that stack + * doesn't spend time verifying the checksum. + * + *********************************************************************/ +static void +ixl_rx_checksum(struct mbuf * mp, u32 status, u32 error, u8 ptype) +{ + struct i40e_rx_ptype_decoded decoded; + + decoded = decode_rx_desc_ptype(ptype); + + /* Errors? */ + if (error & ((1 << I40E_RX_DESC_ERROR_IPE_SHIFT) | + (1 << I40E_RX_DESC_ERROR_L4E_SHIFT))) { + mp->m_pkthdr.csum_flags = 0; + return; + } + + /* IPv6 with extension headers likely have bad csum */ + if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP && + decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6) + if (status & + (1 << I40E_RX_DESC_STATUS_IPV6EXADD_SHIFT)) { + mp->m_pkthdr.csum_flags = 0; + return; + } + + + /* IP Checksum Good */ + mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED; + mp->m_pkthdr.csum_flags |= CSUM_IP_VALID; + + if (status & (1 << I40E_RX_DESC_STATUS_L3L4P_SHIFT)) { + mp->m_pkthdr.csum_flags |= + (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); + mp->m_pkthdr.csum_data |= htons(0xffff); + } + return; +} + +#if __FreeBSD_version >= 1100000 +uint64_t +ixl_get_counter(if_t ifp, ift_counter cnt) +{ + struct ixl_vsi *vsi; + + vsi = if_getsoftc(ifp); + + switch (cnt) { + case IFCOUNTER_IPACKETS: + return (vsi->ipackets); + case IFCOUNTER_IERRORS: + return (vsi->ierrors); + case IFCOUNTER_OPACKETS: + return (vsi->opackets); + case IFCOUNTER_OERRORS: + return (vsi->oerrors); + case IFCOUNTER_COLLISIONS: + /* Collisions are by standard impossible in 40G/10G Ethernet */ + return (0); + case IFCOUNTER_IBYTES: + return (vsi->ibytes); + case IFCOUNTER_OBYTES: + return (vsi->obytes); + case IFCOUNTER_IMCASTS: + return (vsi->imcasts); + case IFCOUNTER_OMCASTS: + return (vsi->omcasts); + case IFCOUNTER_IQDROPS: + return (vsi->iqdrops); + case IFCOUNTER_OQDROPS: + return (vsi->oqdrops); + case IFCOUNTER_NOPROTO: + return (vsi->noproto); + default: + return (if_get_counter_default(ifp, cnt)); + } +} +#endif + Index: sys/dev/ixl/legacy_ixlv.h =================================================================== --- /dev/null +++ sys/dev/ixl/legacy_ixlv.h @@ -0,0 +1,213 @@ +/****************************************************************************** + + Copyright (c) 2013-2015, Intel Corporation + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of the Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +******************************************************************************/ +/*$FreeBSD$*/ + + +#ifndef _IXLV_H_ +#define _IXLV_H_ + +#include "ixlv_vc_mgr.h" + +#define IXLV_AQ_MAX_ERR 1000 +#define IXLV_MAX_FILTERS 128 +#define IXLV_MAX_QUEUES 16 +#define IXLV_AQ_TIMEOUT (1 * hz) +#define IXLV_CALLOUT_TIMO (hz / 50) /* 20 msec */ + +#define IXLV_FLAG_AQ_ENABLE_QUEUES (u32)(1) +#define IXLV_FLAG_AQ_DISABLE_QUEUES (u32)(1 << 1) +#define IXLV_FLAG_AQ_ADD_MAC_FILTER (u32)(1 << 2) +#define IXLV_FLAG_AQ_ADD_VLAN_FILTER (u32)(1 << 3) +#define IXLV_FLAG_AQ_DEL_MAC_FILTER (u32)(1 << 4) +#define IXLV_FLAG_AQ_DEL_VLAN_FILTER (u32)(1 << 5) +#define IXLV_FLAG_AQ_CONFIGURE_QUEUES (u32)(1 << 6) +#define IXLV_FLAG_AQ_MAP_VECTORS (u32)(1 << 7) +#define IXLV_FLAG_AQ_HANDLE_RESET (u32)(1 << 8) +#define IXLV_FLAG_AQ_CONFIGURE_PROMISC (u32)(1 << 9) +#define IXLV_FLAG_AQ_GET_STATS (u32)(1 << 10) + +/* printf %b arg */ +#define IXLV_FLAGS \ + "\20\1ENABLE_QUEUES\2DISABLE_QUEUES\3ADD_MAC_FILTER" \ + "\4ADD_VLAN_FILTER\5DEL_MAC_FILTER\6DEL_VLAN_FILTER" \ + "\7CONFIGURE_QUEUES\10MAP_VECTORS\11HANDLE_RESET" \ + "\12CONFIGURE_PROMISC\13GET_STATS" + +/* Hack for compatibility with 1.0.x linux pf driver */ +#define I40E_VIRTCHNL_OP_EVENT 17 + +/* Driver state */ +enum ixlv_state_t { + IXLV_START, + IXLV_FAILED, + IXLV_RESET_REQUIRED, + IXLV_RESET_PENDING, + IXLV_VERSION_CHECK, + IXLV_GET_RESOURCES, + IXLV_INIT_READY, + IXLV_INIT_START, + IXLV_INIT_CONFIG, + IXLV_INIT_MAPPING, + IXLV_INIT_ENABLE, + IXLV_INIT_COMPLETE, + IXLV_RUNNING, +}; + +struct ixlv_mac_filter { + SLIST_ENTRY(ixlv_mac_filter) next; + u8 macaddr[ETHER_ADDR_LEN]; + u16 flags; +}; +SLIST_HEAD(mac_list, ixlv_mac_filter); + +struct ixlv_vlan_filter { + SLIST_ENTRY(ixlv_vlan_filter) next; + u16 vlan; + u16 flags; +}; +SLIST_HEAD(vlan_list, ixlv_vlan_filter); + +/* Software controller structure */ +struct ixlv_sc { + struct i40e_hw hw; + struct i40e_osdep osdep; + struct device *dev; + + struct resource *pci_mem; + struct resource *msix_mem; + + enum ixlv_state_t init_state; + + /* + * Interrupt resources + */ + void *tag; + struct resource *res; /* For the AQ */ + + struct ifmedia media; + struct callout timer; + int msix; + int pf_version; + int if_flags; + + bool link_up; + u32 link_speed; + + struct mtx mtx; + + u32 qbase; + u32 admvec; + struct timeout_task timeout; + struct task aq_irq; + struct task aq_sched; + struct taskqueue *tq; + + struct ixl_vsi vsi; + + /* Filter lists */ + struct mac_list *mac_filters; + struct vlan_list *vlan_filters; + + /* Promiscuous mode */ + u32 promiscuous_flags; + + /* Admin queue task flags */ + u32 aq_wait_count; + + struct ixl_vc_mgr vc_mgr; + struct ixl_vc_cmd add_mac_cmd; + struct ixl_vc_cmd del_mac_cmd; + struct ixl_vc_cmd config_queues_cmd; + struct ixl_vc_cmd map_vectors_cmd; + struct ixl_vc_cmd enable_queues_cmd; + struct ixl_vc_cmd add_vlan_cmd; + struct ixl_vc_cmd del_vlan_cmd; + struct ixl_vc_cmd add_multi_cmd; + struct ixl_vc_cmd del_multi_cmd; + + /* Virtual comm channel */ + struct i40e_virtchnl_vf_resource *vf_res; + struct i40e_virtchnl_vsi_resource *vsi_res; + + /* Misc stats maintained by the driver */ + u64 watchdog_events; + u64 admin_irq; + + u8 aq_buffer[IXL_AQ_BUF_SZ]; +}; + +#define IXLV_CORE_LOCK_ASSERT(sc) mtx_assert(&(sc)->mtx, MA_OWNED) +/* +** This checks for a zero mac addr, something that will be likely +** unless the Admin on the Host has created one. +*/ +static inline bool +ixlv_check_ether_addr(u8 *addr) +{ + bool status = TRUE; + + if ((addr[0] == 0 && addr[1]== 0 && addr[2] == 0 && + addr[3] == 0 && addr[4]== 0 && addr[5] == 0)) + status = FALSE; + return (status); +} + +/* +** VF Common function prototypes +*/ +int ixlv_send_api_ver(struct ixlv_sc *); +int ixlv_verify_api_ver(struct ixlv_sc *); +int ixlv_send_vf_config_msg(struct ixlv_sc *); +int ixlv_get_vf_config(struct ixlv_sc *); +void ixlv_init(void *); +int ixlv_reinit_locked(struct ixlv_sc *); +void ixlv_configure_queues(struct ixlv_sc *); +void ixlv_enable_queues(struct ixlv_sc *); +void ixlv_disable_queues(struct ixlv_sc *); +void ixlv_map_queues(struct ixlv_sc *); +void ixlv_enable_intr(struct ixl_vsi *); +void ixlv_disable_intr(struct ixl_vsi *); +void ixlv_add_ether_filters(struct ixlv_sc *); +void ixlv_del_ether_filters(struct ixlv_sc *); +void ixlv_request_stats(struct ixlv_sc *); +void ixlv_request_reset(struct ixlv_sc *); +void ixlv_vc_completion(struct ixlv_sc *, + enum i40e_virtchnl_ops, i40e_status, u8 *, u16); +void ixlv_add_ether_filter(struct ixlv_sc *); +void ixlv_add_vlans(struct ixlv_sc *); +void ixlv_del_vlans(struct ixlv_sc *); +void ixlv_update_stats_counters(struct ixlv_sc *, + struct i40e_eth_stats *); +void ixlv_update_link_status(struct ixlv_sc *); + +#endif /* _IXLV_H_ */ Index: sys/dev/ixl/legacy_ixlvc.c =================================================================== --- /dev/null +++ sys/dev/ixl/legacy_ixlvc.c @@ -0,0 +1,1133 @@ +/****************************************************************************** + + Copyright (c) 2013-2015, Intel Corporation + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of the Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +******************************************************************************/ +/*$FreeBSD$*/ + +/* +** Virtual Channel support +** These are support functions to communication +** between the VF and PF drivers. +*/ + +#include "ixl.h" +#include "ixlv.h" +#include "i40e_prototype.h" + + +/* busy wait delay in msec */ +#define IXLV_BUSY_WAIT_DELAY 10 +#define IXLV_BUSY_WAIT_COUNT 50 + +static void ixl_vc_process_resp(struct ixl_vc_mgr *, uint32_t, + enum i40e_status_code); +static void ixl_vc_process_next(struct ixl_vc_mgr *mgr); +static void ixl_vc_schedule_retry(struct ixl_vc_mgr *mgr); +static void ixl_vc_send_current(struct ixl_vc_mgr *mgr); + +#ifdef IXL_DEBUG +/* +** Validate VF messages +*/ +static int ixl_vc_validate_vf_msg(struct ixlv_sc *sc, u32 v_opcode, + u8 *msg, u16 msglen) +{ + bool err_msg_format = false; + int valid_len; + + /* Validate message length. */ + switch (v_opcode) { + case I40E_VIRTCHNL_OP_VERSION: + valid_len = sizeof(struct i40e_virtchnl_version_info); + break; + case I40E_VIRTCHNL_OP_RESET_VF: + case I40E_VIRTCHNL_OP_GET_VF_RESOURCES: + valid_len = 0; + break; + case I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE: + valid_len = sizeof(struct i40e_virtchnl_txq_info); + break; + case I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE: + valid_len = sizeof(struct i40e_virtchnl_rxq_info); + break; + case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES: + valid_len = sizeof(struct i40e_virtchnl_vsi_queue_config_info); + if (msglen >= valid_len) { + struct i40e_virtchnl_vsi_queue_config_info *vqc = + (struct i40e_virtchnl_vsi_queue_config_info *)msg; + valid_len += (vqc->num_queue_pairs * + sizeof(struct + i40e_virtchnl_queue_pair_info)); + if (vqc->num_queue_pairs == 0) + err_msg_format = true; + } + break; + case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP: + valid_len = sizeof(struct i40e_virtchnl_irq_map_info); + if (msglen >= valid_len) { + struct i40e_virtchnl_irq_map_info *vimi = + (struct i40e_virtchnl_irq_map_info *)msg; + valid_len += (vimi->num_vectors * + sizeof(struct i40e_virtchnl_vector_map)); + if (vimi->num_vectors == 0) + err_msg_format = true; + } + break; + case I40E_VIRTCHNL_OP_ENABLE_QUEUES: + case I40E_VIRTCHNL_OP_DISABLE_QUEUES: + valid_len = sizeof(struct i40e_virtchnl_queue_select); + break; + case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS: + case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS: + valid_len = sizeof(struct i40e_virtchnl_ether_addr_list); + if (msglen >= valid_len) { + struct i40e_virtchnl_ether_addr_list *veal = + (struct i40e_virtchnl_ether_addr_list *)msg; + valid_len += veal->num_elements * + sizeof(struct i40e_virtchnl_ether_addr); + if (veal->num_elements == 0) + err_msg_format = true; + } + break; + case I40E_VIRTCHNL_OP_ADD_VLAN: + case I40E_VIRTCHNL_OP_DEL_VLAN: + valid_len = sizeof(struct i40e_virtchnl_vlan_filter_list); + if (msglen >= valid_len) { + struct i40e_virtchnl_vlan_filter_list *vfl = + (struct i40e_virtchnl_vlan_filter_list *)msg; + valid_len += vfl->num_elements * sizeof(u16); + if (vfl->num_elements == 0) + err_msg_format = true; + } + break; + case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE: + valid_len = sizeof(struct i40e_virtchnl_promisc_info); + break; + case I40E_VIRTCHNL_OP_GET_STATS: + valid_len = sizeof(struct i40e_virtchnl_queue_select); + break; + /* These are always errors coming from the VF. */ + case I40E_VIRTCHNL_OP_EVENT: + case I40E_VIRTCHNL_OP_UNKNOWN: + default: + return EPERM; + break; + } + /* few more checks */ + if ((valid_len != msglen) || (err_msg_format)) + return EINVAL; + else + return 0; +} +#endif + +/* +** ixlv_send_pf_msg +** +** Send message to PF and print status if failure. +*/ +static int +ixlv_send_pf_msg(struct ixlv_sc *sc, + enum i40e_virtchnl_ops op, u8 *msg, u16 len) +{ + struct i40e_hw *hw = &sc->hw; + device_t dev = sc->dev; + i40e_status err; + +#ifdef IXL_DEBUG + /* + ** Pre-validating messages to the PF + */ + int val_err; + val_err = ixl_vc_validate_vf_msg(sc, op, msg, len); + if (val_err) + device_printf(dev, "Error validating msg to PF for op %d," + " msglen %d: error %d\n", op, len, val_err); +#endif + + err = i40e_aq_send_msg_to_pf(hw, op, I40E_SUCCESS, msg, len, NULL); + if (err) + device_printf(dev, "Unable to send opcode %d to PF, " + "error %d, aq status %d\n", op, err, hw->aq.asq_last_status); + return err; +} + + +/* +** ixlv_send_api_ver +** +** Send API version admin queue message to the PF. The reply is not checked +** in this function. Returns 0 if the message was successfully +** sent, or one of the I40E_ADMIN_QUEUE_ERROR_ statuses if not. +*/ +int +ixlv_send_api_ver(struct ixlv_sc *sc) +{ + struct i40e_virtchnl_version_info vvi; + + vvi.major = I40E_VIRTCHNL_VERSION_MAJOR; + vvi.minor = I40E_VIRTCHNL_VERSION_MINOR; + + return ixlv_send_pf_msg(sc, I40E_VIRTCHNL_OP_VERSION, + (u8 *)&vvi, sizeof(vvi)); +} + +/* +** ixlv_verify_api_ver +** +** Compare API versions with the PF. Must be called after admin queue is +** initialized. Returns 0 if API versions match, EIO if +** they do not, or I40E_ERR_ADMIN_QUEUE_NO_WORK if the admin queue is empty. +*/ +int +ixlv_verify_api_ver(struct ixlv_sc *sc) +{ + struct i40e_virtchnl_version_info *pf_vvi; + struct i40e_hw *hw = &sc->hw; + struct i40e_arq_event_info event; + i40e_status err; + int retries = 0; + + event.buf_len = IXL_AQ_BUFSZ; + event.msg_buf = malloc(event.buf_len, M_DEVBUF, M_NOWAIT); + if (!event.msg_buf) { + err = ENOMEM; + goto out; + } + + do { + if (++retries > IXLV_AQ_MAX_ERR) + goto out_alloc; + + /* NOTE: initial delay is necessary */ + i40e_msec_delay(100); + err = i40e_clean_arq_element(hw, &event, NULL); + } while (err == I40E_ERR_ADMIN_QUEUE_NO_WORK); + if (err) + goto out_alloc; + + err = (i40e_status)le32toh(event.desc.cookie_low); + if (err) { + err = EIO; + goto out_alloc; + } + + if ((enum i40e_virtchnl_ops)le32toh(event.desc.cookie_high) != + I40E_VIRTCHNL_OP_VERSION) { + DDPRINTF(sc->dev, "Received unexpected op response: %d\n", + le32toh(event.desc.cookie_high)); + err = EIO; + goto out_alloc; + } + + pf_vvi = (struct i40e_virtchnl_version_info *)event.msg_buf; + if ((pf_vvi->major > I40E_VIRTCHNL_VERSION_MAJOR) || + ((pf_vvi->major == I40E_VIRTCHNL_VERSION_MAJOR) && + (pf_vvi->minor > I40E_VIRTCHNL_VERSION_MINOR))) + err = EIO; + else + sc->pf_version = pf_vvi->minor; + +out_alloc: + free(event.msg_buf, M_DEVBUF); +out: + return err; +} + +/* +** ixlv_send_vf_config_msg +** +** Send VF configuration request admin queue message to the PF. The reply +** is not checked in this function. Returns 0 if the message was +** successfully sent, or one of the I40E_ADMIN_QUEUE_ERROR_ statuses if not. +*/ +int +ixlv_send_vf_config_msg(struct ixlv_sc *sc) +{ + u32 caps; + + caps = I40E_VIRTCHNL_VF_OFFLOAD_L2 | + I40E_VIRTCHNL_VF_OFFLOAD_RSS_AQ | + I40E_VIRTCHNL_VF_OFFLOAD_RSS_REG | + I40E_VIRTCHNL_VF_OFFLOAD_VLAN; + + if (sc->pf_version) + return ixlv_send_pf_msg(sc, I40E_VIRTCHNL_OP_GET_VF_RESOURCES, + (u8 *)&caps, sizeof(caps)); + else + return ixlv_send_pf_msg(sc, I40E_VIRTCHNL_OP_GET_VF_RESOURCES, + NULL, 0); +} + +/* +** ixlv_get_vf_config +** +** Get VF configuration from PF and populate hw structure. Must be called after +** admin queue is initialized. Busy waits until response is received from PF, +** with maximum timeout. Response from PF is returned in the buffer for further +** processing by the caller. +*/ +int +ixlv_get_vf_config(struct ixlv_sc *sc) +{ + struct i40e_hw *hw = &sc->hw; + device_t dev = sc->dev; + struct i40e_arq_event_info event; + u16 len; + i40e_status err = 0; + u32 retries = 0; + + /* Note this assumes a single VSI */ + len = sizeof(struct i40e_virtchnl_vf_resource) + + sizeof(struct i40e_virtchnl_vsi_resource); + event.buf_len = len; + event.msg_buf = malloc(event.buf_len, M_DEVBUF, M_NOWAIT); + if (!event.msg_buf) { + err = ENOMEM; + goto out; + } + + for (;;) { + err = i40e_clean_arq_element(hw, &event, NULL); + if (err == I40E_ERR_ADMIN_QUEUE_NO_WORK) { + if (++retries <= IXLV_AQ_MAX_ERR) + i40e_msec_delay(10); + } else if ((enum i40e_virtchnl_ops)le32toh(event.desc.cookie_high) != + I40E_VIRTCHNL_OP_GET_VF_RESOURCES) { + DDPRINTF(dev, "Received a response from PF," + " opcode %d, error %d", + le32toh(event.desc.cookie_high), + le32toh(event.desc.cookie_low)); + retries++; + continue; + } else { + err = (i40e_status)le32toh(event.desc.cookie_low); + if (err) { + device_printf(dev, "%s: Error returned from PF," + " opcode %d, error %d\n", __func__, + le32toh(event.desc.cookie_high), + le32toh(event.desc.cookie_low)); + err = EIO; + goto out_alloc; + } + /* We retrieved the config message, with no errors */ + break; + } + + if (retries > IXLV_AQ_MAX_ERR) { + INIT_DBG_DEV(dev, "Did not receive response after %d tries.", + retries); + err = ETIMEDOUT; + goto out_alloc; + } + } + + memcpy(sc->vf_res, event.msg_buf, min(event.msg_len, len)); + i40e_vf_parse_hw_config(hw, sc->vf_res); + +out_alloc: + free(event.msg_buf, M_DEVBUF); +out: + return err; +} + +/* +** ixlv_configure_queues +** +** Request that the PF set up our queues. +*/ +void +ixlv_configure_queues(struct ixlv_sc *sc) +{ + device_t dev = sc->dev; + struct ixl_vsi *vsi = &sc->vsi; + struct ixl_queue *que = vsi->queues; + struct tx_ring *txr; + struct rx_ring *rxr; + int len, pairs; + + struct i40e_virtchnl_vsi_queue_config_info *vqci; + struct i40e_virtchnl_queue_pair_info *vqpi; + + pairs = vsi->num_queues; + len = sizeof(struct i40e_virtchnl_vsi_queue_config_info) + + (sizeof(struct i40e_virtchnl_queue_pair_info) * pairs); + vqci = malloc(len, M_DEVBUF, M_NOWAIT | M_ZERO); + if (!vqci) { + device_printf(dev, "%s: unable to allocate memory\n", __func__); + ixl_vc_schedule_retry(&sc->vc_mgr); + return; + } + vqci->vsi_id = sc->vsi_res->vsi_id; + vqci->num_queue_pairs = pairs; + vqpi = vqci->qpair; + /* Size check is not needed here - HW max is 16 queue pairs, and we + * can fit info for 31 of them into the AQ buffer before it overflows. + */ + for (int i = 0; i < pairs; i++, que++, vqpi++) { + txr = &que->txr; + rxr = &que->rxr; + vqpi->txq.vsi_id = vqci->vsi_id; + vqpi->txq.queue_id = i; + vqpi->txq.ring_len = que->num_desc; + vqpi->txq.dma_ring_addr = txr->dma.pa; + /* Enable Head writeback */ + vqpi->txq.headwb_enabled = 1; + vqpi->txq.dma_headwb_addr = txr->dma.pa + + (que->num_desc * sizeof(struct i40e_tx_desc)); + + vqpi->rxq.vsi_id = vqci->vsi_id; + vqpi->rxq.queue_id = i; + vqpi->rxq.ring_len = que->num_desc; + vqpi->rxq.dma_ring_addr = rxr->dma.pa; + vqpi->rxq.max_pkt_size = vsi->max_frame_size; + vqpi->rxq.databuffer_size = rxr->mbuf_sz; + vqpi->rxq.splithdr_enabled = 0; + } + + ixlv_send_pf_msg(sc, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, + (u8 *)vqci, len); + free(vqci, M_DEVBUF); +} + +/* +** ixlv_enable_queues +** +** Request that the PF enable all of our queues. +*/ +void +ixlv_enable_queues(struct ixlv_sc *sc) +{ + struct i40e_virtchnl_queue_select vqs; + + vqs.vsi_id = sc->vsi_res->vsi_id; + vqs.tx_queues = (1 << sc->vsi_res->num_queue_pairs) - 1; + vqs.rx_queues = vqs.tx_queues; + ixlv_send_pf_msg(sc, I40E_VIRTCHNL_OP_ENABLE_QUEUES, + (u8 *)&vqs, sizeof(vqs)); +} + +/* +** ixlv_disable_queues +** +** Request that the PF disable all of our queues. +*/ +void +ixlv_disable_queues(struct ixlv_sc *sc) +{ + struct i40e_virtchnl_queue_select vqs; + + vqs.vsi_id = sc->vsi_res->vsi_id; + vqs.tx_queues = (1 << sc->vsi_res->num_queue_pairs) - 1; + vqs.rx_queues = vqs.tx_queues; + ixlv_send_pf_msg(sc, I40E_VIRTCHNL_OP_DISABLE_QUEUES, + (u8 *)&vqs, sizeof(vqs)); +} + +/* +** ixlv_map_queues +** +** Request that the PF map queues to interrupt vectors. Misc causes, including +** admin queue, are always mapped to vector 0. +*/ +void +ixlv_map_queues(struct ixlv_sc *sc) +{ + struct i40e_virtchnl_irq_map_info *vm; + int i, q, len; + struct ixl_vsi *vsi = &sc->vsi; + struct ixl_queue *que = vsi->queues; + + /* How many queue vectors, adminq uses one */ + q = sc->msix - 1; + + len = sizeof(struct i40e_virtchnl_irq_map_info) + + (sc->msix * sizeof(struct i40e_virtchnl_vector_map)); + vm = malloc(len, M_DEVBUF, M_NOWAIT); + if (!vm) { + printf("%s: unable to allocate memory\n", __func__); + ixl_vc_schedule_retry(&sc->vc_mgr); + return; + } + + vm->num_vectors = sc->msix; + /* Queue vectors first */ + for (i = 0; i < q; i++, que++) { + vm->vecmap[i].vsi_id = sc->vsi_res->vsi_id; + vm->vecmap[i].vector_id = i + 1; /* first is adminq */ + vm->vecmap[i].txq_map = (1 << que->me); + vm->vecmap[i].rxq_map = (1 << que->me); + vm->vecmap[i].rxitr_idx = 0; + vm->vecmap[i].txitr_idx = 0; + } + + /* Misc vector last - this is only for AdminQ messages */ + vm->vecmap[i].vsi_id = sc->vsi_res->vsi_id; + vm->vecmap[i].vector_id = 0; + vm->vecmap[i].txq_map = 0; + vm->vecmap[i].rxq_map = 0; + vm->vecmap[i].rxitr_idx = 0; + vm->vecmap[i].txitr_idx = 0; + + ixlv_send_pf_msg(sc, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, + (u8 *)vm, len); + free(vm, M_DEVBUF); +} + +/* +** Scan the Filter List looking for vlans that need +** to be added, then create the data to hand to the AQ +** for handling. +*/ +void +ixlv_add_vlans(struct ixlv_sc *sc) +{ + struct i40e_virtchnl_vlan_filter_list *v; + struct ixlv_vlan_filter *f, *ftmp; + device_t dev = sc->dev; + int len, i = 0, cnt = 0; + + /* Get count of VLAN filters to add */ + SLIST_FOREACH(f, sc->vlan_filters, next) { + if (f->flags & IXL_FILTER_ADD) + cnt++; + } + + if (!cnt) { /* no work... */ + ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_ADD_VLAN_FILTER, + I40E_SUCCESS); + return; + } + + len = sizeof(struct i40e_virtchnl_vlan_filter_list) + + (cnt * sizeof(u16)); + + if (len > IXL_AQ_BUF_SZ) { + device_printf(dev, "%s: Exceeded Max AQ Buf size\n", + __func__); + ixl_vc_schedule_retry(&sc->vc_mgr); + return; + } + + v = malloc(len, M_DEVBUF, M_NOWAIT); + if (!v) { + device_printf(dev, "%s: unable to allocate memory\n", + __func__); + ixl_vc_schedule_retry(&sc->vc_mgr); + return; + } + + v->vsi_id = sc->vsi_res->vsi_id; + v->num_elements = cnt; + + /* Scan the filter array */ + SLIST_FOREACH_SAFE(f, sc->vlan_filters, next, ftmp) { + if (f->flags & IXL_FILTER_ADD) { + bcopy(&f->vlan, &v->vlan_id[i], sizeof(u16)); + f->flags = IXL_FILTER_USED; + i++; + } + if (i == cnt) + break; + } + // ERJ: Should this be taken out? + if (i == 0) { /* Should not happen... */ + device_printf(dev, "%s: i == 0?\n", __func__); + ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_ADD_VLAN_FILTER, + I40E_SUCCESS); + return; + } + + ixlv_send_pf_msg(sc, I40E_VIRTCHNL_OP_ADD_VLAN, (u8 *)v, len); + free(v, M_DEVBUF); + /* add stats? */ +} + +/* +** Scan the Filter Table looking for vlans that need +** to be removed, then create the data to hand to the AQ +** for handling. +*/ +void +ixlv_del_vlans(struct ixlv_sc *sc) +{ + device_t dev = sc->dev; + struct i40e_virtchnl_vlan_filter_list *v; + struct ixlv_vlan_filter *f, *ftmp; + int len, i = 0, cnt = 0; + + /* Get count of VLAN filters to delete */ + SLIST_FOREACH(f, sc->vlan_filters, next) { + if (f->flags & IXL_FILTER_DEL) + cnt++; + } + + if (!cnt) { /* no work... */ + ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_DEL_VLAN_FILTER, + I40E_SUCCESS); + return; + } + + len = sizeof(struct i40e_virtchnl_vlan_filter_list) + + (cnt * sizeof(u16)); + + if (len > IXL_AQ_BUF_SZ) { + device_printf(dev, "%s: Exceeded Max AQ Buf size\n", + __func__); + ixl_vc_schedule_retry(&sc->vc_mgr); + return; + } + + v = malloc(len, M_DEVBUF, M_NOWAIT | M_ZERO); + if (!v) { + device_printf(dev, "%s: unable to allocate memory\n", + __func__); + ixl_vc_schedule_retry(&sc->vc_mgr); + return; + } + + v->vsi_id = sc->vsi_res->vsi_id; + v->num_elements = cnt; + + /* Scan the filter array */ + SLIST_FOREACH_SAFE(f, sc->vlan_filters, next, ftmp) { + if (f->flags & IXL_FILTER_DEL) { + bcopy(&f->vlan, &v->vlan_id[i], sizeof(u16)); + i++; + SLIST_REMOVE(sc->vlan_filters, f, ixlv_vlan_filter, next); + free(f, M_DEVBUF); + } + if (i == cnt) + break; + } + // ERJ: Take this out? + if (i == 0) { /* Should not happen... */ + device_printf(dev, "%s: i == 0?\n", __func__); + ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_DEL_VLAN_FILTER, + I40E_SUCCESS); + return; + } + + ixlv_send_pf_msg(sc, I40E_VIRTCHNL_OP_DEL_VLAN, (u8 *)v, len); + free(v, M_DEVBUF); + /* add stats? */ +} + + +/* +** This routine takes additions to the vsi filter +** table and creates an Admin Queue call to create +** the filters in the hardware. +*/ +void +ixlv_add_ether_filters(struct ixlv_sc *sc) +{ + struct i40e_virtchnl_ether_addr_list *a; + struct ixlv_mac_filter *f; + device_t dev = sc->dev; + int len, j = 0, cnt = 0; + + /* Get count of MAC addresses to add */ + SLIST_FOREACH(f, sc->mac_filters, next) { + if (f->flags & IXL_FILTER_ADD) + cnt++; + } + if (cnt == 0) { /* Should not happen... */ + DDPRINTF(dev, "cnt == 0, exiting..."); + ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_ADD_MAC_FILTER, + I40E_SUCCESS); + return; + } + + len = sizeof(struct i40e_virtchnl_ether_addr_list) + + (cnt * sizeof(struct i40e_virtchnl_ether_addr)); + + a = malloc(len, M_DEVBUF, M_NOWAIT | M_ZERO); + if (a == NULL) { + device_printf(dev, "%s: Failed to get memory for " + "virtchnl_ether_addr_list\n", __func__); + ixl_vc_schedule_retry(&sc->vc_mgr); + return; + } + a->vsi_id = sc->vsi.id; + a->num_elements = cnt; + + /* Scan the filter array */ + SLIST_FOREACH(f, sc->mac_filters, next) { + if (f->flags & IXL_FILTER_ADD) { + bcopy(f->macaddr, a->list[j].addr, ETHER_ADDR_LEN); + f->flags &= ~IXL_FILTER_ADD; + j++; + + DDPRINTF(dev, "ADD: " MAC_FORMAT, + MAC_FORMAT_ARGS(f->macaddr)); + } + if (j == cnt) + break; + } + DDPRINTF(dev, "len %d, j %d, cnt %d", + len, j, cnt); + ixlv_send_pf_msg(sc, + I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, (u8 *)a, len); + /* add stats? */ + free(a, M_DEVBUF); + return; +} + +/* +** This routine takes filters flagged for deletion in the +** sc MAC filter list and creates an Admin Queue call +** to delete those filters in the hardware. +*/ +void +ixlv_del_ether_filters(struct ixlv_sc *sc) +{ + struct i40e_virtchnl_ether_addr_list *d; + device_t dev = sc->dev; + struct ixlv_mac_filter *f, *f_temp; + int len, j = 0, cnt = 0; + + /* Get count of MAC addresses to delete */ + SLIST_FOREACH(f, sc->mac_filters, next) { + if (f->flags & IXL_FILTER_DEL) + cnt++; + } + if (cnt == 0) { + DDPRINTF(dev, "cnt == 0, exiting..."); + ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_DEL_MAC_FILTER, + I40E_SUCCESS); + return; + } + + len = sizeof(struct i40e_virtchnl_ether_addr_list) + + (cnt * sizeof(struct i40e_virtchnl_ether_addr)); + + d = malloc(len, M_DEVBUF, M_NOWAIT | M_ZERO); + if (d == NULL) { + device_printf(dev, "%s: Failed to get memory for " + "virtchnl_ether_addr_list\n", __func__); + ixl_vc_schedule_retry(&sc->vc_mgr); + return; + } + d->vsi_id = sc->vsi.id; + d->num_elements = cnt; + + /* Scan the filter array */ + SLIST_FOREACH_SAFE(f, sc->mac_filters, next, f_temp) { + if (f->flags & IXL_FILTER_DEL) { + bcopy(f->macaddr, d->list[j].addr, ETHER_ADDR_LEN); + DDPRINTF(dev, "DEL: " MAC_FORMAT, + MAC_FORMAT_ARGS(f->macaddr)); + j++; + SLIST_REMOVE(sc->mac_filters, f, ixlv_mac_filter, next); + free(f, M_DEVBUF); + } + if (j == cnt) + break; + } + ixlv_send_pf_msg(sc, + I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS, (u8 *)d, len); + /* add stats? */ + free(d, M_DEVBUF); + return; +} + +/* +** ixlv_request_reset +** Request that the PF reset this VF. No response is expected. +*/ +void +ixlv_request_reset(struct ixlv_sc *sc) +{ + /* + ** Set the reset status to "in progress" before + ** the request, this avoids any possibility of + ** a mistaken early detection of completion. + */ + wr32(&sc->hw, I40E_VFGEN_RSTAT, I40E_VFR_INPROGRESS); + ixlv_send_pf_msg(sc, I40E_VIRTCHNL_OP_RESET_VF, NULL, 0); +} + +/* +** ixlv_request_stats +** Request the statistics for this VF's VSI from PF. +*/ +void +ixlv_request_stats(struct ixlv_sc *sc) +{ + struct i40e_virtchnl_queue_select vqs; + int error = 0; + + vqs.vsi_id = sc->vsi_res->vsi_id; + /* Low priority, we don't need to error check */ + error = ixlv_send_pf_msg(sc, I40E_VIRTCHNL_OP_GET_STATS, + (u8 *)&vqs, sizeof(vqs)); +#ifdef IXL_DEBUG + if (error) + device_printf(sc->dev, "Error sending stats request to PF: %d\n", error); +#endif +} + +/* +** Updates driver's stats counters with VSI stats returned from PF. +*/ +void +ixlv_update_stats_counters(struct ixlv_sc *sc, struct i40e_eth_stats *es) +{ + struct ixl_vsi *vsi = &sc->vsi; + uint64_t tx_discards; + + tx_discards = es->tx_discards; + for (int i = 0; i < vsi->num_queues; i++) + tx_discards += sc->vsi.queues[i].txr.br->br_drops; + + /* Update ifnet stats */ + IXL_SET_IPACKETS(vsi, es->rx_unicast + + es->rx_multicast + + es->rx_broadcast); + IXL_SET_OPACKETS(vsi, es->tx_unicast + + es->tx_multicast + + es->tx_broadcast); + IXL_SET_IBYTES(vsi, es->rx_bytes); + IXL_SET_OBYTES(vsi, es->tx_bytes); + IXL_SET_IMCASTS(vsi, es->rx_multicast); + IXL_SET_OMCASTS(vsi, es->tx_multicast); + + IXL_SET_OERRORS(vsi, es->tx_errors); + IXL_SET_IQDROPS(vsi, es->rx_discards); + IXL_SET_OQDROPS(vsi, tx_discards); + IXL_SET_NOPROTO(vsi, es->rx_unknown_protocol); + IXL_SET_COLLISIONS(vsi, 0); + + vsi->eth_stats = *es; +} + +/* +** ixlv_vc_completion +** +** Asynchronous completion function for admin queue messages. Rather than busy +** wait, we fire off our requests and assume that no errors will be returned. +** This function handles the reply messages. +*/ +void +ixlv_vc_completion(struct ixlv_sc *sc, + enum i40e_virtchnl_ops v_opcode, + i40e_status v_retval, u8 *msg, u16 msglen) +{ + device_t dev = sc->dev; + struct ixl_vsi *vsi = &sc->vsi; + + if (v_opcode == I40E_VIRTCHNL_OP_EVENT) { + struct i40e_virtchnl_pf_event *vpe = + (struct i40e_virtchnl_pf_event *)msg; + + switch (vpe->event) { + case I40E_VIRTCHNL_EVENT_LINK_CHANGE: +#ifdef IXL_DEBUG + device_printf(dev, "Link change: status %d, speed %d\n", + vpe->event_data.link_event.link_status, + vpe->event_data.link_event.link_speed); +#endif + sc->link_up = + vpe->event_data.link_event.link_status; + sc->link_speed = + vpe->event_data.link_event.link_speed; + ixlv_update_link_status(sc); + break; + case I40E_VIRTCHNL_EVENT_RESET_IMPENDING: + device_printf(dev, "PF initiated reset!\n"); + sc->init_state = IXLV_RESET_PENDING; + ixlv_init(sc); + break; + default: + device_printf(dev, "%s: Unknown event %d from AQ\n", + __func__, vpe->event); + break; + } + + return; + } + + /* Catch-all error response */ + if (v_retval) { + device_printf(dev, + "%s: AQ returned error %d to our request %d!\n", + __func__, v_retval, v_opcode); + } + +#ifdef IXL_DEBUG + if (v_opcode != I40E_VIRTCHNL_OP_GET_STATS) + DDPRINTF(dev, "opcode %d", v_opcode); +#endif + + switch (v_opcode) { + case I40E_VIRTCHNL_OP_GET_STATS: + ixlv_update_stats_counters(sc, (struct i40e_eth_stats *)msg); + break; + case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS: + ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_ADD_MAC_FILTER, + v_retval); + if (v_retval) { + device_printf(dev, "WARNING: Error adding VF mac filter!\n"); + device_printf(dev, "WARNING: Device may not receive traffic!\n"); + } + break; + case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS: + ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_DEL_MAC_FILTER, + v_retval); + break; + case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE: + ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_CONFIGURE_PROMISC, + v_retval); + break; + case I40E_VIRTCHNL_OP_ADD_VLAN: + ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_ADD_VLAN_FILTER, + v_retval); + break; + case I40E_VIRTCHNL_OP_DEL_VLAN: + ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_DEL_VLAN_FILTER, + v_retval); + break; + case I40E_VIRTCHNL_OP_ENABLE_QUEUES: + ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_ENABLE_QUEUES, + v_retval); + if (v_retval == 0) { + /* Update link status */ + ixlv_update_link_status(sc); + /* Turn on all interrupts */ + ixlv_enable_intr(vsi); + /* And inform the stack we're ready */ + vsi->ifp->if_drv_flags |= IFF_DRV_RUNNING; + vsi->ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; + } + break; + case I40E_VIRTCHNL_OP_DISABLE_QUEUES: + ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_DISABLE_QUEUES, + v_retval); + if (v_retval == 0) { + /* Turn off all interrupts */ + ixlv_disable_intr(vsi); + /* Tell the stack that the interface is no longer active */ + vsi->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); + } + break; + case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES: + ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_CONFIGURE_QUEUES, + v_retval); + break; + case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP: + ixl_vc_process_resp(&sc->vc_mgr, IXLV_FLAG_AQ_MAP_VECTORS, + v_retval); + break; + default: + device_printf(dev, + "%s: Received unexpected message %d from PF.\n", + __func__, v_opcode); + break; + } + return; +} + +static void +ixl_vc_send_cmd(struct ixlv_sc *sc, uint32_t request) +{ + + switch (request) { + case IXLV_FLAG_AQ_MAP_VECTORS: + ixlv_map_queues(sc); + break; + + case IXLV_FLAG_AQ_ADD_MAC_FILTER: + ixlv_add_ether_filters(sc); + break; + + case IXLV_FLAG_AQ_ADD_VLAN_FILTER: + ixlv_add_vlans(sc); + break; + + case IXLV_FLAG_AQ_DEL_MAC_FILTER: + ixlv_del_ether_filters(sc); + break; + + case IXLV_FLAG_AQ_DEL_VLAN_FILTER: + ixlv_del_vlans(sc); + break; + + case IXLV_FLAG_AQ_CONFIGURE_QUEUES: + ixlv_configure_queues(sc); + break; + + case IXLV_FLAG_AQ_DISABLE_QUEUES: + ixlv_disable_queues(sc); + break; + + case IXLV_FLAG_AQ_ENABLE_QUEUES: + ixlv_enable_queues(sc); + break; + } +} + +void +ixl_vc_init_mgr(struct ixlv_sc *sc, struct ixl_vc_mgr *mgr) +{ + mgr->sc = sc; + mgr->current = NULL; + TAILQ_INIT(&mgr->pending); + callout_init_mtx(&mgr->callout, &sc->mtx, 0); +} + +static void +ixl_vc_process_completion(struct ixl_vc_mgr *mgr, enum i40e_status_code err) +{ + struct ixl_vc_cmd *cmd; + + cmd = mgr->current; + mgr->current = NULL; + cmd->flags &= ~IXLV_VC_CMD_FLAG_BUSY; + + cmd->callback(cmd, cmd->arg, err); + ixl_vc_process_next(mgr); +} + +static void +ixl_vc_process_resp(struct ixl_vc_mgr *mgr, uint32_t request, + enum i40e_status_code err) +{ + struct ixl_vc_cmd *cmd; + + cmd = mgr->current; + if (cmd == NULL || cmd->request != request) + return; + + callout_stop(&mgr->callout); + ixl_vc_process_completion(mgr, err); +} + +static void +ixl_vc_cmd_timeout(void *arg) +{ + struct ixl_vc_mgr *mgr = (struct ixl_vc_mgr *)arg; + + IXLV_CORE_LOCK_ASSERT(mgr->sc); + ixl_vc_process_completion(mgr, I40E_ERR_TIMEOUT); +} + +static void +ixl_vc_cmd_retry(void *arg) +{ + struct ixl_vc_mgr *mgr = (struct ixl_vc_mgr *)arg; + + IXLV_CORE_LOCK_ASSERT(mgr->sc); + ixl_vc_send_current(mgr); +} + +static void +ixl_vc_send_current(struct ixl_vc_mgr *mgr) +{ + struct ixl_vc_cmd *cmd; + + cmd = mgr->current; + ixl_vc_send_cmd(mgr->sc, cmd->request); + callout_reset(&mgr->callout, IXLV_VC_TIMEOUT, ixl_vc_cmd_timeout, mgr); +} + +static void +ixl_vc_process_next(struct ixl_vc_mgr *mgr) +{ + struct ixl_vc_cmd *cmd; + + if (mgr->current != NULL) + return; + + if (TAILQ_EMPTY(&mgr->pending)) + return; + + cmd = TAILQ_FIRST(&mgr->pending); + TAILQ_REMOVE(&mgr->pending, cmd, next); + + mgr->current = cmd; + ixl_vc_send_current(mgr); +} + +static void +ixl_vc_schedule_retry(struct ixl_vc_mgr *mgr) +{ + + callout_reset(&mgr->callout, howmany(hz, 100), ixl_vc_cmd_retry, mgr); +} + +void +ixl_vc_enqueue(struct ixl_vc_mgr *mgr, struct ixl_vc_cmd *cmd, + uint32_t req, ixl_vc_callback_t *callback, void *arg) +{ + IXLV_CORE_LOCK_ASSERT(mgr->sc); + + if (cmd->flags & IXLV_VC_CMD_FLAG_BUSY) { + if (mgr->current == cmd) + mgr->current = NULL; + else + TAILQ_REMOVE(&mgr->pending, cmd, next); + } + + cmd->request = req; + cmd->callback = callback; + cmd->arg = arg; + cmd->flags |= IXLV_VC_CMD_FLAG_BUSY; + TAILQ_INSERT_TAIL(&mgr->pending, cmd, next); + + ixl_vc_process_next(mgr); +} + +void +ixl_vc_flush(struct ixl_vc_mgr *mgr) +{ + struct ixl_vc_cmd *cmd; + + IXLV_CORE_LOCK_ASSERT(mgr->sc); + KASSERT(TAILQ_EMPTY(&mgr->pending) || mgr->current != NULL, + ("ixlv: pending commands waiting but no command in progress")); + + cmd = mgr->current; + if (cmd != NULL) { + mgr->current = NULL; + cmd->flags &= ~IXLV_VC_CMD_FLAG_BUSY; + cmd->callback(cmd, cmd->arg, I40E_ERR_ADAPTER_STOPPED); + } + + while ((cmd = TAILQ_FIRST(&mgr->pending)) != NULL) { + TAILQ_REMOVE(&mgr->pending, cmd, next); + cmd->flags &= ~IXLV_VC_CMD_FLAG_BUSY; + cmd->callback(cmd, cmd->arg, I40E_ERR_ADAPTER_STOPPED); + } + + callout_stop(&mgr->callout); +} + Index: sys/modules/ixl/Makefile =================================================================== --- sys/modules/ixl/Makefile +++ sys/modules/ixl/Makefile @@ -3,13 +3,12 @@ .PATH: ${.CURDIR}/../../dev/ixl KMOD = if_ixl -SRCS = device_if.h bus_if.h pci_if.h pci_iov_if.h opt_bdg.h -SRCS += opt_inet.h opt_inet6.h opt_rss.h -SRCS += if_ixl.c ixl_txrx.c i40e_osdep.c +SRCS = device_if.h bus_if.h pci_if.h pci_iov_if.h ifdi_if.h +SRCS += opt_inet.h opt_inet6.h opt_rss.h opt_bdg.h opt_iflib.h +SRCS += if_ixl.c if_ixl_common.c ixl_txrx.c i40e_osdep.c # Shared source SRCS += i40e_common.c i40e_nvm.c i40e_adminq.c i40e_lan_hmc.c i40e_hmc.c - CFLAGS += -DSMP -DIXL_DEBUG_SYSCTL # Add Flow Director support Index: sys/modules/ixlv/Makefile =================================================================== --- sys/modules/ixlv/Makefile +++ sys/modules/ixlv/Makefile @@ -3,9 +3,9 @@ .PATH: ${.CURDIR}/../../dev/ixl KMOD = if_ixlv -SRCS = device_if.h bus_if.h pci_if.h pci_iov_if.h opt_bdg.h -SRCS += opt_inet.h opt_inet6.h opt_rss.h -SRCS += if_ixlv.c ixlvc.c ixl_txrx.c i40e_osdep.c +SRCS = device_if.h bus_if.h pci_if.h pci_iov_if.h ifdi_if.h +SRCS += opt_inet.h opt_inet6.h opt_rss.h opt_bdg.h opt_iflib.h +SRCS += if_ixlv.c ixlvc.c if_ixl_common.c ixl_txrx.c i40e_osdep.c # Shared source SRCS += i40e_common.c i40e_nvm.c i40e_adminq.c