Index: head/sys/dev/e1000/if_em.c
===================================================================
--- head/sys/dev/e1000/if_em.c	(revision 345304)
+++ head/sys/dev/e1000/if_em.c	(revision 345305)
@@ -1,4548 +1,4541 @@
 /*-
  * SPDX-License-Identifier: BSD-2-Clause
  *
  * Copyright (c) 2016 Nicole Graziano <nicole@nextbsd.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 
 /* $FreeBSD$ */
 #include "if_em.h"
 #include <sys/sbuf.h>
 #include <machine/_inttypes.h>
 
 #define em_mac_min e1000_82547
 #define igb_mac_min e1000_82575
 
 /*********************************************************************
  *  Driver version:
  *********************************************************************/
 char em_driver_version[] = "7.6.1-k";
 
 /*********************************************************************
  *  PCI Device ID Table
  *
  *  Used by probe to select devices to load on
  *  Last field stores an index into e1000_strings
  *  Last entry must be all 0s
  *
  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  *********************************************************************/
 
 static pci_vendor_info_t em_vendor_info_array[] =
 {
 	/* Intel(R) PRO/1000 Network Connection - Legacy em*/
 	PVID(0x8086, E1000_DEV_ID_82540EM, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82540EM_LOM, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82540EP, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82540EP_LOM, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82540EP_LP, "Intel(R) PRO/1000 Network Connection"),
 
 	PVID(0x8086, E1000_DEV_ID_82541EI, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82541ER, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82541ER_LOM, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82541EI_MOBILE, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82541GI, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82541GI_LF, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82541GI_MOBILE, "Intel(R) PRO/1000 Network Connection"),
 
 	PVID(0x8086, E1000_DEV_ID_82542, "Intel(R) PRO/1000 Network Connection"),
 
 	PVID(0x8086, E1000_DEV_ID_82543GC_FIBER, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82543GC_COPPER, "Intel(R) PRO/1000 Network Connection"),
 
 	PVID(0x8086, E1000_DEV_ID_82544EI_COPPER, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82544EI_FIBER, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82544GC_COPPER, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82544GC_LOM, "Intel(R) PRO/1000 Network Connection"),
 
 	PVID(0x8086, E1000_DEV_ID_82545EM_COPPER, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82545EM_FIBER, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82545GM_COPPER, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82545GM_FIBER, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82545GM_SERDES, "Intel(R) PRO/1000 Network Connection"),
 
 	PVID(0x8086, E1000_DEV_ID_82546EB_COPPER, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82546EB_FIBER, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82546EB_QUAD_COPPER, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82546GB_COPPER, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82546GB_FIBER, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82546GB_SERDES, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82546GB_PCIE, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3, "Intel(R) PRO/1000 Network Connection"),
 
 	PVID(0x8086, E1000_DEV_ID_82547EI, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82547EI_MOBILE, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82547GI, "Intel(R) PRO/1000 Network Connection"),
 
 	/* Intel(R) PRO/1000 Network Connection - em */
 	PVID(0x8086, E1000_DEV_ID_82571EB_COPPER, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82571EB_FIBER, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82571EB_SERDES, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82571EB_SERDES_DUAL, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82571EB_SERDES_QUAD, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER_LP, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82571EB_QUAD_FIBER, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82571PT_QUAD_COPPER, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82572EI, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82572EI_COPPER, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82572EI_FIBER, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82572EI_SERDES, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82573E, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82573E_IAMT, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82573L, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82583V, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_SPT, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_SPT, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_DPT, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_DPT, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_ICH8_IGP_M_AMT, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_ICH8_IGP_AMT, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_ICH8_IGP_C, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_ICH8_IFE, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_ICH8_IFE_GT, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_ICH8_IFE_G, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_ICH8_IGP_M, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_ICH8_82567V_3, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_ICH9_IGP_M_AMT, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_ICH9_IGP_AMT, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_ICH9_IGP_C, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_ICH9_IGP_M, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_ICH9_IGP_M_V, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_ICH9_IFE, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_ICH9_IFE_GT, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_ICH9_IFE_G, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_ICH9_BM, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82574L, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_82574LA, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_ICH10_R_BM_LM, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_ICH10_R_BM_LF, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_ICH10_R_BM_V, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_ICH10_D_BM_LM, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_ICH10_D_BM_LF, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_ICH10_D_BM_V, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_PCH_M_HV_LM, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_PCH_M_HV_LC, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_PCH_D_HV_DM, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_PCH_D_HV_DC, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_PCH2_LV_LM, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_PCH2_LV_V, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_PCH_LPT_I217_LM, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_PCH_LPT_I217_V, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_PCH_LPTLP_I218_LM, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_PCH_LPTLP_I218_V, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_PCH_I218_LM2, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_PCH_I218_V2, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_PCH_I218_LM3, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_PCH_I218_V3, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_LM, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_V, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_LM2, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_V2, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_PCH_LBG_I219_LM3, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_LM4, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_V4, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_LM5, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_V5, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_PCH_CNP_I219_LM6, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_PCH_CNP_I219_V6, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_PCH_CNP_I219_LM7, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_PCH_CNP_I219_V7, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_PCH_ICP_I219_LM8, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_PCH_ICP_I219_V8, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_PCH_ICP_I219_LM9, "Intel(R) PRO/1000 Network Connection"),
 	PVID(0x8086, E1000_DEV_ID_PCH_ICP_I219_V9, "Intel(R) PRO/1000 Network Connection"),
 	/* required last entry */
 	PVID_END
 };
 
 static pci_vendor_info_t igb_vendor_info_array[] =
 {
 	/* Intel(R) PRO/1000 Network Connection - igb */
 	PVID(0x8086, E1000_DEV_ID_82575EB_COPPER, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_82575EB_FIBER_SERDES, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_82575GB_QUAD_COPPER, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_82576, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_82576_NS, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_82576_NS_SERDES, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_82576_FIBER, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_82576_SERDES, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_82576_SERDES_QUAD, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_82576_QUAD_COPPER, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_82576_QUAD_COPPER_ET2, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_82576_VF, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_82580_COPPER, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_82580_FIBER, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_82580_SERDES, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_82580_SGMII, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_82580_COPPER_DUAL, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_82580_QUAD_FIBER, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_DH89XXCC_SERDES, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_DH89XXCC_SGMII, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_DH89XXCC_SFP, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_DH89XXCC_BACKPLANE, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_I350_COPPER, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_I350_FIBER, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_I350_SERDES, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_I350_SGMII, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_I350_VF, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_I210_COPPER, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_I210_COPPER_IT, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_I210_COPPER_OEM1, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_I210_COPPER_FLASHLESS, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_I210_SERDES_FLASHLESS, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_I210_FIBER, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_I210_SERDES, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_I210_SGMII, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_I211_COPPER, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_I354_BACKPLANE_1GBPS, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	PVID(0x8086, E1000_DEV_ID_I354_SGMII, "Intel(R) PRO/1000 PCI-Express Network Driver"),
 	/* required last entry */
 	PVID_END
 };
 
 /*********************************************************************
  *  Function prototypes
  *********************************************************************/
 static void	*em_register(device_t dev);
 static void	*igb_register(device_t dev);
 static int	em_if_attach_pre(if_ctx_t ctx);
 static int	em_if_attach_post(if_ctx_t ctx);
 static int	em_if_detach(if_ctx_t ctx);
 static int	em_if_shutdown(if_ctx_t ctx);
 static int	em_if_suspend(if_ctx_t ctx);
 static int	em_if_resume(if_ctx_t ctx);
 
 static int	em_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int ntxqs, int ntxqsets);
 static int	em_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nrxqs, int nrxqsets);
 static void	em_if_queues_free(if_ctx_t ctx);
 
 static uint64_t	em_if_get_counter(if_ctx_t, ift_counter);
 static void	em_if_init(if_ctx_t ctx);
 static void	em_if_stop(if_ctx_t ctx);
 static void	em_if_media_status(if_ctx_t, struct ifmediareq *);
 static int	em_if_media_change(if_ctx_t ctx);
 static int	em_if_mtu_set(if_ctx_t ctx, uint32_t mtu);
 static void	em_if_timer(if_ctx_t ctx, uint16_t qid);
 static void	em_if_vlan_register(if_ctx_t ctx, u16 vtag);
 static void	em_if_vlan_unregister(if_ctx_t ctx, u16 vtag);
 static void	em_if_watchdog_reset(if_ctx_t ctx);
 
 static void	em_identify_hardware(if_ctx_t ctx);
 static int	em_allocate_pci_resources(if_ctx_t ctx);
 static void	em_free_pci_resources(if_ctx_t ctx);
 static void	em_reset(if_ctx_t ctx);
 static int	em_setup_interface(if_ctx_t ctx);
 static int	em_setup_msix(if_ctx_t ctx);
 
 static void	em_initialize_transmit_unit(if_ctx_t ctx);
 static void	em_initialize_receive_unit(if_ctx_t ctx);
 
 static void	em_if_enable_intr(if_ctx_t ctx);
 static void	em_if_disable_intr(if_ctx_t ctx);
 static int	em_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid);
 static int	em_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid);
 static void	em_if_multi_set(if_ctx_t ctx);
 static void	em_if_update_admin_status(if_ctx_t ctx);
 static void	em_if_debug(if_ctx_t ctx);
 static void	em_update_stats_counters(struct adapter *);
 static void	em_add_hw_stats(struct adapter *adapter);
 static int	em_if_set_promisc(if_ctx_t ctx, int flags);
 static void	em_setup_vlan_hw_support(struct adapter *);
 static int	em_sysctl_nvm_info(SYSCTL_HANDLER_ARGS);
 static void	em_print_nvm_info(struct adapter *);
 static int	em_sysctl_debug_info(SYSCTL_HANDLER_ARGS);
 static int	em_get_rs(SYSCTL_HANDLER_ARGS);
 static void	em_print_debug_info(struct adapter *);
 static int 	em_is_valid_ether_addr(u8 *);
 static int	em_sysctl_int_delay(SYSCTL_HANDLER_ARGS);
 static void	em_add_int_delay_sysctl(struct adapter *, const char *,
 		    const char *, struct em_int_delay_info *, int, int);
 /* Management and WOL Support */
 static void	em_init_manageability(struct adapter *);
 static void	em_release_manageability(struct adapter *);
 static void	em_get_hw_control(struct adapter *);
 static void	em_release_hw_control(struct adapter *);
 static void	em_get_wakeup(if_ctx_t ctx);
 static void	em_enable_wakeup(if_ctx_t ctx);
 static int	em_enable_phy_wakeup(struct adapter *);
 static void	em_disable_aspm(struct adapter *);
 
 int		em_intr(void *arg);
 static void	em_disable_promisc(if_ctx_t ctx);
 
 /* MSI-X handlers */
 static int	em_if_msix_intr_assign(if_ctx_t, int);
 static int	em_msix_link(void *);
 static void	em_handle_link(void *context);
 
 static void	em_enable_vectors_82574(if_ctx_t);
 
 static int	em_set_flowcntl(SYSCTL_HANDLER_ARGS);
 static int	em_sysctl_eee(SYSCTL_HANDLER_ARGS);
 static void	em_if_led_func(if_ctx_t ctx, int onoff);
 
 static int	em_get_regs(SYSCTL_HANDLER_ARGS);
 
 static void	lem_smartspeed(struct adapter *adapter);
 static void	igb_configure_queues(struct adapter *adapter);
 
 
 /*********************************************************************
  *  FreeBSD Device Interface Entry Points
  *********************************************************************/
 static device_method_t em_methods[] = {
 	/* Device interface */
 	DEVMETHOD(device_register, em_register),
 	DEVMETHOD(device_probe, iflib_device_probe),
 	DEVMETHOD(device_attach, iflib_device_attach),
 	DEVMETHOD(device_detach, iflib_device_detach),
 	DEVMETHOD(device_shutdown, iflib_device_shutdown),
 	DEVMETHOD(device_suspend, iflib_device_suspend),
 	DEVMETHOD(device_resume, iflib_device_resume),
 	DEVMETHOD_END
 };
 
 static device_method_t igb_methods[] = {
 	/* Device interface */
 	DEVMETHOD(device_register, igb_register),
 	DEVMETHOD(device_probe, iflib_device_probe),
 	DEVMETHOD(device_attach, iflib_device_attach),
 	DEVMETHOD(device_detach, iflib_device_detach),
 	DEVMETHOD(device_shutdown, iflib_device_shutdown),
 	DEVMETHOD(device_suspend, iflib_device_suspend),
 	DEVMETHOD(device_resume, iflib_device_resume),
 	DEVMETHOD_END
 };
 
 
 static driver_t em_driver = {
 	"em", em_methods, sizeof(struct adapter),
 };
 
 static devclass_t em_devclass;
 DRIVER_MODULE(em, pci, em_driver, em_devclass, 0, 0);
 
 MODULE_DEPEND(em, pci, 1, 1, 1);
 MODULE_DEPEND(em, ether, 1, 1, 1);
 MODULE_DEPEND(em, iflib, 1, 1, 1);
 
 IFLIB_PNP_INFO(pci, em, em_vendor_info_array);
 
 static driver_t igb_driver = {
 	"igb", igb_methods, sizeof(struct adapter),
 };
 
 static devclass_t igb_devclass;
 DRIVER_MODULE(igb, pci, igb_driver, igb_devclass, 0, 0);
 
 MODULE_DEPEND(igb, pci, 1, 1, 1);
 MODULE_DEPEND(igb, ether, 1, 1, 1);
 MODULE_DEPEND(igb, iflib, 1, 1, 1);
 
 IFLIB_PNP_INFO(pci, igb, igb_vendor_info_array);
 
 static device_method_t em_if_methods[] = {
 	DEVMETHOD(ifdi_attach_pre, em_if_attach_pre),
 	DEVMETHOD(ifdi_attach_post, em_if_attach_post),
 	DEVMETHOD(ifdi_detach, em_if_detach),
 	DEVMETHOD(ifdi_shutdown, em_if_shutdown),
 	DEVMETHOD(ifdi_suspend, em_if_suspend),
 	DEVMETHOD(ifdi_resume, em_if_resume),
 	DEVMETHOD(ifdi_init, em_if_init),
 	DEVMETHOD(ifdi_stop, em_if_stop),
 	DEVMETHOD(ifdi_msix_intr_assign, em_if_msix_intr_assign),
 	DEVMETHOD(ifdi_intr_enable, em_if_enable_intr),
 	DEVMETHOD(ifdi_intr_disable, em_if_disable_intr),
 	DEVMETHOD(ifdi_tx_queues_alloc, em_if_tx_queues_alloc),
 	DEVMETHOD(ifdi_rx_queues_alloc, em_if_rx_queues_alloc),
 	DEVMETHOD(ifdi_queues_free, em_if_queues_free),
 	DEVMETHOD(ifdi_update_admin_status, em_if_update_admin_status),
 	DEVMETHOD(ifdi_multi_set, em_if_multi_set),
 	DEVMETHOD(ifdi_media_status, em_if_media_status),
 	DEVMETHOD(ifdi_media_change, em_if_media_change),
 	DEVMETHOD(ifdi_mtu_set, em_if_mtu_set),
 	DEVMETHOD(ifdi_promisc_set, em_if_set_promisc),
 	DEVMETHOD(ifdi_timer, em_if_timer),
 	DEVMETHOD(ifdi_watchdog_reset, em_if_watchdog_reset),
 	DEVMETHOD(ifdi_vlan_register, em_if_vlan_register),
 	DEVMETHOD(ifdi_vlan_unregister, em_if_vlan_unregister),
 	DEVMETHOD(ifdi_get_counter, em_if_get_counter),
 	DEVMETHOD(ifdi_led_func, em_if_led_func),
 	DEVMETHOD(ifdi_rx_queue_intr_enable, em_if_rx_queue_intr_enable),
 	DEVMETHOD(ifdi_tx_queue_intr_enable, em_if_tx_queue_intr_enable),
 	DEVMETHOD(ifdi_debug, em_if_debug),
 	DEVMETHOD_END
 };
 
 /*
  * note that if (adapter->msix_mem) is replaced by:
  * if (adapter->intr_type == IFLIB_INTR_MSIX)
  */
 static driver_t em_if_driver = {
 	"em_if", em_if_methods, sizeof(struct adapter)
 };
 
 /*********************************************************************
  *  Tunable default values.
  *********************************************************************/
 
 #define EM_TICKS_TO_USECS(ticks)	((1024 * (ticks) + 500) / 1000)
 #define EM_USECS_TO_TICKS(usecs)	((1000 * (usecs) + 512) / 1024)
 
 #define MAX_INTS_PER_SEC	8000
 #define DEFAULT_ITR		(1000000000/(MAX_INTS_PER_SEC * 256))
 
 /* Allow common code without TSO */
 #ifndef CSUM_TSO
 #define CSUM_TSO	0
 #endif
 
 static SYSCTL_NODE(_hw, OID_AUTO, em, CTLFLAG_RD, 0, "EM driver parameters");
 
 static int em_disable_crc_stripping = 0;
 SYSCTL_INT(_hw_em, OID_AUTO, disable_crc_stripping, CTLFLAG_RDTUN,
     &em_disable_crc_stripping, 0, "Disable CRC Stripping");
 
 static int em_tx_int_delay_dflt = EM_TICKS_TO_USECS(EM_TIDV);
 static int em_rx_int_delay_dflt = EM_TICKS_TO_USECS(EM_RDTR);
 SYSCTL_INT(_hw_em, OID_AUTO, tx_int_delay, CTLFLAG_RDTUN, &em_tx_int_delay_dflt,
     0, "Default transmit interrupt delay in usecs");
 SYSCTL_INT(_hw_em, OID_AUTO, rx_int_delay, CTLFLAG_RDTUN, &em_rx_int_delay_dflt,
     0, "Default receive interrupt delay in usecs");
 
 static int em_tx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_TADV);
 static int em_rx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_RADV);
 SYSCTL_INT(_hw_em, OID_AUTO, tx_abs_int_delay, CTLFLAG_RDTUN,
     &em_tx_abs_int_delay_dflt, 0,
     "Default transmit interrupt delay limit in usecs");
 SYSCTL_INT(_hw_em, OID_AUTO, rx_abs_int_delay, CTLFLAG_RDTUN,
     &em_rx_abs_int_delay_dflt, 0,
     "Default receive interrupt delay limit in usecs");
 
 static int em_smart_pwr_down = FALSE;
 SYSCTL_INT(_hw_em, OID_AUTO, smart_pwr_down, CTLFLAG_RDTUN, &em_smart_pwr_down,
     0, "Set to true to leave smart power down enabled on newer adapters");
 
 /* Controls whether promiscuous also shows bad packets */
 static int em_debug_sbp = TRUE;
 SYSCTL_INT(_hw_em, OID_AUTO, sbp, CTLFLAG_RDTUN, &em_debug_sbp, 0,
     "Show bad packets in promiscuous mode");
 
 /* How many packets rxeof tries to clean at a time */
 static int em_rx_process_limit = 100;
 SYSCTL_INT(_hw_em, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN,
     &em_rx_process_limit, 0,
     "Maximum number of received packets to process "
     "at a time, -1 means unlimited");
 
 /* Energy efficient ethernet - default to OFF */
 static int eee_setting = 1;
 SYSCTL_INT(_hw_em, OID_AUTO, eee_setting, CTLFLAG_RDTUN, &eee_setting, 0,
     "Enable Energy Efficient Ethernet");
 
 /*
 ** Tuneable Interrupt rate
 */
 static int em_max_interrupt_rate = 8000;
 SYSCTL_INT(_hw_em, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN,
     &em_max_interrupt_rate, 0, "Maximum interrupts per second");
 
 
 
 /* Global used in WOL setup with multiport cards */
 static int global_quad_port_a = 0;
 
 extern struct if_txrx igb_txrx;
 extern struct if_txrx em_txrx;
 extern struct if_txrx lem_txrx;
 
 static struct if_shared_ctx em_sctx_init = {
 	.isc_magic = IFLIB_MAGIC,
 	.isc_q_align = PAGE_SIZE,
 	.isc_tx_maxsize = EM_TSO_SIZE + sizeof(struct ether_vlan_header),
 	.isc_tx_maxsegsize = PAGE_SIZE,
 	.isc_tso_maxsize = EM_TSO_SIZE + sizeof(struct ether_vlan_header),
 	.isc_tso_maxsegsize = EM_TSO_SEG_SIZE,
 	.isc_rx_maxsize = MJUM9BYTES,
 	.isc_rx_nsegments = 1,
 	.isc_rx_maxsegsize = MJUM9BYTES,
 	.isc_nfl = 1,
 	.isc_nrxqs = 1,
 	.isc_ntxqs = 1,
 	.isc_admin_intrcnt = 1,
 	.isc_vendor_info = em_vendor_info_array,
 	.isc_driver_version = em_driver_version,
 	.isc_driver = &em_if_driver,
 	.isc_flags = IFLIB_NEED_SCRATCH | IFLIB_TSO_INIT_IP | IFLIB_NEED_ZERO_CSUM,
 
 	.isc_nrxd_min = {EM_MIN_RXD},
 	.isc_ntxd_min = {EM_MIN_TXD},
 	.isc_nrxd_max = {EM_MAX_RXD},
 	.isc_ntxd_max = {EM_MAX_TXD},
 	.isc_nrxd_default = {EM_DEFAULT_RXD},
 	.isc_ntxd_default = {EM_DEFAULT_TXD},
 };
 
 if_shared_ctx_t em_sctx = &em_sctx_init;
 
 static struct if_shared_ctx igb_sctx_init = {
 	.isc_magic = IFLIB_MAGIC,
 	.isc_q_align = PAGE_SIZE,
 	.isc_tx_maxsize = EM_TSO_SIZE + sizeof(struct ether_vlan_header),
 	.isc_tx_maxsegsize = PAGE_SIZE,
 	.isc_tso_maxsize = EM_TSO_SIZE + sizeof(struct ether_vlan_header),
 	.isc_tso_maxsegsize = EM_TSO_SEG_SIZE,
 	.isc_rx_maxsize = MJUM9BYTES,
 	.isc_rx_nsegments = 1,
 	.isc_rx_maxsegsize = MJUM9BYTES,
 	.isc_nfl = 1,
 	.isc_nrxqs = 1,
 	.isc_ntxqs = 1,
 	.isc_admin_intrcnt = 1,
 	.isc_vendor_info = igb_vendor_info_array,
 	.isc_driver_version = em_driver_version,
 	.isc_driver = &em_if_driver,
 	.isc_flags = IFLIB_NEED_SCRATCH | IFLIB_TSO_INIT_IP | IFLIB_NEED_ZERO_CSUM,
 
 	.isc_nrxd_min = {EM_MIN_RXD},
 	.isc_ntxd_min = {EM_MIN_TXD},
 	.isc_nrxd_max = {IGB_MAX_RXD},
 	.isc_ntxd_max = {IGB_MAX_TXD},
 	.isc_nrxd_default = {EM_DEFAULT_RXD},
 	.isc_ntxd_default = {EM_DEFAULT_TXD},
 };
 
 if_shared_ctx_t igb_sctx = &igb_sctx_init;
 
 /*****************************************************************
  *
  * Dump Registers
  *
  ****************************************************************/
 #define IGB_REGS_LEN 739
 
 static int em_get_regs(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter *adapter = (struct adapter *)arg1;
 	struct e1000_hw *hw = &adapter->hw;
 	struct sbuf *sb;
 	u32 *regs_buff;
 	int rc;
 
 	regs_buff = malloc(sizeof(u32) * IGB_REGS_LEN, M_DEVBUF, M_WAITOK);
 	memset(regs_buff, 0, IGB_REGS_LEN * sizeof(u32));
 
 	rc = sysctl_wire_old_buffer(req, 0);
 	MPASS(rc == 0);
 	if (rc != 0) {
 		free(regs_buff, M_DEVBUF);
 		return (rc);
 	}
 
 	sb = sbuf_new_for_sysctl(NULL, NULL, 32*400, req);
 	MPASS(sb != NULL);
 	if (sb == NULL) {
 		free(regs_buff, M_DEVBUF);
 		return (ENOMEM);
 	}
 
 	/* General Registers */
 	regs_buff[0] = E1000_READ_REG(hw, E1000_CTRL);
 	regs_buff[1] = E1000_READ_REG(hw, E1000_STATUS);
 	regs_buff[2] = E1000_READ_REG(hw, E1000_CTRL_EXT);
 	regs_buff[3] = E1000_READ_REG(hw, E1000_ICR);
 	regs_buff[4] = E1000_READ_REG(hw, E1000_RCTL);
 	regs_buff[5] = E1000_READ_REG(hw, E1000_RDLEN(0));
 	regs_buff[6] = E1000_READ_REG(hw, E1000_RDH(0));
 	regs_buff[7] = E1000_READ_REG(hw, E1000_RDT(0));
 	regs_buff[8] = E1000_READ_REG(hw, E1000_RXDCTL(0));
 	regs_buff[9] = E1000_READ_REG(hw, E1000_RDBAL(0));
 	regs_buff[10] = E1000_READ_REG(hw, E1000_RDBAH(0));
 	regs_buff[11] = E1000_READ_REG(hw, E1000_TCTL);
 	regs_buff[12] = E1000_READ_REG(hw, E1000_TDBAL(0));
 	regs_buff[13] = E1000_READ_REG(hw, E1000_TDBAH(0));
 	regs_buff[14] = E1000_READ_REG(hw, E1000_TDLEN(0));
 	regs_buff[15] = E1000_READ_REG(hw, E1000_TDH(0));
 	regs_buff[16] = E1000_READ_REG(hw, E1000_TDT(0));
 	regs_buff[17] = E1000_READ_REG(hw, E1000_TXDCTL(0));
 	regs_buff[18] = E1000_READ_REG(hw, E1000_TDFH);
 	regs_buff[19] = E1000_READ_REG(hw, E1000_TDFT);
 	regs_buff[20] = E1000_READ_REG(hw, E1000_TDFHS);
 	regs_buff[21] = E1000_READ_REG(hw, E1000_TDFPC);
 
 	sbuf_printf(sb, "General Registers\n");
 	sbuf_printf(sb, "\tCTRL\t %08x\n", regs_buff[0]);
 	sbuf_printf(sb, "\tSTATUS\t %08x\n", regs_buff[1]);
 	sbuf_printf(sb, "\tCTRL_EXIT\t %08x\n\n", regs_buff[2]);
 
 	sbuf_printf(sb, "Interrupt Registers\n");
 	sbuf_printf(sb, "\tICR\t %08x\n\n", regs_buff[3]);
 
 	sbuf_printf(sb, "RX Registers\n");
 	sbuf_printf(sb, "\tRCTL\t %08x\n", regs_buff[4]);
 	sbuf_printf(sb, "\tRDLEN\t %08x\n", regs_buff[5]);
 	sbuf_printf(sb, "\tRDH\t %08x\n", regs_buff[6]);
 	sbuf_printf(sb, "\tRDT\t %08x\n", regs_buff[7]);
 	sbuf_printf(sb, "\tRXDCTL\t %08x\n", regs_buff[8]);
 	sbuf_printf(sb, "\tRDBAL\t %08x\n", regs_buff[9]);
 	sbuf_printf(sb, "\tRDBAH\t %08x\n\n", regs_buff[10]);
 
 	sbuf_printf(sb, "TX Registers\n");
 	sbuf_printf(sb, "\tTCTL\t %08x\n", regs_buff[11]);
 	sbuf_printf(sb, "\tTDBAL\t %08x\n", regs_buff[12]);
 	sbuf_printf(sb, "\tTDBAH\t %08x\n", regs_buff[13]);
 	sbuf_printf(sb, "\tTDLEN\t %08x\n", regs_buff[14]);
 	sbuf_printf(sb, "\tTDH\t %08x\n", regs_buff[15]);
 	sbuf_printf(sb, "\tTDT\t %08x\n", regs_buff[16]);
 	sbuf_printf(sb, "\tTXDCTL\t %08x\n", regs_buff[17]);
 	sbuf_printf(sb, "\tTDFH\t %08x\n", regs_buff[18]);
 	sbuf_printf(sb, "\tTDFT\t %08x\n", regs_buff[19]);
 	sbuf_printf(sb, "\tTDFHS\t %08x\n", regs_buff[20]);
 	sbuf_printf(sb, "\tTDFPC\t %08x\n\n", regs_buff[21]);
 
 	free(regs_buff, M_DEVBUF);
 
 #ifdef DUMP_DESCS
 	{
 		if_softc_ctx_t scctx = adapter->shared;
 		struct rx_ring *rxr = &rx_que->rxr;
 		struct tx_ring *txr = &tx_que->txr;
 		int ntxd = scctx->isc_ntxd[0];
 		int nrxd = scctx->isc_nrxd[0];
 		int j;
 
 	for (j = 0; j < nrxd; j++) {
 		u32 staterr = le32toh(rxr->rx_base[j].wb.upper.status_error);
 		u32 length =  le32toh(rxr->rx_base[j].wb.upper.length);
 		sbuf_printf(sb, "\tReceive Descriptor Address %d: %08" PRIx64 "  Error:%d  Length:%d\n", j, rxr->rx_base[j].read.buffer_addr, staterr, length);
 	}
 
 	for (j = 0; j < min(ntxd, 256); j++) {
 		unsigned int *ptr = (unsigned int *)&txr->tx_base[j];
 
 		sbuf_printf(sb, "\tTXD[%03d] [0]: %08x [1]: %08x [2]: %08x [3]: %08x  eop: %d DD=%d\n",
 			    j, ptr[0], ptr[1], ptr[2], ptr[3], buf->eop,
 			    buf->eop != -1 ? txr->tx_base[buf->eop].upper.fields.status & E1000_TXD_STAT_DD : 0);
 
 	}
 	}
 #endif
 
 	rc = sbuf_finish(sb);
 	sbuf_delete(sb);
 	return(rc);
 }
 
 static void *
 em_register(device_t dev)
 {
 	return (em_sctx);
 }
 
 static void *
 igb_register(device_t dev)
 {
 	return (igb_sctx);
 }
 
 static int
 em_set_num_queues(if_ctx_t ctx)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	int maxqueues;
 
 	/* Sanity check based on HW */
 	switch (adapter->hw.mac.type) {
 	case e1000_82576:
 	case e1000_82580:
 	case e1000_i350:
 	case e1000_i354:
 		maxqueues = 8;
 		break;
 	case e1000_i210:
 	case e1000_82575:
 		maxqueues = 4;
 		break;
 	case e1000_i211:
 	case e1000_82574:
 		maxqueues = 2;
 		break;
 	default:
 		maxqueues = 1;
 		break;
 	}
 
 	return (maxqueues);
 }
 
 #define	LEM_CAPS							\
     IFCAP_HWCSUM | IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING |		\
     IFCAP_VLAN_HWCSUM | IFCAP_WOL | IFCAP_VLAN_HWFILTER
 
 #define	EM_CAPS								\
     IFCAP_HWCSUM | IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING |		\
     IFCAP_VLAN_HWCSUM | IFCAP_WOL | IFCAP_VLAN_HWFILTER | IFCAP_TSO4 |	\
     IFCAP_LRO | IFCAP_VLAN_HWTSO
 
 #define	IGB_CAPS							\
     IFCAP_HWCSUM | IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING |		\
     IFCAP_VLAN_HWCSUM | IFCAP_WOL | IFCAP_VLAN_HWFILTER | IFCAP_TSO4 |	\
     IFCAP_LRO | IFCAP_VLAN_HWTSO | IFCAP_JUMBO_MTU | IFCAP_HWCSUM_IPV6 |\
     IFCAP_TSO6
 
 /*********************************************************************
  *  Device initialization routine
  *
  *  The attach entry point is called when the driver is being loaded.
  *  This routine identifies the type of hardware, allocates all resources
  *  and initializes the hardware.
  *
  *  return 0 on success, positive on failure
  *********************************************************************/
 static int
 em_if_attach_pre(if_ctx_t ctx)
 {
 	struct adapter *adapter;
 	if_softc_ctx_t scctx;
 	device_t dev;
 	struct e1000_hw *hw;
 	int error = 0;
 
 	INIT_DEBUGOUT("em_if_attach_pre: begin");
 	dev = iflib_get_dev(ctx);
 	adapter = iflib_get_softc(ctx);
 
 	adapter->ctx = adapter->osdep.ctx = ctx;
 	adapter->dev = adapter->osdep.dev = dev;
 	scctx = adapter->shared = iflib_get_softc_ctx(ctx);
 	adapter->media = iflib_get_media(ctx);
 	hw = &adapter->hw;
 
 	adapter->tx_process_limit = scctx->isc_ntxd[0];
 
 	/* SYSCTL stuff */
 	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 	    OID_AUTO, "nvm", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
 	    em_sysctl_nvm_info, "I", "NVM Information");
 
 	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 	    OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
 	    em_sysctl_debug_info, "I", "Debug Information");
 
 	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 	    OID_AUTO, "fc", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
 	    em_set_flowcntl, "I", "Flow Control");
 
 	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 	    OID_AUTO, "reg_dump", CTLTYPE_STRING | CTLFLAG_RD, adapter, 0,
 	    em_get_regs, "A", "Dump Registers");
 
 	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 	    OID_AUTO, "rs_dump", CTLTYPE_INT | CTLFLAG_RW, adapter, 0,
 	    em_get_rs, "I", "Dump RS indexes");
 
 	/* Determine hardware and mac info */
 	em_identify_hardware(ctx);
 
 	scctx->isc_tx_nsegments = EM_MAX_SCATTER;
 	scctx->isc_nrxqsets_max = scctx->isc_ntxqsets_max = em_set_num_queues(ctx);
 	if (bootverbose)
 		device_printf(dev, "attach_pre capping queues at %d\n",
 		    scctx->isc_ntxqsets_max);
 
 	if (adapter->hw.mac.type >= igb_mac_min) {
 		scctx->isc_txqsizes[0] = roundup2(scctx->isc_ntxd[0] * sizeof(union e1000_adv_tx_desc), EM_DBA_ALIGN);
 		scctx->isc_rxqsizes[0] = roundup2(scctx->isc_nrxd[0] * sizeof(union e1000_adv_rx_desc), EM_DBA_ALIGN);
 		scctx->isc_txd_size[0] = sizeof(union e1000_adv_tx_desc);
 		scctx->isc_rxd_size[0] = sizeof(union e1000_adv_rx_desc);
 		scctx->isc_txrx = &igb_txrx;
 		scctx->isc_tx_tso_segments_max = EM_MAX_SCATTER;
 		scctx->isc_tx_tso_size_max = EM_TSO_SIZE;
 		scctx->isc_tx_tso_segsize_max = EM_TSO_SEG_SIZE;
 		scctx->isc_capabilities = scctx->isc_capenable = IGB_CAPS;
 		scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_TSO |
 		     CSUM_IP6_TCP | CSUM_IP6_UDP;
 		if (adapter->hw.mac.type != e1000_82575)
 			scctx->isc_tx_csum_flags |= CSUM_SCTP | CSUM_IP6_SCTP;
 		/*
 		** Some new devices, as with ixgbe, now may
 		** use a different BAR, so we need to keep
 		** track of which is used.
 		*/
 		scctx->isc_msix_bar = PCIR_BAR(EM_MSIX_BAR);
 		if (pci_read_config(dev, scctx->isc_msix_bar, 4) == 0)
 			scctx->isc_msix_bar += 4;
 	} else if (adapter->hw.mac.type >= em_mac_min) {
 		scctx->isc_txqsizes[0] = roundup2(scctx->isc_ntxd[0]* sizeof(struct e1000_tx_desc), EM_DBA_ALIGN);
 		scctx->isc_rxqsizes[0] = roundup2(scctx->isc_nrxd[0] * sizeof(union e1000_rx_desc_extended), EM_DBA_ALIGN);
 		scctx->isc_txd_size[0] = sizeof(struct e1000_tx_desc);
 		scctx->isc_rxd_size[0] = sizeof(union e1000_rx_desc_extended);
 		scctx->isc_txrx = &em_txrx;
 		scctx->isc_tx_tso_segments_max = EM_MAX_SCATTER;
 		scctx->isc_tx_tso_size_max = EM_TSO_SIZE;
 		scctx->isc_tx_tso_segsize_max = EM_TSO_SEG_SIZE;
 		scctx->isc_capabilities = scctx->isc_capenable = EM_CAPS;
 		/*
 		 * For EM-class devices, don't enable IFCAP_{TSO4,VLAN_HWTSO}
 		 * by default as we don't have workarounds for all associated
 		 * silicon errata.  E. g., with several MACs such as 82573E,
 		 * TSO only works at Gigabit speed and otherwise can cause the
 		 * hardware to hang (which also would be next to impossible to
 		 * work around given that already queued TSO-using descriptors
 		 * would need to be flushed and vlan(4) reconfigured at runtime
 		 * in case of a link speed change).  Moreover, MACs like 82579
 		 * still can hang at Gigabit even with all publicly documented
 		 * TSO workarounds implemented.  Generally, the penality of
 		 * these workarounds is rather high and may involve copying
 		 * mbuf data around so advantages of TSO lapse.  Still, TSO may
 		 * work for a few MACs of this class - at least when sticking
 		 * with Gigabit - in which case users may enable TSO manually.
 		 */
 		scctx->isc_capenable &= ~(IFCAP_TSO4 | IFCAP_VLAN_HWTSO);
 		scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_IP_TSO;
 		/*
 		 * We support MSI-X with 82574 only, but indicate to iflib(4)
 		 * that it shall give MSI at least a try with other devices.
 		 */
 		if (adapter->hw.mac.type == e1000_82574) {
 			scctx->isc_msix_bar = PCIR_BAR(EM_MSIX_BAR);
 		} else {
 			scctx->isc_msix_bar = -1;
 			scctx->isc_disable_msix = 1;
 		}
 	} else {
 		scctx->isc_txqsizes[0] = roundup2((scctx->isc_ntxd[0] + 1) * sizeof(struct e1000_tx_desc), EM_DBA_ALIGN);
 		scctx->isc_rxqsizes[0] = roundup2((scctx->isc_nrxd[0] + 1) * sizeof(struct e1000_rx_desc), EM_DBA_ALIGN);
 		scctx->isc_txd_size[0] = sizeof(struct e1000_tx_desc);
 		scctx->isc_rxd_size[0] = sizeof(struct e1000_rx_desc);
 		scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP;
 		scctx->isc_txrx = &lem_txrx;
 		scctx->isc_capabilities = scctx->isc_capenable = LEM_CAPS;
 		if (adapter->hw.mac.type < e1000_82543)
 			scctx->isc_capenable &= ~(IFCAP_HWCSUM|IFCAP_VLAN_HWCSUM);
 		/* INTx only */
 		scctx->isc_msix_bar = 0;
 	}
 
 	/* Setup PCI resources */
 	if (em_allocate_pci_resources(ctx)) {
 		device_printf(dev, "Allocation of PCI resources failed\n");
 		error = ENXIO;
 		goto err_pci;
 	}
 
 	/*
 	** For ICH8 and family we need to
 	** map the flash memory, and this
 	** must happen after the MAC is
 	** identified
 	*/
 	if ((hw->mac.type == e1000_ich8lan) ||
 	    (hw->mac.type == e1000_ich9lan) ||
 	    (hw->mac.type == e1000_ich10lan) ||
 	    (hw->mac.type == e1000_pchlan) ||
 	    (hw->mac.type == e1000_pch2lan) ||
 	    (hw->mac.type == e1000_pch_lpt)) {
 		int rid = EM_BAR_TYPE_FLASH;
 		adapter->flash = bus_alloc_resource_any(dev,
 		    SYS_RES_MEMORY, &rid, RF_ACTIVE);
 		if (adapter->flash == NULL) {
 			device_printf(dev, "Mapping of Flash failed\n");
 			error = ENXIO;
 			goto err_pci;
 		}
 		/* This is used in the shared code */
 		hw->flash_address = (u8 *)adapter->flash;
 		adapter->osdep.flash_bus_space_tag =
 		    rman_get_bustag(adapter->flash);
 		adapter->osdep.flash_bus_space_handle =
 		    rman_get_bushandle(adapter->flash);
 	}
 	/*
 	** In the new SPT device flash is not  a
 	** separate BAR, rather it is also in BAR0,
 	** so use the same tag and an offset handle for the
 	** FLASH read/write macros in the shared code.
 	*/
 	else if (hw->mac.type >= e1000_pch_spt) {
 		adapter->osdep.flash_bus_space_tag =
 		    adapter->osdep.mem_bus_space_tag;
 		adapter->osdep.flash_bus_space_handle =
 		    adapter->osdep.mem_bus_space_handle
 		    + E1000_FLASH_BASE_ADDR;
 	}
 
 	/* Do Shared Code initialization */
 	error = e1000_setup_init_funcs(hw, TRUE);
 	if (error) {
 		device_printf(dev, "Setup of Shared code failed, error %d\n",
 		    error);
 		error = ENXIO;
 		goto err_pci;
 	}
 
 	em_setup_msix(ctx);
 	e1000_get_bus_info(hw);
 
 	/* Set up some sysctls for the tunable interrupt delays */
 	em_add_int_delay_sysctl(adapter, "rx_int_delay",
 	    "receive interrupt delay in usecs", &adapter->rx_int_delay,
 	    E1000_REGISTER(hw, E1000_RDTR), em_rx_int_delay_dflt);
 	em_add_int_delay_sysctl(adapter, "tx_int_delay",
 	    "transmit interrupt delay in usecs", &adapter->tx_int_delay,
 	    E1000_REGISTER(hw, E1000_TIDV), em_tx_int_delay_dflt);
 	em_add_int_delay_sysctl(adapter, "rx_abs_int_delay",
 	    "receive interrupt delay limit in usecs",
 	    &adapter->rx_abs_int_delay,
 	    E1000_REGISTER(hw, E1000_RADV),
 	    em_rx_abs_int_delay_dflt);
 	em_add_int_delay_sysctl(adapter, "tx_abs_int_delay",
 	    "transmit interrupt delay limit in usecs",
 	    &adapter->tx_abs_int_delay,
 	    E1000_REGISTER(hw, E1000_TADV),
 	    em_tx_abs_int_delay_dflt);
 	em_add_int_delay_sysctl(adapter, "itr",
 	    "interrupt delay limit in usecs/4",
 	    &adapter->tx_itr,
 	    E1000_REGISTER(hw, E1000_ITR),
 	    DEFAULT_ITR);
 
 	hw->mac.autoneg = DO_AUTO_NEG;
 	hw->phy.autoneg_wait_to_complete = FALSE;
 	hw->phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
 
 	if (adapter->hw.mac.type < em_mac_min) {
 		e1000_init_script_state_82541(&adapter->hw, TRUE);
 		e1000_set_tbi_compatibility_82543(&adapter->hw, TRUE);
 	}
 	/* Copper options */
 	if (hw->phy.media_type == e1000_media_type_copper) {
 		hw->phy.mdix = AUTO_ALL_MODES;
 		hw->phy.disable_polarity_correction = FALSE;
 		hw->phy.ms_type = EM_MASTER_SLAVE;
 	}
 
 	/*
 	 * Set the frame limits assuming
 	 * standard ethernet sized frames.
 	 */
 	scctx->isc_max_frame_size = adapter->hw.mac.max_frame_size =
 	    ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE;
 
 	/*
 	 * This controls when hardware reports transmit completion
 	 * status.
 	 */
 	hw->mac.report_tx_early = 1;
 
 	/* Allocate multicast array memory. */
 	adapter->mta = malloc(sizeof(u8) * ETH_ADDR_LEN *
 	    MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
 	if (adapter->mta == NULL) {
 		device_printf(dev, "Can not allocate multicast setup array\n");
 		error = ENOMEM;
 		goto err_late;
 	}
 
 	/* Check SOL/IDER usage */
 	if (e1000_check_reset_block(hw))
 		device_printf(dev, "PHY reset is blocked"
 			      " due to SOL/IDER session.\n");
 
 	/* Sysctl for setting Energy Efficient Ethernet */
 	hw->dev_spec.ich8lan.eee_disable = eee_setting;
 	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 	    OID_AUTO, "eee_control", CTLTYPE_INT|CTLFLAG_RW,
 	    adapter, 0, em_sysctl_eee, "I",
 	    "Disable Energy Efficient Ethernet");
 
 	/*
 	** Start from a known state, this is
 	** important in reading the nvm and
 	** mac from that.
 	*/
 	e1000_reset_hw(hw);
 
 	/* Make sure we have a good EEPROM before we read from it */
 	if (e1000_validate_nvm_checksum(hw) < 0) {
 		/*
 		** Some PCI-E parts fail the first check due to
 		** the link being in sleep state, call it again,
 		** if it fails a second time its a real issue.
 		*/
 		if (e1000_validate_nvm_checksum(hw) < 0) {
 			device_printf(dev,
 			    "The EEPROM Checksum Is Not Valid\n");
 			error = EIO;
 			goto err_late;
 		}
 	}
 
 	/* Copy the permanent MAC address out of the EEPROM */
 	if (e1000_read_mac_addr(hw) < 0) {
 		device_printf(dev, "EEPROM read error while reading MAC"
 			      " address\n");
 		error = EIO;
 		goto err_late;
 	}
 
 	if (!em_is_valid_ether_addr(hw->mac.addr)) {
 		device_printf(dev, "Invalid MAC address\n");
 		error = EIO;
 		goto err_late;
 	}
 
 	/* Disable ULP support */
 	e1000_disable_ulp_lpt_lp(hw, TRUE);
 
 	/*
 	 * Get Wake-on-Lan and Management info for later use
 	 */
 	em_get_wakeup(ctx);
 
 	/* Enable only WOL MAGIC by default */
 	scctx->isc_capenable &= ~IFCAP_WOL;
 	if (adapter->wol != 0)
 		scctx->isc_capenable |= IFCAP_WOL_MAGIC;
 
 	iflib_set_mac(ctx, hw->mac.addr);
 
 	return (0);
 
 err_late:
 	em_release_hw_control(adapter);
 err_pci:
 	em_free_pci_resources(ctx);
 	free(adapter->mta, M_DEVBUF);
 
 	return (error);
 }
 
 static int
 em_if_attach_post(if_ctx_t ctx)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	struct e1000_hw *hw = &adapter->hw;
 	int error = 0;
 	
 	/* Setup OS specific network interface */
 	error = em_setup_interface(ctx);
 	if (error != 0) {
 		goto err_late;
 	}
 
 	em_reset(ctx);
 
 	/* Initialize statistics */
 	em_update_stats_counters(adapter);
 	hw->mac.get_link_status = 1;
 	em_if_update_admin_status(ctx);
 	em_add_hw_stats(adapter);
 
 	/* Non-AMT based hardware can now take control from firmware */
 	if (adapter->has_manage && !adapter->has_amt)
 		em_get_hw_control(adapter);
 
 	INIT_DEBUGOUT("em_if_attach_post: end");
 
 	return (error);
 
 err_late:
 	em_release_hw_control(adapter);
 	em_free_pci_resources(ctx);
 	em_if_queues_free(ctx);
 	free(adapter->mta, M_DEVBUF);
 
 	return (error);
 }
 
 /*********************************************************************
  *  Device removal routine
  *
  *  The detach entry point is called when the driver is being removed.
  *  This routine stops the adapter and deallocates all the resources
  *  that were allocated for driver operation.
  *
  *  return 0 on success, positive on failure
  *********************************************************************/
 static int
 em_if_detach(if_ctx_t ctx)
 {
 	struct adapter	*adapter = iflib_get_softc(ctx);
 
 	INIT_DEBUGOUT("em_if_detach: begin");
 
 	e1000_phy_hw_reset(&adapter->hw);
 
 	em_release_manageability(adapter);
 	em_release_hw_control(adapter);
 	em_free_pci_resources(ctx);
 
 	return (0);
 }
 
 /*********************************************************************
  *
  *  Shutdown entry point
  *
  **********************************************************************/
 
 static int
 em_if_shutdown(if_ctx_t ctx)
 {
 	return em_if_suspend(ctx);
 }
 
 /*
  * Suspend/resume device methods.
  */
 static int
 em_if_suspend(if_ctx_t ctx)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 
 	em_release_manageability(adapter);
 	em_release_hw_control(adapter);
 	em_enable_wakeup(ctx);
 	return (0);
 }
 
 static int
 em_if_resume(if_ctx_t ctx)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 
 	if (adapter->hw.mac.type == e1000_pch2lan)
 		e1000_resume_workarounds_pchlan(&adapter->hw);
 	em_if_init(ctx);
 	em_init_manageability(adapter);
 
 	return(0);
 }
 
 static int
 em_if_mtu_set(if_ctx_t ctx, uint32_t mtu)
 {
 	int max_frame_size;
 	struct adapter *adapter = iflib_get_softc(ctx);
 	if_softc_ctx_t scctx = iflib_get_softc_ctx(ctx);
 
 	 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
 
 	switch (adapter->hw.mac.type) {
 	case e1000_82571:
 	case e1000_82572:
 	case e1000_ich9lan:
 	case e1000_ich10lan:
 	case e1000_pch2lan:
 	case e1000_pch_lpt:
 	case e1000_pch_spt:
 	case e1000_pch_cnp:
 	case e1000_82574:
 	case e1000_82583:
 	case e1000_80003es2lan:
 		/* 9K Jumbo Frame size */
 		max_frame_size = 9234;
 		break;
 	case e1000_pchlan:
 		max_frame_size = 4096;
 		break;
 	case e1000_82542:
 	case e1000_ich8lan:
 		/* Adapters that do not support jumbo frames */
 		max_frame_size = ETHER_MAX_LEN;
 		break;
 	default:
 		if (adapter->hw.mac.type >= igb_mac_min)
 			max_frame_size = 9234;
 		else /* lem */
 			max_frame_size = MAX_JUMBO_FRAME_SIZE;
 	}
 	if (mtu > max_frame_size - ETHER_HDR_LEN - ETHER_CRC_LEN) {
 		return (EINVAL);
 	}
 
 	scctx->isc_max_frame_size = adapter->hw.mac.max_frame_size =
 	    mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 	return (0);
 }
 
 /*********************************************************************
  *  Init entry point
  *
  *  This routine is used in two ways. It is used by the stack as
  *  init entry point in network interface structure. It is also used
  *  by the driver as a hw/sw initialization routine to get to a
  *  consistent state.
  *
  **********************************************************************/
 static void
 em_if_init(if_ctx_t ctx)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	if_softc_ctx_t scctx = adapter->shared;
 	struct ifnet *ifp = iflib_get_ifp(ctx);
 	struct em_tx_queue *tx_que;
 	int i;
 
 	INIT_DEBUGOUT("em_if_init: begin");
 
 	/* Get the latest mac address, User can use a LAA */
 	bcopy(if_getlladdr(ifp), adapter->hw.mac.addr,
 	    ETHER_ADDR_LEN);
 
 	/* Put the address into the Receive Address Array */
 	e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
 
 	/*
 	 * With the 82571 adapter, RAR[0] may be overwritten
 	 * when the other port is reset, we make a duplicate
 	 * in RAR[14] for that eventuality, this assures
 	 * the interface continues to function.
 	 */
 	if (adapter->hw.mac.type == e1000_82571) {
 		e1000_set_laa_state_82571(&adapter->hw, TRUE);
 		e1000_rar_set(&adapter->hw, adapter->hw.mac.addr,
 		    E1000_RAR_ENTRIES - 1);
 	}
 
 
 	/* Initialize the hardware */
 	em_reset(ctx);
 	em_if_update_admin_status(ctx);
 
 	for (i = 0, tx_que = adapter->tx_queues; i < adapter->tx_num_queues; i++, tx_que++) {
 		struct tx_ring *txr = &tx_que->txr;
 
 		txr->tx_rs_cidx = txr->tx_rs_pidx;
 
 		/* Initialize the last processed descriptor to be the end of
 		 * the ring, rather than the start, so that we avoid an
 		 * off-by-one error when calculating how many descriptors are
 		 * done in the credits_update function.
 		 */
 		txr->tx_cidx_processed = scctx->isc_ntxd[0] - 1;
 	}
 
 	/* Setup VLAN support, basic and offload if available */
 	E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
 
 	/* Clear bad data from Rx FIFOs */
 	if (adapter->hw.mac.type >= igb_mac_min)
 		e1000_rx_fifo_flush_82575(&adapter->hw);
 
 	/* Configure for OS presence */
 	em_init_manageability(adapter);
 
 	/* Prepare transmit descriptors and buffers */
 	em_initialize_transmit_unit(ctx);
 
 	/* Setup Multicast table */
 	em_if_multi_set(ctx);
 
-	/*
-	 * Figure out the desired mbuf
-	 * pool for doing jumbos
-	 */
-	if (adapter->hw.mac.max_frame_size <= 2048)
-		adapter->rx_mbuf_sz = MCLBYTES;
-	else
-		adapter->rx_mbuf_sz = MJUMPAGESIZE;
+	adapter->rx_mbuf_sz = iflib_get_rx_mbuf_sz(ctx);
 	em_initialize_receive_unit(ctx);
 
 	/* Use real VLAN Filter support? */
 	if (if_getcapenable(ifp) & IFCAP_VLAN_HWTAGGING) {
 		if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER)
 			/* Use real VLAN Filter support */
 			em_setup_vlan_hw_support(adapter);
 		else {
 			u32 ctrl;
 			ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL);
 			ctrl |= E1000_CTRL_VME;
 			E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl);
 		}
 	}
 
 	/* Don't lose promiscuous settings */
 	em_if_set_promisc(ctx, IFF_PROMISC);
 	e1000_clear_hw_cntrs_base_generic(&adapter->hw);
 
 	/* MSI-X configuration for 82574 */
 	if (adapter->hw.mac.type == e1000_82574) {
 		int tmp = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
 
 		tmp |= E1000_CTRL_EXT_PBA_CLR;
 		E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, tmp);
 		/* Set the IVAR - interrupt vector routing. */
 		E1000_WRITE_REG(&adapter->hw, E1000_IVAR, adapter->ivars);
 	} else if (adapter->intr_type == IFLIB_INTR_MSIX) /* Set up queue routing */
 		igb_configure_queues(adapter);
 
 	/* this clears any pending interrupts */
 	E1000_READ_REG(&adapter->hw, E1000_ICR);
 	E1000_WRITE_REG(&adapter->hw, E1000_ICS, E1000_ICS_LSC);
 
 	/* AMT based hardware can now take control from firmware */
 	if (adapter->has_manage && adapter->has_amt)
 		em_get_hw_control(adapter);
 
 	/* Set Energy Efficient Ethernet */
 	if (adapter->hw.mac.type >= igb_mac_min &&
 	    adapter->hw.phy.media_type == e1000_media_type_copper) {
 		if (adapter->hw.mac.type == e1000_i354)
 			e1000_set_eee_i354(&adapter->hw, TRUE, TRUE);
 		else
 			e1000_set_eee_i350(&adapter->hw, TRUE, TRUE);
 	}
 }
 
 /*********************************************************************
  *
  *  Fast Legacy/MSI Combined Interrupt Service routine
  *
  *********************************************************************/
 int
 em_intr(void *arg)
 {
 	struct adapter *adapter = arg;
 	if_ctx_t ctx = adapter->ctx;
 	u32 reg_icr;
 
 	reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
 
 	if (adapter->intr_type != IFLIB_INTR_LEGACY)
 		goto skip_stray;
 	/* Hot eject? */
 	if (reg_icr == 0xffffffff)
 		return FILTER_STRAY;
 
 	/* Definitely not our interrupt. */
 	if (reg_icr == 0x0)
 		return FILTER_STRAY;
 
 	/*
 	 * Starting with the 82571 chip, bit 31 should be used to
 	 * determine whether the interrupt belongs to us.
 	 */
 	if (adapter->hw.mac.type >= e1000_82571 &&
 	    (reg_icr & E1000_ICR_INT_ASSERTED) == 0)
 		return FILTER_STRAY;
 
 skip_stray:
 	/* Link status change */
 	if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
 		adapter->hw.mac.get_link_status = 1;
 		iflib_admin_intr_deferred(ctx);
 	}
 
 	if (reg_icr & E1000_ICR_RXO)
 		adapter->rx_overruns++;
 
 	return (FILTER_SCHEDULE_THREAD);
 }
 
 static void
 igb_rx_enable_queue(struct adapter *adapter, struct em_rx_queue *rxq)
 {
 	E1000_WRITE_REG(&adapter->hw, E1000_EIMS, rxq->eims);
 }
 
 static void
 em_rx_enable_queue(struct adapter *adapter, struct em_rx_queue *rxq)
 {
 	E1000_WRITE_REG(&adapter->hw, E1000_IMS, rxq->eims);
 }
 
 static void
 igb_tx_enable_queue(struct adapter *adapter, struct em_tx_queue *txq)
 {
 	E1000_WRITE_REG(&adapter->hw, E1000_EIMS, txq->eims);
 }
 
 static void
 em_tx_enable_queue(struct adapter *adapter, struct em_tx_queue *txq)
 {
 	E1000_WRITE_REG(&adapter->hw, E1000_IMS, txq->eims);
 }
 
 static int
 em_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	struct em_rx_queue *rxq = &adapter->rx_queues[rxqid];
 
 	if (adapter->hw.mac.type >= igb_mac_min)
 		igb_rx_enable_queue(adapter, rxq);
 	else
 		em_rx_enable_queue(adapter, rxq);
 	return (0);
 }
 
 static int
 em_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	struct em_tx_queue *txq = &adapter->tx_queues[txqid];
 
 	if (adapter->hw.mac.type >= igb_mac_min)
 		igb_tx_enable_queue(adapter, txq);
 	else
 		em_tx_enable_queue(adapter, txq);
 	return (0);
 }
 
 /*********************************************************************
  *
  *  MSI-X RX Interrupt Service routine
  *
  **********************************************************************/
 static int
 em_msix_que(void *arg)
 {
 	struct em_rx_queue *que = arg;
 
 	++que->irqs;
 
 	return (FILTER_SCHEDULE_THREAD);
 }
 
 /*********************************************************************
  *
  *  MSI-X Link Fast Interrupt Service routine
  *
  **********************************************************************/
 static int
 em_msix_link(void *arg)
 {
 	struct adapter *adapter = arg;
 	u32 reg_icr;
 
 	++adapter->link_irq;
 	MPASS(adapter->hw.back != NULL);
 	reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
 
 	if (reg_icr & E1000_ICR_RXO)
 		adapter->rx_overruns++;
 
 	if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
 		em_handle_link(adapter->ctx);
 	} else {
 		E1000_WRITE_REG(&adapter->hw, E1000_IMS,
 				EM_MSIX_LINK | E1000_IMS_LSC);
 		if (adapter->hw.mac.type >= igb_mac_min)
 			E1000_WRITE_REG(&adapter->hw, E1000_EIMS, adapter->link_mask);
 	}
 
 	/*
 	 * Because we must read the ICR for this interrupt
 	 * it may clear other causes using autoclear, for
 	 * this reason we simply create a soft interrupt
 	 * for all these vectors.
 	 */
 	if (reg_icr && adapter->hw.mac.type < igb_mac_min) {
 		E1000_WRITE_REG(&adapter->hw,
 			E1000_ICS, adapter->ims);
 	}
 
 	return (FILTER_HANDLED);
 }
 
 static void
 em_handle_link(void *context)
 {
 	if_ctx_t ctx = context;
 	struct adapter *adapter = iflib_get_softc(ctx);
 
 	adapter->hw.mac.get_link_status = 1;
 	iflib_admin_intr_deferred(ctx);
 }
 
 
 /*********************************************************************
  *
  *  Media Ioctl callback
  *
  *  This routine is called whenever the user queries the status of
  *  the interface using ifconfig.
  *
  **********************************************************************/
 static void
 em_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	u_char fiber_type = IFM_1000_SX;
 
 	INIT_DEBUGOUT("em_if_media_status: begin");
 
 	iflib_admin_intr_deferred(ctx);
 
 	ifmr->ifm_status = IFM_AVALID;
 	ifmr->ifm_active = IFM_ETHER;
 
 	if (!adapter->link_active) {
 		return;
 	}
 
 	ifmr->ifm_status |= IFM_ACTIVE;
 
 	if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
 	    (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
 		if (adapter->hw.mac.type == e1000_82545)
 			fiber_type = IFM_1000_LX;
 		ifmr->ifm_active |= fiber_type | IFM_FDX;
 	} else {
 		switch (adapter->link_speed) {
 		case 10:
 			ifmr->ifm_active |= IFM_10_T;
 			break;
 		case 100:
 			ifmr->ifm_active |= IFM_100_TX;
 			break;
 		case 1000:
 			ifmr->ifm_active |= IFM_1000_T;
 			break;
 		}
 		if (adapter->link_duplex == FULL_DUPLEX)
 			ifmr->ifm_active |= IFM_FDX;
 		else
 			ifmr->ifm_active |= IFM_HDX;
 	}
 }
 
 /*********************************************************************
  *
  *  Media Ioctl callback
  *
  *  This routine is called when the user changes speed/duplex using
  *  media/mediopt option with ifconfig.
  *
  **********************************************************************/
 static int
 em_if_media_change(if_ctx_t ctx)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	struct ifmedia *ifm = iflib_get_media(ctx);
 
 	INIT_DEBUGOUT("em_if_media_change: begin");
 
 	if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
 		return (EINVAL);
 
 	switch (IFM_SUBTYPE(ifm->ifm_media)) {
 	case IFM_AUTO:
 		adapter->hw.mac.autoneg = DO_AUTO_NEG;
 		adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
 		break;
 	case IFM_1000_LX:
 	case IFM_1000_SX:
 	case IFM_1000_T:
 		adapter->hw.mac.autoneg = DO_AUTO_NEG;
 		adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
 		break;
 	case IFM_100_TX:
 		adapter->hw.mac.autoneg = FALSE;
 		adapter->hw.phy.autoneg_advertised = 0;
 		if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
 			adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL;
 		else
 			adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF;
 		break;
 	case IFM_10_T:
 		adapter->hw.mac.autoneg = FALSE;
 		adapter->hw.phy.autoneg_advertised = 0;
 		if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
 			adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL;
 		else
 			adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF;
 		break;
 	default:
 		device_printf(adapter->dev, "Unsupported media type\n");
 	}
 
 	em_if_init(ctx);
 
 	return (0);
 }
 
 static int
 em_if_set_promisc(if_ctx_t ctx, int flags)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	u32 reg_rctl;
 
 	em_disable_promisc(ctx);
 
 	reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
 
 	if (flags & IFF_PROMISC) {
 		reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
 		/* Turn this on if you want to see bad packets */
 		if (em_debug_sbp)
 			reg_rctl |= E1000_RCTL_SBP;
 		E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
 	} else if (flags & IFF_ALLMULTI) {
 		reg_rctl |= E1000_RCTL_MPE;
 		reg_rctl &= ~E1000_RCTL_UPE;
 		E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
 	}
 	return (0);
 }
 
 static void
 em_disable_promisc(if_ctx_t ctx)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	struct ifnet *ifp = iflib_get_ifp(ctx);
 	u32 reg_rctl;
 	int mcnt = 0;
 
 	reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
 	reg_rctl &= (~E1000_RCTL_UPE);
 	if (if_getflags(ifp) & IFF_ALLMULTI)
 		mcnt = MAX_NUM_MULTICAST_ADDRESSES;
 	else
 		mcnt = if_multiaddr_count(ifp, MAX_NUM_MULTICAST_ADDRESSES);
 	/* Don't disable if in MAX groups */
 	if (mcnt < MAX_NUM_MULTICAST_ADDRESSES)
 		reg_rctl &=  (~E1000_RCTL_MPE);
 	reg_rctl &=  (~E1000_RCTL_SBP);
 	E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
 }
 
 
 /*********************************************************************
  *  Multicast Update
  *
  *  This routine is called whenever multicast address list is updated.
  *
  **********************************************************************/
 
 static void
 em_if_multi_set(if_ctx_t ctx)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	struct ifnet *ifp = iflib_get_ifp(ctx);
 	u32 reg_rctl = 0;
 	u8  *mta; /* Multicast array memory */
 	int mcnt = 0;
 
 	IOCTL_DEBUGOUT("em_set_multi: begin");
 
 	mta = adapter->mta;
 	bzero(mta, sizeof(u8) * ETH_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES);
 
 	if (adapter->hw.mac.type == e1000_82542 &&
 	    adapter->hw.revision_id == E1000_REVISION_2) {
 		reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
 		if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
 			e1000_pci_clear_mwi(&adapter->hw);
 		reg_rctl |= E1000_RCTL_RST;
 		E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
 		msec_delay(5);
 	}
 
 	if_multiaddr_array(ifp, mta, &mcnt, MAX_NUM_MULTICAST_ADDRESSES);
 
 	if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
 		reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
 		reg_rctl |= E1000_RCTL_MPE;
 		E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
 	} else
 		e1000_update_mc_addr_list(&adapter->hw, mta, mcnt);
 
 	if (adapter->hw.mac.type == e1000_82542 &&
 	    adapter->hw.revision_id == E1000_REVISION_2) {
 		reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
 		reg_rctl &= ~E1000_RCTL_RST;
 		E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
 		msec_delay(5);
 		if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
 			e1000_pci_set_mwi(&adapter->hw);
 	}
 }
 
 /*********************************************************************
  *  Timer routine
  *
  *  This routine schedules em_if_update_admin_status() to check for
  *  link status and to gather statistics as well as to perform some
  *  controller-specific hardware patting.
  *
  **********************************************************************/
 static void
 em_if_timer(if_ctx_t ctx, uint16_t qid)
 {
 
 	if (qid != 0)
 		return;
 
 	iflib_admin_intr_deferred(ctx);
 }
 
 static void
 em_if_update_admin_status(if_ctx_t ctx)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	struct e1000_hw *hw = &adapter->hw;
 	device_t dev = iflib_get_dev(ctx);
 	u32 link_check, thstat, ctrl;
 
 	link_check = thstat = ctrl = 0;
 	/* Get the cached link value or read phy for real */
 	switch (hw->phy.media_type) {
 	case e1000_media_type_copper:
 		if (hw->mac.get_link_status) {
 			if (hw->mac.type == e1000_pch_spt)
 				msec_delay(50);
 			/* Do the work to read phy */
 			e1000_check_for_link(hw);
 			link_check = !hw->mac.get_link_status;
 			if (link_check) /* ESB2 fix */
 				e1000_cfg_on_link_up(hw);
 		} else {
 			link_check = TRUE;
 		}
 		break;
 	case e1000_media_type_fiber:
 		e1000_check_for_link(hw);
 		link_check = (E1000_READ_REG(hw, E1000_STATUS) &
 			    E1000_STATUS_LU);
 		break;
 	case e1000_media_type_internal_serdes:
 		e1000_check_for_link(hw);
 		link_check = adapter->hw.mac.serdes_has_link;
 		break;
 	/* VF device is type_unknown */
 	case e1000_media_type_unknown:
 		e1000_check_for_link(hw);
 		link_check = !hw->mac.get_link_status;
 		/* FALLTHROUGH */
 	default:
 		break;
 	}
 
 	/* Check for thermal downshift or shutdown */
 	if (hw->mac.type == e1000_i350) {
 		thstat = E1000_READ_REG(hw, E1000_THSTAT);
 		ctrl = E1000_READ_REG(hw, E1000_CTRL_EXT);
 	}
 
 	/* Now check for a transition */
 	if (link_check && (adapter->link_active == 0)) {
 		e1000_get_speed_and_duplex(hw, &adapter->link_speed,
 		    &adapter->link_duplex);
 		/* Check if we must disable SPEED_MODE bit on PCI-E */
 		if ((adapter->link_speed != SPEED_1000) &&
 		    ((hw->mac.type == e1000_82571) ||
 		    (hw->mac.type == e1000_82572))) {
 			int tarc0;
 			tarc0 = E1000_READ_REG(hw, E1000_TARC(0));
 			tarc0 &= ~TARC_SPEED_MODE_BIT;
 			E1000_WRITE_REG(hw, E1000_TARC(0), tarc0);
 		}
 		if (bootverbose)
 			device_printf(dev, "Link is up %d Mbps %s\n",
 			    adapter->link_speed,
 			    ((adapter->link_duplex == FULL_DUPLEX) ?
 			    "Full Duplex" : "Half Duplex"));
 		adapter->link_active = 1;
 		adapter->smartspeed = 0;
 		if ((ctrl & E1000_CTRL_EXT_LINK_MODE_MASK) ==
 		    E1000_CTRL_EXT_LINK_MODE_GMII &&
 		    (thstat & E1000_THSTAT_LINK_THROTTLE))
 			device_printf(dev, "Link: thermal downshift\n");
 		/* Delay Link Up for Phy update */
 		if (((hw->mac.type == e1000_i210) ||
 		    (hw->mac.type == e1000_i211)) &&
 		    (hw->phy.id == I210_I_PHY_ID))
 			msec_delay(I210_LINK_DELAY);
 		/* Reset if the media type changed. */
 		if ((hw->dev_spec._82575.media_changed) &&
 			(adapter->hw.mac.type >= igb_mac_min)) {
 			hw->dev_spec._82575.media_changed = false;
 			adapter->flags |= IGB_MEDIA_RESET;
 			em_reset(ctx);
 		}
 		iflib_link_state_change(ctx, LINK_STATE_UP,
 		    IF_Mbps(adapter->link_speed));
 	} else if (!link_check && (adapter->link_active == 1)) {
 		adapter->link_speed = 0;
 		adapter->link_duplex = 0;
 		adapter->link_active = 0;
 		iflib_link_state_change(ctx, LINK_STATE_DOWN, 0);
 	}
 	em_update_stats_counters(adapter);
 
 	/* Reset LAA into RAR[0] on 82571 */
 	if ((adapter->hw.mac.type == e1000_82571) &&
 	    e1000_get_laa_state_82571(&adapter->hw))
 		e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
 
 	if (adapter->hw.mac.type < em_mac_min)
 		lem_smartspeed(adapter);
 
 	E1000_WRITE_REG(&adapter->hw, E1000_IMS, EM_MSIX_LINK | E1000_IMS_LSC);
 }
 
 static void
 em_if_watchdog_reset(if_ctx_t ctx)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 
 	/*
 	 * Just count the event; iflib(4) will already trigger a
 	 * sufficient reset of the controller.
 	 */
 	adapter->watchdog_events++;
 }
 
 /*********************************************************************
  *
  *  This routine disables all traffic on the adapter by issuing a
  *  global reset on the MAC.
  *
  **********************************************************************/
 static void
 em_if_stop(if_ctx_t ctx)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 
 	INIT_DEBUGOUT("em_if_stop: begin");
 
 	e1000_reset_hw(&adapter->hw);
 	if (adapter->hw.mac.type >= e1000_82544)
 		E1000_WRITE_REG(&adapter->hw, E1000_WUFC, 0);
 
 	e1000_led_off(&adapter->hw);
 	e1000_cleanup_led(&adapter->hw);
 }
 
 /*********************************************************************
  *
  *  Determine hardware revision.
  *
  **********************************************************************/
 static void
 em_identify_hardware(if_ctx_t ctx)
 {
 	device_t dev = iflib_get_dev(ctx);
 	struct adapter *adapter = iflib_get_softc(ctx);
 
 	/* Make sure our PCI config space has the necessary stuff set */
 	adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
 
 	/* Save off the information about this board */
 	adapter->hw.vendor_id = pci_get_vendor(dev);
 	adapter->hw.device_id = pci_get_device(dev);
 	adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1);
 	adapter->hw.subsystem_vendor_id =
 	    pci_read_config(dev, PCIR_SUBVEND_0, 2);
 	adapter->hw.subsystem_device_id =
 	    pci_read_config(dev, PCIR_SUBDEV_0, 2);
 
 	/* Do Shared Code Init and Setup */
 	if (e1000_set_mac_type(&adapter->hw)) {
 		device_printf(dev, "Setup init failure\n");
 		return;
 	}
 }
 
 static int
 em_allocate_pci_resources(if_ctx_t ctx)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	device_t dev = iflib_get_dev(ctx);
 	int rid, val;
 
 	rid = PCIR_BAR(0);
 	adapter->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
 	    &rid, RF_ACTIVE);
 	if (adapter->memory == NULL) {
 		device_printf(dev, "Unable to allocate bus resource: memory\n");
 		return (ENXIO);
 	}
 	adapter->osdep.mem_bus_space_tag = rman_get_bustag(adapter->memory);
 	adapter->osdep.mem_bus_space_handle =
 	    rman_get_bushandle(adapter->memory);
 	adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle;
 
 	/* Only older adapters use IO mapping */
 	if (adapter->hw.mac.type < em_mac_min &&
 	    adapter->hw.mac.type > e1000_82543) {
 		/* Figure our where our IO BAR is ? */
 		for (rid = PCIR_BAR(0); rid < PCIR_CIS;) {
 			val = pci_read_config(dev, rid, 4);
 			if (EM_BAR_TYPE(val) == EM_BAR_TYPE_IO) {
 				break;
 			}
 			rid += 4;
 			/* check for 64bit BAR */
 			if (EM_BAR_MEM_TYPE(val) == EM_BAR_MEM_TYPE_64BIT)
 				rid += 4;
 		}
 		if (rid >= PCIR_CIS) {
 			device_printf(dev, "Unable to locate IO BAR\n");
 			return (ENXIO);
 		}
 		adapter->ioport = bus_alloc_resource_any(dev, SYS_RES_IOPORT,
 		    &rid, RF_ACTIVE);
 		if (adapter->ioport == NULL) {
 			device_printf(dev, "Unable to allocate bus resource: "
 			    "ioport\n");
 			return (ENXIO);
 		}
 		adapter->hw.io_base = 0;
 		adapter->osdep.io_bus_space_tag =
 		    rman_get_bustag(adapter->ioport);
 		adapter->osdep.io_bus_space_handle =
 		    rman_get_bushandle(adapter->ioport);
 	}
 
 	adapter->hw.back = &adapter->osdep;
 
 	return (0);
 }
 
 /*********************************************************************
  *
  *  Set up the MSI-X Interrupt handlers
  *
  **********************************************************************/
 static int
 em_if_msix_intr_assign(if_ctx_t ctx, int msix)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	struct em_rx_queue *rx_que = adapter->rx_queues;
 	struct em_tx_queue *tx_que = adapter->tx_queues;
 	int error, rid, i, vector = 0, rx_vectors;
 	char buf[16];
 
 	/* First set up ring resources */
 	for (i = 0; i < adapter->rx_num_queues; i++, rx_que++, vector++) {
 		rid = vector + 1;
 		snprintf(buf, sizeof(buf), "rxq%d", i);
 		error = iflib_irq_alloc_generic(ctx, &rx_que->que_irq, rid, IFLIB_INTR_RXTX, em_msix_que, rx_que, rx_que->me, buf);
 		if (error) {
 			device_printf(iflib_get_dev(ctx), "Failed to allocate que int %d err: %d", i, error);
 			adapter->rx_num_queues = i + 1;
 			goto fail;
 		}
 
 		rx_que->msix =  vector;
 
 		/*
 		 * Set the bit to enable interrupt
 		 * in E1000_IMS -- bits 20 and 21
 		 * are for RX0 and RX1, note this has
 		 * NOTHING to do with the MSI-X vector
 		 */
 		if (adapter->hw.mac.type == e1000_82574) {
 			rx_que->eims = 1 << (20 + i);
 			adapter->ims |= rx_que->eims;
 			adapter->ivars |= (8 | rx_que->msix) << (i * 4);
 		} else if (adapter->hw.mac.type == e1000_82575)
 			rx_que->eims = E1000_EICR_TX_QUEUE0 << vector;
 		else
 			rx_que->eims = 1 << vector;
 	}
 	rx_vectors = vector;
 
 	vector = 0;
 	for (i = 0; i < adapter->tx_num_queues; i++, tx_que++, vector++) {
 		snprintf(buf, sizeof(buf), "txq%d", i);
 		tx_que = &adapter->tx_queues[i];
 		iflib_softirq_alloc_generic(ctx,
 		    &adapter->rx_queues[i % adapter->rx_num_queues].que_irq,
 		    IFLIB_INTR_TX, tx_que, tx_que->me, buf);
 
 		tx_que->msix = (vector % adapter->rx_num_queues);
 
 		/*
 		 * Set the bit to enable interrupt
 		 * in E1000_IMS -- bits 22 and 23
 		 * are for TX0 and TX1, note this has
 		 * NOTHING to do with the MSI-X vector
 		 */
 		if (adapter->hw.mac.type == e1000_82574) {
 			tx_que->eims = 1 << (22 + i);
 			adapter->ims |= tx_que->eims;
 			adapter->ivars |= (8 | tx_que->msix) << (8 + (i * 4));
 		} else if (adapter->hw.mac.type == e1000_82575) {
 			tx_que->eims = E1000_EICR_TX_QUEUE0 << i;
 		} else {
 			tx_que->eims = 1 << i;
 		}
 	}
 
 	/* Link interrupt */
 	rid = rx_vectors + 1;
 	error = iflib_irq_alloc_generic(ctx, &adapter->irq, rid, IFLIB_INTR_ADMIN, em_msix_link, adapter, 0, "aq");
 
 	if (error) {
 		device_printf(iflib_get_dev(ctx), "Failed to register admin handler");
 		goto fail;
 	}
 	adapter->linkvec = rx_vectors;
 	if (adapter->hw.mac.type < igb_mac_min) {
 		adapter->ivars |=  (8 | rx_vectors) << 16;
 		adapter->ivars |= 0x80000000;
 	}
 	return (0);
 fail:
 	iflib_irq_free(ctx, &adapter->irq);
 	rx_que = adapter->rx_queues;
 	for (int i = 0; i < adapter->rx_num_queues; i++, rx_que++)
 		iflib_irq_free(ctx, &rx_que->que_irq);
 	return (error);
 }
 
 static void
 igb_configure_queues(struct adapter *adapter)
 {
 	struct e1000_hw *hw = &adapter->hw;
 	struct em_rx_queue *rx_que;
 	struct em_tx_queue *tx_que;
 	u32 tmp, ivar = 0, newitr = 0;
 
 	/* First turn on RSS capability */
 	if (adapter->hw.mac.type != e1000_82575)
 		E1000_WRITE_REG(hw, E1000_GPIE,
 		    E1000_GPIE_MSIX_MODE | E1000_GPIE_EIAME |
 		    E1000_GPIE_PBA | E1000_GPIE_NSICR);
 
 	/* Turn on MSI-X */
 	switch (adapter->hw.mac.type) {
 	case e1000_82580:
 	case e1000_i350:
 	case e1000_i354:
 	case e1000_i210:
 	case e1000_i211:
 	case e1000_vfadapt:
 	case e1000_vfadapt_i350:
 		/* RX entries */
 		for (int i = 0; i < adapter->rx_num_queues; i++) {
 			u32 index = i >> 1;
 			ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
 			rx_que = &adapter->rx_queues[i];
 			if (i & 1) {
 				ivar &= 0xFF00FFFF;
 				ivar |= (rx_que->msix | E1000_IVAR_VALID) << 16;
 			} else {
 				ivar &= 0xFFFFFF00;
 				ivar |= rx_que->msix | E1000_IVAR_VALID;
 			}
 			E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
 		}
 		/* TX entries */
 		for (int i = 0; i < adapter->tx_num_queues; i++) {
 			u32 index = i >> 1;
 			ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
 			tx_que = &adapter->tx_queues[i];
 			if (i & 1) {
 				ivar &= 0x00FFFFFF;
 				ivar |= (tx_que->msix | E1000_IVAR_VALID) << 24;
 			} else {
 				ivar &= 0xFFFF00FF;
 				ivar |= (tx_que->msix | E1000_IVAR_VALID) << 8;
 			}
 			E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
 			adapter->que_mask |= tx_que->eims;
 		}
 
 		/* And for the link interrupt */
 		ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8;
 		adapter->link_mask = 1 << adapter->linkvec;
 		E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
 		break;
 	case e1000_82576:
 		/* RX entries */
 		for (int i = 0; i < adapter->rx_num_queues; i++) {
 			u32 index = i & 0x7; /* Each IVAR has two entries */
 			ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
 			rx_que = &adapter->rx_queues[i];
 			if (i < 8) {
 				ivar &= 0xFFFFFF00;
 				ivar |= rx_que->msix | E1000_IVAR_VALID;
 			} else {
 				ivar &= 0xFF00FFFF;
 				ivar |= (rx_que->msix | E1000_IVAR_VALID) << 16;
 			}
 			E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
 			adapter->que_mask |= rx_que->eims;
 		}
 		/* TX entries */
 		for (int i = 0; i < adapter->tx_num_queues; i++) {
 			u32 index = i & 0x7; /* Each IVAR has two entries */
 			ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
 			tx_que = &adapter->tx_queues[i];
 			if (i < 8) {
 				ivar &= 0xFFFF00FF;
 				ivar |= (tx_que->msix | E1000_IVAR_VALID) << 8;
 			} else {
 				ivar &= 0x00FFFFFF;
 				ivar |= (tx_que->msix | E1000_IVAR_VALID) << 24;
 			}
 			E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
 			adapter->que_mask |= tx_que->eims;
 		}
 
 		/* And for the link interrupt */
 		ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8;
 		adapter->link_mask = 1 << adapter->linkvec;
 		E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
 		break;
 
 	case e1000_82575:
 		/* enable MSI-X support*/
 		tmp = E1000_READ_REG(hw, E1000_CTRL_EXT);
 		tmp |= E1000_CTRL_EXT_PBA_CLR;
 		/* Auto-Mask interrupts upon ICR read. */
 		tmp |= E1000_CTRL_EXT_EIAME;
 		tmp |= E1000_CTRL_EXT_IRCA;
 		E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmp);
 
 		/* Queues */
 		for (int i = 0; i < adapter->rx_num_queues; i++) {
 			rx_que = &adapter->rx_queues[i];
 			tmp = E1000_EICR_RX_QUEUE0 << i;
 			tmp |= E1000_EICR_TX_QUEUE0 << i;
 			rx_que->eims = tmp;
 			E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0),
 			    i, rx_que->eims);
 			adapter->que_mask |= rx_que->eims;
 		}
 
 		/* Link */
 		E1000_WRITE_REG(hw, E1000_MSIXBM(adapter->linkvec),
 		    E1000_EIMS_OTHER);
 		adapter->link_mask |= E1000_EIMS_OTHER;
 	default:
 		break;
 	}
 
 	/* Set the starting interrupt rate */
 	if (em_max_interrupt_rate > 0)
 		newitr = (4000000 / em_max_interrupt_rate) & 0x7FFC;
 
 	if (hw->mac.type == e1000_82575)
 		newitr |= newitr << 16;
 	else
 		newitr |= E1000_EITR_CNT_IGNR;
 
 	for (int i = 0; i < adapter->rx_num_queues; i++) {
 		rx_que = &adapter->rx_queues[i];
 		E1000_WRITE_REG(hw, E1000_EITR(rx_que->msix), newitr);
 	}
 
 	return;
 }
 
 static void
 em_free_pci_resources(if_ctx_t ctx)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	struct em_rx_queue *que = adapter->rx_queues;
 	device_t dev = iflib_get_dev(ctx);
 
 	/* Release all MSI-X queue resources */
 	if (adapter->intr_type == IFLIB_INTR_MSIX)
 		iflib_irq_free(ctx, &adapter->irq);
 
 	for (int i = 0; i < adapter->rx_num_queues; i++, que++) {
 		iflib_irq_free(ctx, &que->que_irq);
 	}
 
 	if (adapter->memory != NULL) {
 		bus_release_resource(dev, SYS_RES_MEMORY,
 		    rman_get_rid(adapter->memory), adapter->memory);
 		adapter->memory = NULL;
 	}
 
 	if (adapter->flash != NULL) {
 		bus_release_resource(dev, SYS_RES_MEMORY,
 		    rman_get_rid(adapter->flash), adapter->flash);
 		adapter->flash = NULL;
 	}
 
 	if (adapter->ioport != NULL) {
 		bus_release_resource(dev, SYS_RES_IOPORT,
 		    rman_get_rid(adapter->ioport), adapter->ioport);
 		adapter->ioport = NULL;
 	}
 }
 
 /* Set up MSI or MSI-X */
 static int
 em_setup_msix(if_ctx_t ctx)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 
 	if (adapter->hw.mac.type == e1000_82574) {
 		em_enable_vectors_82574(ctx);
 	}
 	return (0);
 }
 
 /*********************************************************************
  *
  *  Workaround for SmartSpeed on 82541 and 82547 controllers
  *
  **********************************************************************/
 static void
 lem_smartspeed(struct adapter *adapter)
 {
 	u16 phy_tmp;
 
 	if (adapter->link_active || (adapter->hw.phy.type != e1000_phy_igp) ||
 	    adapter->hw.mac.autoneg == 0 ||
 	    (adapter->hw.phy.autoneg_advertised & ADVERTISE_1000_FULL) == 0)
 		return;
 
 	if (adapter->smartspeed == 0) {
 		/* If Master/Slave config fault is asserted twice,
 		 * we assume back-to-back */
 		e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
 		if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT))
 			return;
 		e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
 		if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) {
 			e1000_read_phy_reg(&adapter->hw,
 			    PHY_1000T_CTRL, &phy_tmp);
 			if(phy_tmp & CR_1000T_MS_ENABLE) {
 				phy_tmp &= ~CR_1000T_MS_ENABLE;
 				e1000_write_phy_reg(&adapter->hw,
 				    PHY_1000T_CTRL, phy_tmp);
 				adapter->smartspeed++;
 				if(adapter->hw.mac.autoneg &&
 				   !e1000_copper_link_autoneg(&adapter->hw) &&
 				   !e1000_read_phy_reg(&adapter->hw,
 				    PHY_CONTROL, &phy_tmp)) {
 					phy_tmp |= (MII_CR_AUTO_NEG_EN |
 						    MII_CR_RESTART_AUTO_NEG);
 					e1000_write_phy_reg(&adapter->hw,
 					    PHY_CONTROL, phy_tmp);
 				}
 			}
 		}
 		return;
 	} else if(adapter->smartspeed == EM_SMARTSPEED_DOWNSHIFT) {
 		/* If still no link, perhaps using 2/3 pair cable */
 		e1000_read_phy_reg(&adapter->hw, PHY_1000T_CTRL, &phy_tmp);
 		phy_tmp |= CR_1000T_MS_ENABLE;
 		e1000_write_phy_reg(&adapter->hw, PHY_1000T_CTRL, phy_tmp);
 		if(adapter->hw.mac.autoneg &&
 		   !e1000_copper_link_autoneg(&adapter->hw) &&
 		   !e1000_read_phy_reg(&adapter->hw, PHY_CONTROL, &phy_tmp)) {
 			phy_tmp |= (MII_CR_AUTO_NEG_EN |
 				    MII_CR_RESTART_AUTO_NEG);
 			e1000_write_phy_reg(&adapter->hw, PHY_CONTROL, phy_tmp);
 		}
 	}
 	/* Restart process after EM_SMARTSPEED_MAX iterations */
 	if(adapter->smartspeed++ == EM_SMARTSPEED_MAX)
 		adapter->smartspeed = 0;
 }
 
 /*********************************************************************
  *
  *  Initialize the DMA Coalescing feature
  *
  **********************************************************************/
 static void
 igb_init_dmac(struct adapter *adapter, u32 pba)
 {
 	device_t	dev = adapter->dev;
 	struct e1000_hw *hw = &adapter->hw;
 	u32 		dmac, reg = ~E1000_DMACR_DMAC_EN;
 	u16		hwm;
 	u16		max_frame_size;
 
 	if (hw->mac.type == e1000_i211)
 		return;
 
 	max_frame_size = adapter->shared->isc_max_frame_size;
 	if (hw->mac.type > e1000_82580) {
 
 		if (adapter->dmac == 0) { /* Disabling it */
 			E1000_WRITE_REG(hw, E1000_DMACR, reg);
 			return;
 		} else
 			device_printf(dev, "DMA Coalescing enabled\n");
 
 		/* Set starting threshold */
 		E1000_WRITE_REG(hw, E1000_DMCTXTH, 0);
 
 		hwm = 64 * pba - max_frame_size / 16;
 		if (hwm < 64 * (pba - 6))
 			hwm = 64 * (pba - 6);
 		reg = E1000_READ_REG(hw, E1000_FCRTC);
 		reg &= ~E1000_FCRTC_RTH_COAL_MASK;
 		reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT)
 		    & E1000_FCRTC_RTH_COAL_MASK);
 		E1000_WRITE_REG(hw, E1000_FCRTC, reg);
 
 
 		dmac = pba - max_frame_size / 512;
 		if (dmac < pba - 10)
 			dmac = pba - 10;
 		reg = E1000_READ_REG(hw, E1000_DMACR);
 		reg &= ~E1000_DMACR_DMACTHR_MASK;
 		reg |= ((dmac << E1000_DMACR_DMACTHR_SHIFT)
 		    & E1000_DMACR_DMACTHR_MASK);
 
 		/* transition to L0x or L1 if available..*/
 		reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);
 
 		/* Check if status is 2.5Gb backplane connection
 		* before configuration of watchdog timer, which is
 		* in msec values in 12.8usec intervals
 		* watchdog timer= msec values in 32usec intervals
 		* for non 2.5Gb connection
 		*/
 		if (hw->mac.type == e1000_i354) {
 			int status = E1000_READ_REG(hw, E1000_STATUS);
 			if ((status & E1000_STATUS_2P5_SKU) &&
 			    (!(status & E1000_STATUS_2P5_SKU_OVER)))
 				reg |= ((adapter->dmac * 5) >> 6);
 			else
 				reg |= (adapter->dmac >> 5);
 		} else {
 			reg |= (adapter->dmac >> 5);
 		}
 
 		E1000_WRITE_REG(hw, E1000_DMACR, reg);
 
 		E1000_WRITE_REG(hw, E1000_DMCRTRH, 0);
 
 		/* Set the interval before transition */
 		reg = E1000_READ_REG(hw, E1000_DMCTLX);
 		if (hw->mac.type == e1000_i350)
 			reg |= IGB_DMCTLX_DCFLUSH_DIS;
 		/*
 		** in 2.5Gb connection, TTLX unit is 0.4 usec
 		** which is 0x4*2 = 0xA. But delay is still 4 usec
 		*/
 		if (hw->mac.type == e1000_i354) {
 			int status = E1000_READ_REG(hw, E1000_STATUS);
 			if ((status & E1000_STATUS_2P5_SKU) &&
 			    (!(status & E1000_STATUS_2P5_SKU_OVER)))
 				reg |= 0xA;
 			else
 				reg |= 0x4;
 		} else {
 			reg |= 0x4;
 		}
 
 		E1000_WRITE_REG(hw, E1000_DMCTLX, reg);
 
 		/* free space in tx packet buffer to wake from DMA coal */
 		E1000_WRITE_REG(hw, E1000_DMCTXTH, (IGB_TXPBSIZE -
 		    (2 * max_frame_size)) >> 6);
 
 		/* make low power state decision controlled by DMA coal */
 		reg = E1000_READ_REG(hw, E1000_PCIEMISC);
 		reg &= ~E1000_PCIEMISC_LX_DECISION;
 		E1000_WRITE_REG(hw, E1000_PCIEMISC, reg);
 
 	} else if (hw->mac.type == e1000_82580) {
 		u32 reg = E1000_READ_REG(hw, E1000_PCIEMISC);
 		E1000_WRITE_REG(hw, E1000_PCIEMISC,
 		    reg & ~E1000_PCIEMISC_LX_DECISION);
 		E1000_WRITE_REG(hw, E1000_DMACR, 0);
 	}
 }
 
 /*********************************************************************
  *
  *  Initialize the hardware to a configuration as specified by the
  *  adapter structure.
  *
  **********************************************************************/
 static void
 em_reset(if_ctx_t ctx)
 {
 	device_t dev = iflib_get_dev(ctx);
 	struct adapter *adapter = iflib_get_softc(ctx);
 	struct ifnet *ifp = iflib_get_ifp(ctx);
 	struct e1000_hw *hw = &adapter->hw;
 	u16 rx_buffer_size;
 	u32 pba;
 
 	INIT_DEBUGOUT("em_reset: begin");
 	/* Let the firmware know the OS is in control */
 	em_get_hw_control(adapter);
 
 	/* Set up smart power down as default off on newer adapters. */
 	if (!em_smart_pwr_down && (hw->mac.type == e1000_82571 ||
 	    hw->mac.type == e1000_82572)) {
 		u16 phy_tmp = 0;
 
 		/* Speed up time to link by disabling smart power down. */
 		e1000_read_phy_reg(hw, IGP02E1000_PHY_POWER_MGMT, &phy_tmp);
 		phy_tmp &= ~IGP02E1000_PM_SPD;
 		e1000_write_phy_reg(hw, IGP02E1000_PHY_POWER_MGMT, phy_tmp);
 	}
 
 	/*
 	 * Packet Buffer Allocation (PBA)
 	 * Writing PBA sets the receive portion of the buffer
 	 * the remainder is used for the transmit buffer.
 	 */
 	switch (hw->mac.type) {
 	/* Total Packet Buffer on these is 48K */
 	case e1000_82571:
 	case e1000_82572:
 	case e1000_80003es2lan:
 			pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */
 		break;
 	case e1000_82573: /* 82573: Total Packet Buffer is 32K */
 			pba = E1000_PBA_12K; /* 12K for Rx, 20K for Tx */
 		break;
 	case e1000_82574:
 	case e1000_82583:
 			pba = E1000_PBA_20K; /* 20K for Rx, 20K for Tx */
 		break;
 	case e1000_ich8lan:
 		pba = E1000_PBA_8K;
 		break;
 	case e1000_ich9lan:
 	case e1000_ich10lan:
 		/* Boost Receive side for jumbo frames */
 		if (adapter->hw.mac.max_frame_size > 4096)
 			pba = E1000_PBA_14K;
 		else
 			pba = E1000_PBA_10K;
 		break;
 	case e1000_pchlan:
 	case e1000_pch2lan:
 	case e1000_pch_lpt:
 	case e1000_pch_spt:
 	case e1000_pch_cnp:
 		pba = E1000_PBA_26K;
 		break;
 	case e1000_82575:
 		pba = E1000_PBA_32K;
 		break;
 	case e1000_82576:
 	case e1000_vfadapt:
 		pba = E1000_READ_REG(hw, E1000_RXPBS);
 		pba &= E1000_RXPBS_SIZE_MASK_82576;
 		break;
 	case e1000_82580:
 	case e1000_i350:
 	case e1000_i354:
 	case e1000_vfadapt_i350:
 		pba = E1000_READ_REG(hw, E1000_RXPBS);
 		pba = e1000_rxpbs_adjust_82580(pba);
 		break;
 	case e1000_i210:
 	case e1000_i211:
 		pba = E1000_PBA_34K;
 		break;
 	default:
 		if (adapter->hw.mac.max_frame_size > 8192)
 			pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */
 		else
 			pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */
 	}
 
 	/* Special needs in case of Jumbo frames */
 	if ((hw->mac.type == e1000_82575) && (ifp->if_mtu > ETHERMTU)) {
 		u32 tx_space, min_tx, min_rx;
 		pba = E1000_READ_REG(hw, E1000_PBA);
 		tx_space = pba >> 16;
 		pba &= 0xffff;
 		min_tx = (adapter->hw.mac.max_frame_size +
 		    sizeof(struct e1000_tx_desc) - ETHERNET_FCS_SIZE) * 2;
 		min_tx = roundup2(min_tx, 1024);
 		min_tx >>= 10;
 		min_rx = adapter->hw.mac.max_frame_size;
 		min_rx = roundup2(min_rx, 1024);
 		min_rx >>= 10;
 		if (tx_space < min_tx &&
 		    ((min_tx - tx_space) < pba)) {
 			pba = pba - (min_tx - tx_space);
 			/*
 			 * if short on rx space, rx wins
 			 * and must trump tx adjustment
 			 */
 			if (pba < min_rx)
 				pba = min_rx;
 		}
 		E1000_WRITE_REG(hw, E1000_PBA, pba);
 	}
 
 	if (hw->mac.type < igb_mac_min)
 		E1000_WRITE_REG(&adapter->hw, E1000_PBA, pba);
 
 	INIT_DEBUGOUT1("em_reset: pba=%dK",pba);
 
 	/*
 	 * These parameters control the automatic generation (Tx) and
 	 * response (Rx) to Ethernet PAUSE frames.
 	 * - High water mark should allow for at least two frames to be
 	 *   received after sending an XOFF.
 	 * - Low water mark works best when it is very near the high water mark.
 	 *   This allows the receiver to restart by sending XON when it has
 	 *   drained a bit. Here we use an arbitrary value of 1500 which will
 	 *   restart after one full frame is pulled from the buffer. There
 	 *   could be several smaller frames in the buffer and if so they will
 	 *   not trigger the XON until their total number reduces the buffer
 	 *   by 1500.
 	 * - The pause time is fairly large at 1000 x 512ns = 512 usec.
 	 */
 	rx_buffer_size = (pba & 0xffff) << 10;
 	hw->fc.high_water = rx_buffer_size -
 	    roundup2(adapter->hw.mac.max_frame_size, 1024);
 	hw->fc.low_water = hw->fc.high_water - 1500;
 
 	if (adapter->fc) /* locally set flow control value? */
 		hw->fc.requested_mode = adapter->fc;
 	else
 		hw->fc.requested_mode = e1000_fc_full;
 
 	if (hw->mac.type == e1000_80003es2lan)
 		hw->fc.pause_time = 0xFFFF;
 	else
 		hw->fc.pause_time = EM_FC_PAUSE_TIME;
 
 	hw->fc.send_xon = TRUE;
 
 	/* Device specific overrides/settings */
 	switch (hw->mac.type) {
 	case e1000_pchlan:
 		/* Workaround: no TX flow ctrl for PCH */
 		hw->fc.requested_mode = e1000_fc_rx_pause;
 		hw->fc.pause_time = 0xFFFF; /* override */
 		if (if_getmtu(ifp) > ETHERMTU) {
 			hw->fc.high_water = 0x3500;
 			hw->fc.low_water = 0x1500;
 		} else {
 			hw->fc.high_water = 0x5000;
 			hw->fc.low_water = 0x3000;
 		}
 		hw->fc.refresh_time = 0x1000;
 		break;
 	case e1000_pch2lan:
 	case e1000_pch_lpt:
 	case e1000_pch_spt:
 	case e1000_pch_cnp:
 		hw->fc.high_water = 0x5C20;
 		hw->fc.low_water = 0x5048;
 		hw->fc.pause_time = 0x0650;
 		hw->fc.refresh_time = 0x0400;
 		/* Jumbos need adjusted PBA */
 		if (if_getmtu(ifp) > ETHERMTU)
 			E1000_WRITE_REG(hw, E1000_PBA, 12);
 		else
 			E1000_WRITE_REG(hw, E1000_PBA, 26);
 		break;
 	case e1000_82575:
 	case e1000_82576:
 		/* 8-byte granularity */
 		hw->fc.low_water = hw->fc.high_water - 8;
 		break;
 	case e1000_82580:
 	case e1000_i350:
 	case e1000_i354:
 	case e1000_i210:
 	case e1000_i211:
 	case e1000_vfadapt:
 	case e1000_vfadapt_i350:
 		/* 16-byte granularity */
 		hw->fc.low_water = hw->fc.high_water - 16;
 		break;
 	case e1000_ich9lan:
 	case e1000_ich10lan:
 		if (if_getmtu(ifp) > ETHERMTU) {
 			hw->fc.high_water = 0x2800;
 			hw->fc.low_water = hw->fc.high_water - 8;
 			break;
 		}
 		/* FALLTHROUGH */
 	default:
 		if (hw->mac.type == e1000_80003es2lan)
 			hw->fc.pause_time = 0xFFFF;
 		break;
 	}
 
 	/* Issue a global reset */
 	e1000_reset_hw(hw);
 	if (adapter->hw.mac.type >= igb_mac_min) {
 		E1000_WRITE_REG(hw, E1000_WUC, 0);
 	} else {
 		E1000_WRITE_REG(hw, E1000_WUFC, 0);
 		em_disable_aspm(adapter);
 	}
 	if (adapter->flags & IGB_MEDIA_RESET) {
 		e1000_setup_init_funcs(hw, TRUE);
 		e1000_get_bus_info(hw);
 		adapter->flags &= ~IGB_MEDIA_RESET;
 	}
 	/* and a re-init */
 	if (e1000_init_hw(hw) < 0) {
 		device_printf(dev, "Hardware Initialization Failed\n");
 		return;
 	}
 	if (adapter->hw.mac.type >= igb_mac_min)
 		igb_init_dmac(adapter, pba);
 
 	E1000_WRITE_REG(hw, E1000_VET, ETHERTYPE_VLAN);
 	e1000_get_phy_info(hw);
 	e1000_check_for_link(hw);
 }
 
 /*
  * Initialise the RSS mapping for NICs that support multiple transmit/
  * receive rings.
  */
 
 #define RSSKEYLEN 10
 static void
 em_initialize_rss_mapping(struct adapter *adapter)
 {
 	uint8_t  rss_key[4 * RSSKEYLEN];
 	uint32_t reta = 0;
 	struct e1000_hw	*hw = &adapter->hw;
 	int i;
 
 	/*
 	 * Configure RSS key
 	 */
 	arc4rand(rss_key, sizeof(rss_key), 0);
 	for (i = 0; i < RSSKEYLEN; ++i) {
 		uint32_t rssrk = 0;
 
 		rssrk = EM_RSSRK_VAL(rss_key, i);
 		E1000_WRITE_REG(hw,E1000_RSSRK(i), rssrk);
 	}
 
 	/*
 	 * Configure RSS redirect table in following fashion:
 	 * (hash & ring_cnt_mask) == rdr_table[(hash & rdr_table_mask)]
 	 */
 	for (i = 0; i < sizeof(reta); ++i) {
 		uint32_t q;
 
 		q = (i % adapter->rx_num_queues) << 7;
 		reta |= q << (8 * i);
 	}
 
 	for (i = 0; i < 32; ++i)
 		E1000_WRITE_REG(hw, E1000_RETA(i), reta);
 
 	E1000_WRITE_REG(hw, E1000_MRQC, E1000_MRQC_RSS_ENABLE_2Q |
 			E1000_MRQC_RSS_FIELD_IPV4_TCP |
 			E1000_MRQC_RSS_FIELD_IPV4 |
 			E1000_MRQC_RSS_FIELD_IPV6_TCP_EX |
 			E1000_MRQC_RSS_FIELD_IPV6_EX |
 			E1000_MRQC_RSS_FIELD_IPV6);
 }
 
 static void
 igb_initialize_rss_mapping(struct adapter *adapter)
 {
 	struct e1000_hw *hw = &adapter->hw;
 	int i;
 	int queue_id;
 	u32 reta;
 	u32 rss_key[10], mrqc, shift = 0;
 
 	/* XXX? */
 	if (adapter->hw.mac.type == e1000_82575)
 		shift = 6;
 
 	/*
 	 * The redirection table controls which destination
 	 * queue each bucket redirects traffic to.
 	 * Each DWORD represents four queues, with the LSB
 	 * being the first queue in the DWORD.
 	 *
 	 * This just allocates buckets to queues using round-robin
 	 * allocation.
 	 *
 	 * NOTE: It Just Happens to line up with the default
 	 * RSS allocation method.
 	 */
 
 	/* Warning FM follows */
 	reta = 0;
 	for (i = 0; i < 128; i++) {
 #ifdef RSS
 		queue_id = rss_get_indirection_to_bucket(i);
 		/*
 		 * If we have more queues than buckets, we'll
 		 * end up mapping buckets to a subset of the
 		 * queues.
 		 *
 		 * If we have more buckets than queues, we'll
 		 * end up instead assigning multiple buckets
 		 * to queues.
 		 *
 		 * Both are suboptimal, but we need to handle
 		 * the case so we don't go out of bounds
 		 * indexing arrays and such.
 		 */
 		queue_id = queue_id % adapter->rx_num_queues;
 #else
 		queue_id = (i % adapter->rx_num_queues);
 #endif
 		/* Adjust if required */
 		queue_id = queue_id << shift;
 
 		/*
 		 * The low 8 bits are for hash value (n+0);
 		 * The next 8 bits are for hash value (n+1), etc.
 		 */
 		reta = reta >> 8;
 		reta = reta | ( ((uint32_t) queue_id) << 24);
 		if ((i & 3) == 3) {
 			E1000_WRITE_REG(hw, E1000_RETA(i >> 2), reta);
 			reta = 0;
 		}
 	}
 
 	/* Now fill in hash table */
 
 	/*
 	 * MRQC: Multiple Receive Queues Command
 	 * Set queuing to RSS control, number depends on the device.
 	 */
 	mrqc = E1000_MRQC_ENABLE_RSS_8Q;
 
 #ifdef RSS
 	/* XXX ew typecasting */
 	rss_getkey((uint8_t *) &rss_key);
 #else
 	arc4rand(&rss_key, sizeof(rss_key), 0);
 #endif
 	for (i = 0; i < 10; i++)
 		E1000_WRITE_REG_ARRAY(hw, E1000_RSSRK(0), i, rss_key[i]);
 
 	/*
 	 * Configure the RSS fields to hash upon.
 	 */
 	mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
 	    E1000_MRQC_RSS_FIELD_IPV4_TCP);
 	mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 |
 	    E1000_MRQC_RSS_FIELD_IPV6_TCP);
 	mrqc |=( E1000_MRQC_RSS_FIELD_IPV4_UDP |
 	    E1000_MRQC_RSS_FIELD_IPV6_UDP);
 	mrqc |=( E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
 	    E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
 
 	E1000_WRITE_REG(hw, E1000_MRQC, mrqc);
 }
 
 /*********************************************************************
  *
  *  Setup networking device structure and register interface media.
  *
  **********************************************************************/
 static int
 em_setup_interface(if_ctx_t ctx)
 {
 	struct ifnet *ifp = iflib_get_ifp(ctx);
 	struct adapter *adapter = iflib_get_softc(ctx);
 	if_softc_ctx_t scctx = adapter->shared;
 
 	INIT_DEBUGOUT("em_setup_interface: begin");
 
 	/* Single Queue */
 	if (adapter->tx_num_queues == 1) {
 		if_setsendqlen(ifp, scctx->isc_ntxd[0] - 1);
 		if_setsendqready(ifp);
 	}
 
 	/*
 	 * Specify the media types supported by this adapter and register
 	 * callbacks to update media and link information
 	 */
 	if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
 	    (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
 		u_char fiber_type = IFM_1000_SX;	/* default type */
 
 		if (adapter->hw.mac.type == e1000_82545)
 			fiber_type = IFM_1000_LX;
 		ifmedia_add(adapter->media, IFM_ETHER | fiber_type | IFM_FDX, 0, NULL);
 		ifmedia_add(adapter->media, IFM_ETHER | fiber_type, 0, NULL);
 	} else {
 		ifmedia_add(adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
 		ifmedia_add(adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX, 0, NULL);
 		ifmedia_add(adapter->media, IFM_ETHER | IFM_100_TX, 0, NULL);
 		ifmedia_add(adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 0, NULL);
 		if (adapter->hw.phy.type != e1000_phy_ife) {
 			ifmedia_add(adapter->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
 			ifmedia_add(adapter->media, IFM_ETHER | IFM_1000_T, 0, NULL);
 		}
 	}
 	ifmedia_add(adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
 	ifmedia_set(adapter->media, IFM_ETHER | IFM_AUTO);
 	return (0);
 }
 
 static int
 em_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int ntxqs, int ntxqsets)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	if_softc_ctx_t scctx = adapter->shared;
 	int error = E1000_SUCCESS;
 	struct em_tx_queue *que;
 	int i, j;
 
 	MPASS(adapter->tx_num_queues > 0);
 	MPASS(adapter->tx_num_queues == ntxqsets);
 
 	/* First allocate the top level queue structs */
 	if (!(adapter->tx_queues =
 	    (struct em_tx_queue *) malloc(sizeof(struct em_tx_queue) *
 	    adapter->tx_num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
 		device_printf(iflib_get_dev(ctx), "Unable to allocate queue memory\n");
 		return(ENOMEM);
 	}
 
 	for (i = 0, que = adapter->tx_queues; i < adapter->tx_num_queues; i++, que++) {
 		/* Set up some basics */
 
 		struct tx_ring *txr = &que->txr;
 		txr->adapter = que->adapter = adapter;
 		que->me = txr->me =  i;
 
 		/* Allocate report status array */
 		if (!(txr->tx_rsq = (qidx_t *) malloc(sizeof(qidx_t) * scctx->isc_ntxd[0], M_DEVBUF, M_NOWAIT | M_ZERO))) {
 			device_printf(iflib_get_dev(ctx), "failed to allocate rs_idxs memory\n");
 			error = ENOMEM;
 			goto fail;
 		}
 		for (j = 0; j < scctx->isc_ntxd[0]; j++)
 			txr->tx_rsq[j] = QIDX_INVALID;
 		/* get the virtual and physical address of the hardware queues */
 		txr->tx_base = (struct e1000_tx_desc *)vaddrs[i*ntxqs];
 		txr->tx_paddr = paddrs[i*ntxqs];
 	}
 
 	if (bootverbose)
 		device_printf(iflib_get_dev(ctx),
 		    "allocated for %d tx_queues\n", adapter->tx_num_queues);
 	return (0);
 fail:
 	em_if_queues_free(ctx);
 	return (error);
 }
 
 static int
 em_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nrxqs, int nrxqsets)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	int error = E1000_SUCCESS;
 	struct em_rx_queue *que;
 	int i;
 
 	MPASS(adapter->rx_num_queues > 0);
 	MPASS(adapter->rx_num_queues == nrxqsets);
 
 	/* First allocate the top level queue structs */
 	if (!(adapter->rx_queues =
 	    (struct em_rx_queue *) malloc(sizeof(struct em_rx_queue) *
 	    adapter->rx_num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
 		device_printf(iflib_get_dev(ctx), "Unable to allocate queue memory\n");
 		error = ENOMEM;
 		goto fail;
 	}
 
 	for (i = 0, que = adapter->rx_queues; i < nrxqsets; i++, que++) {
 		/* Set up some basics */
 		struct rx_ring *rxr = &que->rxr;
 		rxr->adapter = que->adapter = adapter;
 		rxr->que = que;
 		que->me = rxr->me =  i;
 
 		/* get the virtual and physical address of the hardware queues */
 		rxr->rx_base = (union e1000_rx_desc_extended *)vaddrs[i*nrxqs];
 		rxr->rx_paddr = paddrs[i*nrxqs];
 	}
  
 	if (bootverbose)
 		device_printf(iflib_get_dev(ctx),
 		    "allocated for %d rx_queues\n", adapter->rx_num_queues);
 
 	return (0);
 fail:
 	em_if_queues_free(ctx);
 	return (error);
 }
 
 static void
 em_if_queues_free(if_ctx_t ctx)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	struct em_tx_queue *tx_que = adapter->tx_queues;
 	struct em_rx_queue *rx_que = adapter->rx_queues;
 
 	if (tx_que != NULL) {
 		for (int i = 0; i < adapter->tx_num_queues; i++, tx_que++) {
 			struct tx_ring *txr = &tx_que->txr;
 			if (txr->tx_rsq == NULL)
 				break;
 
 			free(txr->tx_rsq, M_DEVBUF);
 			txr->tx_rsq = NULL;
 		}
 		free(adapter->tx_queues, M_DEVBUF);
 		adapter->tx_queues = NULL;
 	}
 
 	if (rx_que != NULL) {
 		free(adapter->rx_queues, M_DEVBUF);
 		adapter->rx_queues = NULL;
 	}
 
 	em_release_hw_control(adapter);
 
 	if (adapter->mta != NULL) {
 		free(adapter->mta, M_DEVBUF);
 	}
 }
 
 /*********************************************************************
  *
  *  Enable transmit unit.
  *
  **********************************************************************/
 static void
 em_initialize_transmit_unit(if_ctx_t ctx)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	if_softc_ctx_t scctx = adapter->shared;
 	struct em_tx_queue *que;
 	struct tx_ring	*txr;
 	struct e1000_hw	*hw = &adapter->hw;
 	u32 tctl, txdctl = 0, tarc, tipg = 0;
 
 	INIT_DEBUGOUT("em_initialize_transmit_unit: begin");
 
 	for (int i = 0; i < adapter->tx_num_queues; i++, txr++) {
 		u64 bus_addr;
 		caddr_t offp, endp;
 
 		que = &adapter->tx_queues[i];
 		txr = &que->txr;
 		bus_addr = txr->tx_paddr;
 
 		/* Clear checksum offload context. */
 		offp = (caddr_t)&txr->csum_flags;
 		endp = (caddr_t)(txr + 1);
 		bzero(offp, endp - offp);
 
 		/* Base and Len of TX Ring */
 		E1000_WRITE_REG(hw, E1000_TDLEN(i),
 		    scctx->isc_ntxd[0] * sizeof(struct e1000_tx_desc));
 		E1000_WRITE_REG(hw, E1000_TDBAH(i),
 		    (u32)(bus_addr >> 32));
 		E1000_WRITE_REG(hw, E1000_TDBAL(i),
 		    (u32)bus_addr);
 		/* Init the HEAD/TAIL indices */
 		E1000_WRITE_REG(hw, E1000_TDT(i), 0);
 		E1000_WRITE_REG(hw, E1000_TDH(i), 0);
 
 		HW_DEBUGOUT2("Base = %x, Length = %x\n",
 		    E1000_READ_REG(&adapter->hw, E1000_TDBAL(i)),
 		    E1000_READ_REG(&adapter->hw, E1000_TDLEN(i)));
 
 		txdctl = 0; /* clear txdctl */
 		txdctl |= 0x1f; /* PTHRESH */
 		txdctl |= 1 << 8; /* HTHRESH */
 		txdctl |= 1 << 16;/* WTHRESH */
 		txdctl |= 1 << 22; /* Reserved bit 22 must always be 1 */
 		txdctl |= E1000_TXDCTL_GRAN;
 		txdctl |= 1 << 25; /* LWTHRESH */
 
 		E1000_WRITE_REG(hw, E1000_TXDCTL(i), txdctl);
 	}
 
 	/* Set the default values for the Tx Inter Packet Gap timer */
 	switch (adapter->hw.mac.type) {
 	case e1000_80003es2lan:
 		tipg = DEFAULT_82543_TIPG_IPGR1;
 		tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 <<
 		    E1000_TIPG_IPGR2_SHIFT;
 		break;
 	case e1000_82542:
 		tipg = DEFAULT_82542_TIPG_IPGT;
 		tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
 		tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
 		break;
 	default:
 		if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
 		    (adapter->hw.phy.media_type ==
 		    e1000_media_type_internal_serdes))
 			tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
 		else
 			tipg = DEFAULT_82543_TIPG_IPGT_COPPER;
 		tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
 		tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
 	}
 
 	E1000_WRITE_REG(&adapter->hw, E1000_TIPG, tipg);
 	E1000_WRITE_REG(&adapter->hw, E1000_TIDV, adapter->tx_int_delay.value);
 
 	if(adapter->hw.mac.type >= e1000_82540)
 		E1000_WRITE_REG(&adapter->hw, E1000_TADV,
 		    adapter->tx_abs_int_delay.value);
 
 	if ((adapter->hw.mac.type == e1000_82571) ||
 	    (adapter->hw.mac.type == e1000_82572)) {
 		tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
 		tarc |= TARC_SPEED_MODE_BIT;
 		E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
 	} else if (adapter->hw.mac.type == e1000_80003es2lan) {
 		/* errata: program both queues to unweighted RR */
 		tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
 		tarc |= 1;
 		E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
 		tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(1));
 		tarc |= 1;
 		E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc);
 	} else if (adapter->hw.mac.type == e1000_82574) {
 		tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
 		tarc |= TARC_ERRATA_BIT;
 		if ( adapter->tx_num_queues > 1) {
 			tarc |= (TARC_COMPENSATION_MODE | TARC_MQ_FIX);
 			E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
 			E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc);
 		} else
 			E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
 	}
 
 	if (adapter->tx_int_delay.value > 0)
 		adapter->txd_cmd |= E1000_TXD_CMD_IDE;
 
 	/* Program the Transmit Control Register */
 	tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL);
 	tctl &= ~E1000_TCTL_CT;
 	tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN |
 		   (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT));
 
 	if (adapter->hw.mac.type >= e1000_82571)
 		tctl |= E1000_TCTL_MULR;
 
 	/* This write will effectively turn on the transmit unit. */
 	E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl);
 
 	/* SPT and KBL errata workarounds */
 	if (hw->mac.type == e1000_pch_spt) {
 		u32 reg;
 		reg = E1000_READ_REG(hw, E1000_IOSFPC);
 		reg |= E1000_RCTL_RDMTS_HEX;
 		E1000_WRITE_REG(hw, E1000_IOSFPC, reg);
 		/* i218-i219 Specification Update 1.5.4.5 */
 		reg = E1000_READ_REG(hw, E1000_TARC(0));
 		reg &= ~E1000_TARC0_CB_MULTIQ_3_REQ;
 		reg |= E1000_TARC0_CB_MULTIQ_2_REQ;
 		E1000_WRITE_REG(hw, E1000_TARC(0), reg);
 	}
 }
 
 /*********************************************************************
  *
  *  Enable receive unit.
  *
  **********************************************************************/
 
 static void
 em_initialize_receive_unit(if_ctx_t ctx)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	if_softc_ctx_t scctx = adapter->shared;
 	struct ifnet *ifp = iflib_get_ifp(ctx);
 	struct e1000_hw	*hw = &adapter->hw;
 	struct em_rx_queue *que;
 	int i;
 	u32 rctl, rxcsum, rfctl;
 
 	INIT_DEBUGOUT("em_initialize_receive_units: begin");
 
 	/*
 	 * Make sure receives are disabled while setting
 	 * up the descriptor ring
 	 */
 	rctl = E1000_READ_REG(hw, E1000_RCTL);
 	/* Do not disable if ever enabled on this hardware */
 	if ((hw->mac.type != e1000_82574) && (hw->mac.type != e1000_82583))
 		E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
 
 	/* Setup the Receive Control Register */
 	rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
 	rctl |= E1000_RCTL_EN | E1000_RCTL_BAM |
 	    E1000_RCTL_LBM_NO | E1000_RCTL_RDMTS_HALF |
 	    (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
 
 	/* Do not store bad packets */
 	rctl &= ~E1000_RCTL_SBP;
 
 	/* Enable Long Packet receive */
 	if (if_getmtu(ifp) > ETHERMTU)
 		rctl |= E1000_RCTL_LPE;
 	else
 		rctl &= ~E1000_RCTL_LPE;
 
 	/* Strip the CRC */
 	if (!em_disable_crc_stripping)
 		rctl |= E1000_RCTL_SECRC;
 
 	if (adapter->hw.mac.type >= e1000_82540) {
 		E1000_WRITE_REG(&adapter->hw, E1000_RADV,
 			    adapter->rx_abs_int_delay.value);
 
 		/*
 		 * Set the interrupt throttling rate. Value is calculated
 		 * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns)
 		 */
 		E1000_WRITE_REG(hw, E1000_ITR, DEFAULT_ITR);
 	}
 	E1000_WRITE_REG(&adapter->hw, E1000_RDTR,
 	    adapter->rx_int_delay.value);
 
 	/* Use extended rx descriptor formats */
 	rfctl = E1000_READ_REG(hw, E1000_RFCTL);
 	rfctl |= E1000_RFCTL_EXTEN;
 	/*
 	 * When using MSI-X interrupts we need to throttle
 	 * using the EITR register (82574 only)
 	 */
 	if (hw->mac.type == e1000_82574) {
 		for (int i = 0; i < 4; i++)
 			E1000_WRITE_REG(hw, E1000_EITR_82574(i),
 			    DEFAULT_ITR);
 		/* Disable accelerated acknowledge */
 		rfctl |= E1000_RFCTL_ACK_DIS;
 	}
 	E1000_WRITE_REG(hw, E1000_RFCTL, rfctl);
 
 	rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
 	if (if_getcapenable(ifp) & IFCAP_RXCSUM &&
 	    adapter->hw.mac.type >= e1000_82543) {
 		if (adapter->tx_num_queues > 1) {
 			if (adapter->hw.mac.type >= igb_mac_min) {
 				rxcsum |= E1000_RXCSUM_PCSD;
 				if (hw->mac.type != e1000_82575)
 					rxcsum |= E1000_RXCSUM_CRCOFL;
 			} else
 				rxcsum |= E1000_RXCSUM_TUOFL |
 					E1000_RXCSUM_IPOFL |
 					E1000_RXCSUM_PCSD;
 		} else {
 			if (adapter->hw.mac.type >= igb_mac_min)
 				rxcsum |= E1000_RXCSUM_IPPCSE;
 			else
 				rxcsum |= E1000_RXCSUM_TUOFL | E1000_RXCSUM_IPOFL;
 			if (adapter->hw.mac.type > e1000_82575)
 				rxcsum |= E1000_RXCSUM_CRCOFL;
 		}
 	} else
 		rxcsum &= ~E1000_RXCSUM_TUOFL;
 
 	E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
 
 	if (adapter->rx_num_queues > 1) {
 		if (adapter->hw.mac.type >= igb_mac_min)
 			igb_initialize_rss_mapping(adapter);
 		else
 			em_initialize_rss_mapping(adapter);
 	}
 
 	/*
 	 * XXX TEMPORARY WORKAROUND: on some systems with 82573
 	 * long latencies are observed, like Lenovo X60. This
 	 * change eliminates the problem, but since having positive
 	 * values in RDTR is a known source of problems on other
 	 * platforms another solution is being sought.
 	 */
 	if (hw->mac.type == e1000_82573)
 		E1000_WRITE_REG(hw, E1000_RDTR, 0x20);
 
 	for (i = 0, que = adapter->rx_queues; i < adapter->rx_num_queues; i++, que++) {
 		struct rx_ring *rxr = &que->rxr;
 		/* Setup the Base and Length of the Rx Descriptor Ring */
 		u64 bus_addr = rxr->rx_paddr;
 #if 0
 		u32 rdt = adapter->rx_num_queues -1;  /* default */
 #endif
 
 		E1000_WRITE_REG(hw, E1000_RDLEN(i),
 		    scctx->isc_nrxd[0] * sizeof(union e1000_rx_desc_extended));
 		E1000_WRITE_REG(hw, E1000_RDBAH(i), (u32)(bus_addr >> 32));
 		E1000_WRITE_REG(hw, E1000_RDBAL(i), (u32)bus_addr);
 		/* Setup the Head and Tail Descriptor Pointers */
 		E1000_WRITE_REG(hw, E1000_RDH(i), 0);
 		E1000_WRITE_REG(hw, E1000_RDT(i), 0);
 	}
 
 	/*
 	 * Set PTHRESH for improved jumbo performance
 	 * According to 10.2.5.11 of Intel 82574 Datasheet,
 	 * RXDCTL(1) is written whenever RXDCTL(0) is written.
 	 * Only write to RXDCTL(1) if there is a need for different
 	 * settings.
 	 */
 
 	if (((adapter->hw.mac.type == e1000_ich9lan) ||
 	    (adapter->hw.mac.type == e1000_pch2lan) ||
 	    (adapter->hw.mac.type == e1000_ich10lan)) &&
 	    (if_getmtu(ifp) > ETHERMTU)) {
 		u32 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(0));
 		E1000_WRITE_REG(hw, E1000_RXDCTL(0), rxdctl | 3);
 	} else if (adapter->hw.mac.type == e1000_82574) {
 		for (int i = 0; i < adapter->rx_num_queues; i++) {
 			u32 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i));
 			rxdctl |= 0x20; /* PTHRESH */
 			rxdctl |= 4 << 8; /* HTHRESH */
 			rxdctl |= 4 << 16;/* WTHRESH */
 			rxdctl |= 1 << 24; /* Switch to granularity */
 			E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl);
 		}
 	} else if (adapter->hw.mac.type >= igb_mac_min) {
 		u32 psize, srrctl = 0;
 
 		if (if_getmtu(ifp) > ETHERMTU) {
 			/* Set maximum packet len */
 			if (adapter->rx_mbuf_sz <= 4096) {
 				srrctl |= 4096 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
 				rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX;
 			} else if (adapter->rx_mbuf_sz > 4096) {
 				srrctl |= 8192 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
 				rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX;
 			}
 			psize = scctx->isc_max_frame_size;
 			/* are we on a vlan? */
 			if (ifp->if_vlantrunk != NULL)
 				psize += VLAN_TAG_SIZE;
 			E1000_WRITE_REG(&adapter->hw, E1000_RLPML, psize);
 		} else {
 			srrctl |= 2048 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
 			rctl |= E1000_RCTL_SZ_2048;
 		}
 
 		/*
 		 * If TX flow control is disabled and there's >1 queue defined,
 		 * enable DROP.
 		 *
 		 * This drops frames rather than hanging the RX MAC for all queues.
 		 */
 		if ((adapter->rx_num_queues > 1) &&
 		    (adapter->fc == e1000_fc_none ||
 		     adapter->fc == e1000_fc_rx_pause)) {
 			srrctl |= E1000_SRRCTL_DROP_EN;
 		}
 			/* Setup the Base and Length of the Rx Descriptor Rings */
 		for (i = 0, que = adapter->rx_queues; i < adapter->rx_num_queues; i++, que++) {
 			struct rx_ring *rxr = &que->rxr;
 			u64 bus_addr = rxr->rx_paddr;
 			u32 rxdctl;
 
 #ifdef notyet
 			/* Configure for header split? -- ignore for now */
 			rxr->hdr_split = igb_header_split;
 #else
 			srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
 #endif
 
 			E1000_WRITE_REG(hw, E1000_RDLEN(i),
 					scctx->isc_nrxd[0] * sizeof(struct e1000_rx_desc));
 			E1000_WRITE_REG(hw, E1000_RDBAH(i),
 					(uint32_t)(bus_addr >> 32));
 			E1000_WRITE_REG(hw, E1000_RDBAL(i),
 					(uint32_t)bus_addr);
 			E1000_WRITE_REG(hw, E1000_SRRCTL(i), srrctl);
 			/* Enable this Queue */
 			rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i));
 			rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
 			rxdctl &= 0xFFF00000;
 			rxdctl |= IGB_RX_PTHRESH;
 			rxdctl |= IGB_RX_HTHRESH << 8;
 			rxdctl |= IGB_RX_WTHRESH << 16;
 			E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl);
 		}		
 	} else if (adapter->hw.mac.type >= e1000_pch2lan) {
 		if (if_getmtu(ifp) > ETHERMTU)
 			e1000_lv_jumbo_workaround_ich8lan(hw, TRUE);
 		else
 			e1000_lv_jumbo_workaround_ich8lan(hw, FALSE);
 	}
 
 	/* Make sure VLAN Filters are off */
 	rctl &= ~E1000_RCTL_VFE;
 
 	if (adapter->hw.mac.type < igb_mac_min) {
 		if (adapter->rx_mbuf_sz == MCLBYTES)
 			rctl |= E1000_RCTL_SZ_2048;
 		else if (adapter->rx_mbuf_sz == MJUMPAGESIZE)
 			rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX;
 		else if (adapter->rx_mbuf_sz > MJUMPAGESIZE)
 			rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX;
 
 		/* ensure we clear use DTYPE of 00 here */
 		rctl &= ~0x00000C00;
 	}
 
 	/* Write out the settings */
 	E1000_WRITE_REG(hw, E1000_RCTL, rctl);
 
 	return;
 }
 
 static void
 em_if_vlan_register(if_ctx_t ctx, u16 vtag)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	u32 index, bit;
 
 	index = (vtag >> 5) & 0x7F;
 	bit = vtag & 0x1F;
 	adapter->shadow_vfta[index] |= (1 << bit);
 	++adapter->num_vlans;
 }
 
 static void
 em_if_vlan_unregister(if_ctx_t ctx, u16 vtag)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	u32 index, bit;
 
 	index = (vtag >> 5) & 0x7F;
 	bit = vtag & 0x1F;
 	adapter->shadow_vfta[index] &= ~(1 << bit);
 	--adapter->num_vlans;
 }
 
 static void
 em_setup_vlan_hw_support(struct adapter *adapter)
 {
 	struct e1000_hw *hw = &adapter->hw;
 	u32 reg;
 
 	/*
 	 * We get here thru init_locked, meaning
 	 * a soft reset, this has already cleared
 	 * the VFTA and other state, so if there
 	 * have been no vlan's registered do nothing.
 	 */
 	if (adapter->num_vlans == 0)
 		return;
 
 	/*
 	 * A soft reset zero's out the VFTA, so
 	 * we need to repopulate it now.
 	 */
 	for (int i = 0; i < EM_VFTA_SIZE; i++)
 		if (adapter->shadow_vfta[i] != 0)
 			E1000_WRITE_REG_ARRAY(hw, E1000_VFTA,
 			    i, adapter->shadow_vfta[i]);
 
 	reg = E1000_READ_REG(hw, E1000_CTRL);
 	reg |= E1000_CTRL_VME;
 	E1000_WRITE_REG(hw, E1000_CTRL, reg);
 
 	/* Enable the Filter Table */
 	reg = E1000_READ_REG(hw, E1000_RCTL);
 	reg &= ~E1000_RCTL_CFIEN;
 	reg |= E1000_RCTL_VFE;
 	E1000_WRITE_REG(hw, E1000_RCTL, reg);
 }
 
 static void
 em_if_enable_intr(if_ctx_t ctx)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	struct e1000_hw *hw = &adapter->hw;
 	u32 ims_mask = IMS_ENABLE_MASK;
 
 	if (hw->mac.type == e1000_82574) {
 		E1000_WRITE_REG(hw, EM_EIAC, EM_MSIX_MASK);
 		ims_mask |= adapter->ims;
 	} else if (adapter->intr_type == IFLIB_INTR_MSIX && hw->mac.type >= igb_mac_min)  {
 		u32 mask = (adapter->que_mask | adapter->link_mask);
 
 		E1000_WRITE_REG(&adapter->hw, E1000_EIAC, mask);
 		E1000_WRITE_REG(&adapter->hw, E1000_EIAM, mask);
 		E1000_WRITE_REG(&adapter->hw, E1000_EIMS, mask);
 		ims_mask = E1000_IMS_LSC;
 	}
 
 	E1000_WRITE_REG(hw, E1000_IMS, ims_mask);
 }
 
 static void
 em_if_disable_intr(if_ctx_t ctx)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	struct e1000_hw *hw = &adapter->hw;
 
 	if (adapter->intr_type == IFLIB_INTR_MSIX) {
 		if (hw->mac.type >= igb_mac_min)
 			E1000_WRITE_REG(&adapter->hw, E1000_EIMC, ~0);
 		E1000_WRITE_REG(&adapter->hw, E1000_EIAC, 0);
 	}
 	E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
 }
 
 /*
  * Bit of a misnomer, what this really means is
  * to enable OS management of the system... aka
  * to disable special hardware management features
  */
 static void
 em_init_manageability(struct adapter *adapter)
 {
 	/* A shared code workaround */
 #define E1000_82542_MANC2H E1000_MANC2H
 	if (adapter->has_manage) {
 		int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H);
 		int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
 
 		/* disable hardware interception of ARP */
 		manc &= ~(E1000_MANC_ARP_EN);
 
 		/* enable receiving management packets to the host */
 		manc |= E1000_MANC_EN_MNG2HOST;
 #define E1000_MNG2HOST_PORT_623 (1 << 5)
 #define E1000_MNG2HOST_PORT_664 (1 << 6)
 		manc2h |= E1000_MNG2HOST_PORT_623;
 		manc2h |= E1000_MNG2HOST_PORT_664;
 		E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h);
 		E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
 	}
 }
 
 /*
  * Give control back to hardware management
  * controller if there is one.
  */
 static void
 em_release_manageability(struct adapter *adapter)
 {
 	if (adapter->has_manage) {
 		int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
 
 		/* re-enable hardware interception of ARP */
 		manc |= E1000_MANC_ARP_EN;
 		manc &= ~E1000_MANC_EN_MNG2HOST;
 
 		E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
 	}
 }
 
 /*
  * em_get_hw_control sets the {CTRL_EXT|FWSM}:DRV_LOAD bit.
  * For ASF and Pass Through versions of f/w this means
  * that the driver is loaded. For AMT version type f/w
  * this means that the network i/f is open.
  */
 static void
 em_get_hw_control(struct adapter *adapter)
 {
 	u32 ctrl_ext, swsm;
 
 	if (adapter->vf_ifp)
 		return;
 
 	if (adapter->hw.mac.type == e1000_82573) {
 		swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM);
 		E1000_WRITE_REG(&adapter->hw, E1000_SWSM,
 		    swsm | E1000_SWSM_DRV_LOAD);
 		return;
 	}
 	/* else */
 	ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
 	E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
 	    ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
 }
 
 /*
  * em_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit.
  * For ASF and Pass Through versions of f/w this means that
  * the driver is no longer loaded. For AMT versions of the
  * f/w this means that the network i/f is closed.
  */
 static void
 em_release_hw_control(struct adapter *adapter)
 {
 	u32 ctrl_ext, swsm;
 
 	if (!adapter->has_manage)
 		return;
 
 	if (adapter->hw.mac.type == e1000_82573) {
 		swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM);
 		E1000_WRITE_REG(&adapter->hw, E1000_SWSM,
 		    swsm & ~E1000_SWSM_DRV_LOAD);
 		return;
 	}
 	/* else */
 	ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
 	E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
 	    ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
 	return;
 }
 
 static int
 em_is_valid_ether_addr(u8 *addr)
 {
 	char zero_addr[6] = { 0, 0, 0, 0, 0, 0 };
 
 	if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) {
 		return (FALSE);
 	}
 
 	return (TRUE);
 }
 
 /*
 ** Parse the interface capabilities with regard
 ** to both system management and wake-on-lan for
 ** later use.
 */
 static void
 em_get_wakeup(if_ctx_t ctx)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	device_t dev = iflib_get_dev(ctx);
 	u16 eeprom_data = 0, device_id, apme_mask;
 
 	adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw);
 	apme_mask = EM_EEPROM_APME;
 
 	switch (adapter->hw.mac.type) {
 	case e1000_82542:
 	case e1000_82543:
 		break;
 	case e1000_82544:
 		e1000_read_nvm(&adapter->hw,
 		    NVM_INIT_CONTROL2_REG, 1, &eeprom_data);
 		apme_mask = EM_82544_APME;
 		break;
 	case e1000_82546:
 	case e1000_82546_rev_3:
 		if (adapter->hw.bus.func == 1) {
 			e1000_read_nvm(&adapter->hw,
 			    NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
 			break;
 		} else
 			e1000_read_nvm(&adapter->hw,
 			    NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
 		break;
 	case e1000_82573:
 	case e1000_82583:
 		adapter->has_amt = TRUE;
 		/* FALLTHROUGH */
 	case e1000_82571:
 	case e1000_82572:
 	case e1000_80003es2lan:
 		if (adapter->hw.bus.func == 1) {
 			e1000_read_nvm(&adapter->hw,
 			    NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
 			break;
 		} else
 			e1000_read_nvm(&adapter->hw,
 			    NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
 		break;
 	case e1000_ich8lan:
 	case e1000_ich9lan:
 	case e1000_ich10lan:
 	case e1000_pchlan:
 	case e1000_pch2lan:
 	case e1000_pch_lpt:
 	case e1000_pch_spt:
 	case e1000_82575:	/* listing all igb devices */
 	case e1000_82576:
 	case e1000_82580:
 	case e1000_i350:
 	case e1000_i354:
 	case e1000_i210:
 	case e1000_i211:
 	case e1000_vfadapt:
 	case e1000_vfadapt_i350:
 		apme_mask = E1000_WUC_APME;
 		adapter->has_amt = TRUE;
 		eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC);
 		break;
 	default:
 		e1000_read_nvm(&adapter->hw,
 		    NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
 		break;
 	}
 	if (eeprom_data & apme_mask)
 		adapter->wol = (E1000_WUFC_MAG | E1000_WUFC_MC);
 	/*
 	 * We have the eeprom settings, now apply the special cases
 	 * where the eeprom may be wrong or the board won't support
 	 * wake on lan on a particular port
 	 */
 	device_id = pci_get_device(dev);
 	switch (device_id) {
 	case E1000_DEV_ID_82546GB_PCIE:
 		adapter->wol = 0;
 		break;
 	case E1000_DEV_ID_82546EB_FIBER:
 	case E1000_DEV_ID_82546GB_FIBER:
 		/* Wake events only supported on port A for dual fiber
 		 * regardless of eeprom setting */
 		if (E1000_READ_REG(&adapter->hw, E1000_STATUS) &
 		    E1000_STATUS_FUNC_1)
 			adapter->wol = 0;
 		break;
 	case E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3:
 		/* if quad port adapter, disable WoL on all but port A */
 		if (global_quad_port_a != 0)
 			adapter->wol = 0;
 		/* Reset for multiple quad port adapters */
 		if (++global_quad_port_a == 4)
 			global_quad_port_a = 0;
 		break;
 	case E1000_DEV_ID_82571EB_FIBER:
 		/* Wake events only supported on port A for dual fiber
 		 * regardless of eeprom setting */
 		if (E1000_READ_REG(&adapter->hw, E1000_STATUS) &
 		    E1000_STATUS_FUNC_1)
 			adapter->wol = 0;
 		break;
 	case E1000_DEV_ID_82571EB_QUAD_COPPER:
 	case E1000_DEV_ID_82571EB_QUAD_FIBER:
 	case E1000_DEV_ID_82571EB_QUAD_COPPER_LP:
 		/* if quad port adapter, disable WoL on all but port A */
 		if (global_quad_port_a != 0)
 			adapter->wol = 0;
 		/* Reset for multiple quad port adapters */
 		if (++global_quad_port_a == 4)
 			global_quad_port_a = 0;
 		break;
 	}
 	return;
 }
 
 
 /*
  * Enable PCI Wake On Lan capability
  */
 static void
 em_enable_wakeup(if_ctx_t ctx)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	device_t dev = iflib_get_dev(ctx);
 	if_t ifp = iflib_get_ifp(ctx);
 	int error = 0;
 	u32 pmc, ctrl, ctrl_ext, rctl;
 	u16 status;
 
 	if (pci_find_cap(dev, PCIY_PMG, &pmc) != 0)
 		return;
 
 	/*
 	 * Determine type of Wakeup: note that wol
 	 * is set with all bits on by default.
 	 */
 	if ((if_getcapenable(ifp) & IFCAP_WOL_MAGIC) == 0)
 		adapter->wol &= ~E1000_WUFC_MAG;
 
 	if ((if_getcapenable(ifp) & IFCAP_WOL_UCAST) == 0)
 		adapter->wol &= ~E1000_WUFC_EX;
 
 	if ((if_getcapenable(ifp) & IFCAP_WOL_MCAST) == 0)
 		adapter->wol &= ~E1000_WUFC_MC;
 	else {
 		rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
 		rctl |= E1000_RCTL_MPE;
 		E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl);
 	}
 
 	if (!(adapter->wol & (E1000_WUFC_EX | E1000_WUFC_MAG | E1000_WUFC_MC)))
 		goto pme;
 
 	/* Advertise the wakeup capability */
 	ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL);
 	ctrl |= (E1000_CTRL_SWDPIN2 | E1000_CTRL_SWDPIN3);
 	E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl);
 
 	/* Keep the laser running on Fiber adapters */
 	if (adapter->hw.phy.media_type == e1000_media_type_fiber ||
 	    adapter->hw.phy.media_type == e1000_media_type_internal_serdes) {
 		ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
 		ctrl_ext |= E1000_CTRL_EXT_SDP3_DATA;
 		E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, ctrl_ext);
 	}
 
 	if ((adapter->hw.mac.type == e1000_ich8lan) ||
 	    (adapter->hw.mac.type == e1000_pchlan) ||
 	    (adapter->hw.mac.type == e1000_ich9lan) ||
 	    (adapter->hw.mac.type == e1000_ich10lan))
 		e1000_suspend_workarounds_ich8lan(&adapter->hw);
 
 	if ( adapter->hw.mac.type >= e1000_pchlan) {
 		error = em_enable_phy_wakeup(adapter);
 		if (error)
 			goto pme;
 	} else {
 		/* Enable wakeup by the MAC */
 		E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
 		E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
 	}
 
 	if (adapter->hw.phy.type == e1000_phy_igp_3)
 		e1000_igp3_phy_powerdown_workaround_ich8lan(&adapter->hw);
 
 pme:
 	status = pci_read_config(dev, pmc + PCIR_POWER_STATUS, 2);
 	status &= ~(PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE);
 	if (!error && (if_getcapenable(ifp) & IFCAP_WOL))
 		status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE;
 	pci_write_config(dev, pmc + PCIR_POWER_STATUS, status, 2);
 
 	return;
 }
 
 /*
  * WOL in the newer chipset interfaces (pchlan)
  * require thing to be copied into the phy
  */
 static int
 em_enable_phy_wakeup(struct adapter *adapter)
 {
 	struct e1000_hw *hw = &adapter->hw;
 	u32 mreg, ret = 0;
 	u16 preg;
 
 	/* copy MAC RARs to PHY RARs */
 	e1000_copy_rx_addrs_to_phy_ich8lan(hw);
 
 	/* copy MAC MTA to PHY MTA */
 	for (int i = 0; i < adapter->hw.mac.mta_reg_count; i++) {
 		mreg = E1000_READ_REG_ARRAY(hw, E1000_MTA, i);
 		e1000_write_phy_reg(hw, BM_MTA(i), (u16)(mreg & 0xFFFF));
 		e1000_write_phy_reg(hw, BM_MTA(i) + 1,
 		    (u16)((mreg >> 16) & 0xFFFF));
 	}
 
 	/* configure PHY Rx Control register */
 	e1000_read_phy_reg(&adapter->hw, BM_RCTL, &preg);
 	mreg = E1000_READ_REG(hw, E1000_RCTL);
 	if (mreg & E1000_RCTL_UPE)
 		preg |= BM_RCTL_UPE;
 	if (mreg & E1000_RCTL_MPE)
 		preg |= BM_RCTL_MPE;
 	preg &= ~(BM_RCTL_MO_MASK);
 	if (mreg & E1000_RCTL_MO_3)
 		preg |= (((mreg & E1000_RCTL_MO_3) >> E1000_RCTL_MO_SHIFT)
 				<< BM_RCTL_MO_SHIFT);
 	if (mreg & E1000_RCTL_BAM)
 		preg |= BM_RCTL_BAM;
 	if (mreg & E1000_RCTL_PMCF)
 		preg |= BM_RCTL_PMCF;
 	mreg = E1000_READ_REG(hw, E1000_CTRL);
 	if (mreg & E1000_CTRL_RFCE)
 		preg |= BM_RCTL_RFCE;
 	e1000_write_phy_reg(&adapter->hw, BM_RCTL, preg);
 
 	/* enable PHY wakeup in MAC register */
 	E1000_WRITE_REG(hw, E1000_WUC,
 	    E1000_WUC_PHY_WAKE | E1000_WUC_PME_EN | E1000_WUC_APME);
 	E1000_WRITE_REG(hw, E1000_WUFC, adapter->wol);
 
 	/* configure and enable PHY wakeup in PHY registers */
 	e1000_write_phy_reg(&adapter->hw, BM_WUFC, adapter->wol);
 	e1000_write_phy_reg(&adapter->hw, BM_WUC, E1000_WUC_PME_EN);
 
 	/* activate PHY wakeup */
 	ret = hw->phy.ops.acquire(hw);
 	if (ret) {
 		printf("Could not acquire PHY\n");
 		return ret;
 	}
 	e1000_write_phy_reg_mdic(hw, IGP01E1000_PHY_PAGE_SELECT,
 	                         (BM_WUC_ENABLE_PAGE << IGP_PAGE_SHIFT));
 	ret = e1000_read_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, &preg);
 	if (ret) {
 		printf("Could not read PHY page 769\n");
 		goto out;
 	}
 	preg |= BM_WUC_ENABLE_BIT | BM_WUC_HOST_WU_BIT;
 	ret = e1000_write_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, preg);
 	if (ret)
 		printf("Could not set PHY Host Wakeup bit\n");
 out:
 	hw->phy.ops.release(hw);
 
 	return ret;
 }
 
 static void
 em_if_led_func(if_ctx_t ctx, int onoff)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 
 	if (onoff) {
 		e1000_setup_led(&adapter->hw);
 		e1000_led_on(&adapter->hw);
 	} else {
 		e1000_led_off(&adapter->hw);
 		e1000_cleanup_led(&adapter->hw);
 	}
 }
 
 /*
  * Disable the L0S and L1 LINK states
  */
 static void
 em_disable_aspm(struct adapter *adapter)
 {
 	int base, reg;
 	u16 link_cap,link_ctrl;
 	device_t dev = adapter->dev;
 
 	switch (adapter->hw.mac.type) {
 	case e1000_82573:
 	case e1000_82574:
 	case e1000_82583:
 		break;
 	default:
 		return;
 	}
 	if (pci_find_cap(dev, PCIY_EXPRESS, &base) != 0)
 		return;
 	reg = base + PCIER_LINK_CAP;
 	link_cap = pci_read_config(dev, reg, 2);
 	if ((link_cap & PCIEM_LINK_CAP_ASPM) == 0)
 		return;
 	reg = base + PCIER_LINK_CTL;
 	link_ctrl = pci_read_config(dev, reg, 2);
 	link_ctrl &= ~PCIEM_LINK_CTL_ASPMC;
 	pci_write_config(dev, reg, link_ctrl, 2);
 	return;
 }
 
 /**********************************************************************
  *
  *  Update the board statistics counters.
  *
  **********************************************************************/
 static void
 em_update_stats_counters(struct adapter *adapter)
 {
 
 	if(adapter->hw.phy.media_type == e1000_media_type_copper ||
 	   (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) {
 		adapter->stats.symerrs += E1000_READ_REG(&adapter->hw, E1000_SYMERRS);
 		adapter->stats.sec += E1000_READ_REG(&adapter->hw, E1000_SEC);
 	}
 	adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, E1000_CRCERRS);
 	adapter->stats.mpc += E1000_READ_REG(&adapter->hw, E1000_MPC);
 	adapter->stats.scc += E1000_READ_REG(&adapter->hw, E1000_SCC);
 	adapter->stats.ecol += E1000_READ_REG(&adapter->hw, E1000_ECOL);
 
 	adapter->stats.mcc += E1000_READ_REG(&adapter->hw, E1000_MCC);
 	adapter->stats.latecol += E1000_READ_REG(&adapter->hw, E1000_LATECOL);
 	adapter->stats.colc += E1000_READ_REG(&adapter->hw, E1000_COLC);
 	adapter->stats.dc += E1000_READ_REG(&adapter->hw, E1000_DC);
 	adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC);
 	adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC);
 	adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC);
 	adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, E1000_XOFFRXC);
 	/*
 	 ** For watchdog management we need to know if we have been
 	 ** paused during the last interval, so capture that here.
 	*/
 	adapter->shared->isc_pause_frames = adapter->stats.xoffrxc;
 	adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC);
 	adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC);
 	adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64);
 	adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, E1000_PRC127);
 	adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, E1000_PRC255);
 	adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, E1000_PRC511);
 	adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, E1000_PRC1023);
 	adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, E1000_PRC1522);
 	adapter->stats.gprc += E1000_READ_REG(&adapter->hw, E1000_GPRC);
 	adapter->stats.bprc += E1000_READ_REG(&adapter->hw, E1000_BPRC);
 	adapter->stats.mprc += E1000_READ_REG(&adapter->hw, E1000_MPRC);
 	adapter->stats.gptc += E1000_READ_REG(&adapter->hw, E1000_GPTC);
 
 	/* For the 64-bit byte counters the low dword must be read first. */
 	/* Both registers clear on the read of the high dword */
 
 	adapter->stats.gorc += E1000_READ_REG(&adapter->hw, E1000_GORCL) +
 	    ((u64)E1000_READ_REG(&adapter->hw, E1000_GORCH) << 32);
 	adapter->stats.gotc += E1000_READ_REG(&adapter->hw, E1000_GOTCL) +
 	    ((u64)E1000_READ_REG(&adapter->hw, E1000_GOTCH) << 32);
 
 	adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, E1000_RNBC);
 	adapter->stats.ruc += E1000_READ_REG(&adapter->hw, E1000_RUC);
 	adapter->stats.rfc += E1000_READ_REG(&adapter->hw, E1000_RFC);
 	adapter->stats.roc += E1000_READ_REG(&adapter->hw, E1000_ROC);
 	adapter->stats.rjc += E1000_READ_REG(&adapter->hw, E1000_RJC);
 
 	adapter->stats.tor += E1000_READ_REG(&adapter->hw, E1000_TORH);
 	adapter->stats.tot += E1000_READ_REG(&adapter->hw, E1000_TOTH);
 
 	adapter->stats.tpr += E1000_READ_REG(&adapter->hw, E1000_TPR);
 	adapter->stats.tpt += E1000_READ_REG(&adapter->hw, E1000_TPT);
 	adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, E1000_PTC64);
 	adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, E1000_PTC127);
 	adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, E1000_PTC255);
 	adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, E1000_PTC511);
 	adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, E1000_PTC1023);
 	adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, E1000_PTC1522);
 	adapter->stats.mptc += E1000_READ_REG(&adapter->hw, E1000_MPTC);
 	adapter->stats.bptc += E1000_READ_REG(&adapter->hw, E1000_BPTC);
 
 	/* Interrupt Counts */
 
 	adapter->stats.iac += E1000_READ_REG(&adapter->hw, E1000_IAC);
 	adapter->stats.icrxptc += E1000_READ_REG(&adapter->hw, E1000_ICRXPTC);
 	adapter->stats.icrxatc += E1000_READ_REG(&adapter->hw, E1000_ICRXATC);
 	adapter->stats.ictxptc += E1000_READ_REG(&adapter->hw, E1000_ICTXPTC);
 	adapter->stats.ictxatc += E1000_READ_REG(&adapter->hw, E1000_ICTXATC);
 	adapter->stats.ictxqec += E1000_READ_REG(&adapter->hw, E1000_ICTXQEC);
 	adapter->stats.ictxqmtc += E1000_READ_REG(&adapter->hw, E1000_ICTXQMTC);
 	adapter->stats.icrxdmtc += E1000_READ_REG(&adapter->hw, E1000_ICRXDMTC);
 	adapter->stats.icrxoc += E1000_READ_REG(&adapter->hw, E1000_ICRXOC);
 
 	if (adapter->hw.mac.type >= e1000_82543) {
 		adapter->stats.algnerrc +=
 		E1000_READ_REG(&adapter->hw, E1000_ALGNERRC);
 		adapter->stats.rxerrc +=
 		E1000_READ_REG(&adapter->hw, E1000_RXERRC);
 		adapter->stats.tncrs +=
 		E1000_READ_REG(&adapter->hw, E1000_TNCRS);
 		adapter->stats.cexterr +=
 		E1000_READ_REG(&adapter->hw, E1000_CEXTERR);
 		adapter->stats.tsctc +=
 		E1000_READ_REG(&adapter->hw, E1000_TSCTC);
 		adapter->stats.tsctfc +=
 		E1000_READ_REG(&adapter->hw, E1000_TSCTFC);
 	}
 }
 
 static uint64_t
 em_if_get_counter(if_ctx_t ctx, ift_counter cnt)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	struct ifnet *ifp = iflib_get_ifp(ctx);
 
 	switch (cnt) {
 	case IFCOUNTER_COLLISIONS:
 		return (adapter->stats.colc);
 	case IFCOUNTER_IERRORS:
 		return (adapter->dropped_pkts + adapter->stats.rxerrc +
 		    adapter->stats.crcerrs + adapter->stats.algnerrc +
 		    adapter->stats.ruc + adapter->stats.roc +
 		    adapter->stats.mpc + adapter->stats.cexterr);
 	case IFCOUNTER_OERRORS:
 		return (adapter->stats.ecol + adapter->stats.latecol +
 		    adapter->watchdog_events);
 	default:
 		return (if_get_counter_default(ifp, cnt));
 	}
 }
 
 /* Export a single 32-bit register via a read-only sysctl. */
 static int
 em_sysctl_reg_handler(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter *adapter;
 	u_int val;
 
 	adapter = oidp->oid_arg1;
 	val = E1000_READ_REG(&adapter->hw, oidp->oid_arg2);
 	return (sysctl_handle_int(oidp, &val, 0, req));
 }
 
 /*
  * Add sysctl variables, one per statistic, to the system.
  */
 static void
 em_add_hw_stats(struct adapter *adapter)
 {
 	device_t dev = iflib_get_dev(adapter->ctx);
 	struct em_tx_queue *tx_que = adapter->tx_queues;
 	struct em_rx_queue *rx_que = adapter->rx_queues;
 
 	struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
 	struct sysctl_oid *tree = device_get_sysctl_tree(dev);
 	struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
 	struct e1000_hw_stats *stats = &adapter->stats;
 
 	struct sysctl_oid *stat_node, *queue_node, *int_node;
 	struct sysctl_oid_list *stat_list, *queue_list, *int_list;
 
 #define QUEUE_NAME_LEN 32
 	char namebuf[QUEUE_NAME_LEN];
 
 	/* Driver Statistics */
 	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
 			CTLFLAG_RD, &adapter->dropped_pkts,
 			"Driver dropped packets");
 	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
 			CTLFLAG_RD, &adapter->link_irq,
 			"Link MSI-X IRQ Handled");
 	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_overruns",
 			CTLFLAG_RD, &adapter->rx_overruns,
 			"RX overruns");
 	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_timeouts",
 			CTLFLAG_RD, &adapter->watchdog_events,
 			"Watchdog timeouts");
 	SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "device_control",
 			CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_CTRL,
 			em_sysctl_reg_handler, "IU",
 			"Device Control Register");
 	SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rx_control",
 			CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RCTL,
 			em_sysctl_reg_handler, "IU",
 			"Receiver Control Register");
 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water",
 			CTLFLAG_RD, &adapter->hw.fc.high_water, 0,
 			"Flow Control High Watermark");
 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water",
 			CTLFLAG_RD, &adapter->hw.fc.low_water, 0,
 			"Flow Control Low Watermark");
 
 	for (int i = 0; i < adapter->tx_num_queues; i++, tx_que++) {
 		struct tx_ring *txr = &tx_que->txr;
 		snprintf(namebuf, QUEUE_NAME_LEN, "queue_tx_%d", i);
 		queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
 					    CTLFLAG_RD, NULL, "TX Queue Name");
 		queue_list = SYSCTL_CHILDREN(queue_node);
 
 		SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
 				CTLTYPE_UINT | CTLFLAG_RD, adapter,
 				E1000_TDH(txr->me),
 				em_sysctl_reg_handler, "IU",
 				"Transmit Descriptor Head");
 		SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
 				CTLTYPE_UINT | CTLFLAG_RD, adapter,
 				E1000_TDT(txr->me),
 				em_sysctl_reg_handler, "IU",
 				"Transmit Descriptor Tail");
 		SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "tx_irq",
 				CTLFLAG_RD, &txr->tx_irq,
 				"Queue MSI-X Transmit Interrupts");
 	}
 
 	for (int j = 0; j < adapter->rx_num_queues; j++, rx_que++) {
 		struct rx_ring *rxr = &rx_que->rxr;
 		snprintf(namebuf, QUEUE_NAME_LEN, "queue_rx_%d", j);
 		queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
 					    CTLFLAG_RD, NULL, "RX Queue Name");
 		queue_list = SYSCTL_CHILDREN(queue_node);
 
 		SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
 				CTLTYPE_UINT | CTLFLAG_RD, adapter,
 				E1000_RDH(rxr->me),
 				em_sysctl_reg_handler, "IU",
 				"Receive Descriptor Head");
 		SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
 				CTLTYPE_UINT | CTLFLAG_RD, adapter,
 				E1000_RDT(rxr->me),
 				em_sysctl_reg_handler, "IU",
 				"Receive Descriptor Tail");
 		SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "rx_irq",
 				CTLFLAG_RD, &rxr->rx_irq,
 				"Queue MSI-X Receive Interrupts");
 	}
 
 	/* MAC stats get their own sub node */
 
 	stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
 				    CTLFLAG_RD, NULL, "Statistics");
 	stat_list = SYSCTL_CHILDREN(stat_node);
 
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "excess_coll",
 			CTLFLAG_RD, &stats->ecol,
 			"Excessive collisions");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "single_coll",
 			CTLFLAG_RD, &stats->scc,
 			"Single collisions");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "multiple_coll",
 			CTLFLAG_RD, &stats->mcc,
 			"Multiple collisions");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "late_coll",
 			CTLFLAG_RD, &stats->latecol,
 			"Late collisions");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "collision_count",
 			CTLFLAG_RD, &stats->colc,
 			"Collision Count");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "symbol_errors",
 			CTLFLAG_RD, &adapter->stats.symerrs,
 			"Symbol Errors");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "sequence_errors",
 			CTLFLAG_RD, &adapter->stats.sec,
 			"Sequence Errors");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "defer_count",
 			CTLFLAG_RD, &adapter->stats.dc,
 			"Defer Count");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "missed_packets",
 			CTLFLAG_RD, &adapter->stats.mpc,
 			"Missed Packets");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_no_buff",
 			CTLFLAG_RD, &adapter->stats.rnbc,
 			"Receive No Buffers");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersize",
 			CTLFLAG_RD, &adapter->stats.ruc,
 			"Receive Undersize");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
 			CTLFLAG_RD, &adapter->stats.rfc,
 			"Fragmented Packets Received ");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversize",
 			CTLFLAG_RD, &adapter->stats.roc,
 			"Oversized Packets Received");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabber",
 			CTLFLAG_RD, &adapter->stats.rjc,
 			"Recevied Jabber");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_errs",
 			CTLFLAG_RD, &adapter->stats.rxerrc,
 			"Receive Errors");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
 			CTLFLAG_RD, &adapter->stats.crcerrs,
 			"CRC errors");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "alignment_errs",
 			CTLFLAG_RD, &adapter->stats.algnerrc,
 			"Alignment Errors");
 	/* On 82575 these are collision counts */
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "coll_ext_errs",
 			CTLFLAG_RD, &adapter->stats.cexterr,
 			"Collision/Carrier extension errors");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_recvd",
 			CTLFLAG_RD, &adapter->stats.xonrxc,
 			"XON Received");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_txd",
 			CTLFLAG_RD, &adapter->stats.xontxc,
 			"XON Transmitted");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_recvd",
 			CTLFLAG_RD, &adapter->stats.xoffrxc,
 			"XOFF Received");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_txd",
 			CTLFLAG_RD, &adapter->stats.xofftxc,
 			"XOFF Transmitted");
 
 	/* Packet Reception Stats */
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_recvd",
 			CTLFLAG_RD, &adapter->stats.tpr,
 			"Total Packets Received ");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd",
 			CTLFLAG_RD, &adapter->stats.gprc,
 			"Good Packets Received");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_recvd",
 			CTLFLAG_RD, &adapter->stats.bprc,
 			"Broadcast Packets Received");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd",
 			CTLFLAG_RD, &adapter->stats.mprc,
 			"Multicast Packets Received");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
 			CTLFLAG_RD, &adapter->stats.prc64,
 			"64 byte frames received ");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
 			CTLFLAG_RD, &adapter->stats.prc127,
 			"65-127 byte frames received");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
 			CTLFLAG_RD, &adapter->stats.prc255,
 			"128-255 byte frames received");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
 			CTLFLAG_RD, &adapter->stats.prc511,
 			"256-511 byte frames received");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
 			CTLFLAG_RD, &adapter->stats.prc1023,
 			"512-1023 byte frames received");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
 			CTLFLAG_RD, &adapter->stats.prc1522,
 			"1023-1522 byte frames received");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd",
 			CTLFLAG_RD, &adapter->stats.gorc,
 			"Good Octets Received");
 
 	/* Packet Transmission Stats */
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
 			CTLFLAG_RD, &adapter->stats.gotc,
 			"Good Octets Transmitted");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
 			CTLFLAG_RD, &adapter->stats.tpt,
 			"Total Packets Transmitted");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
 			CTLFLAG_RD, &adapter->stats.gptc,
 			"Good Packets Transmitted");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
 			CTLFLAG_RD, &adapter->stats.bptc,
 			"Broadcast Packets Transmitted");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
 			CTLFLAG_RD, &adapter->stats.mptc,
 			"Multicast Packets Transmitted");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
 			CTLFLAG_RD, &adapter->stats.ptc64,
 			"64 byte frames transmitted ");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
 			CTLFLAG_RD, &adapter->stats.ptc127,
 			"65-127 byte frames transmitted");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
 			CTLFLAG_RD, &adapter->stats.ptc255,
 			"128-255 byte frames transmitted");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
 			CTLFLAG_RD, &adapter->stats.ptc511,
 			"256-511 byte frames transmitted");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
 			CTLFLAG_RD, &adapter->stats.ptc1023,
 			"512-1023 byte frames transmitted");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
 			CTLFLAG_RD, &adapter->stats.ptc1522,
 			"1024-1522 byte frames transmitted");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tso_txd",
 			CTLFLAG_RD, &adapter->stats.tsctc,
 			"TSO Contexts Transmitted");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tso_ctx_fail",
 			CTLFLAG_RD, &adapter->stats.tsctfc,
 			"TSO Contexts Failed");
 
 
 	/* Interrupt Stats */
 
 	int_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "interrupts",
 				    CTLFLAG_RD, NULL, "Interrupt Statistics");
 	int_list = SYSCTL_CHILDREN(int_node);
 
 	SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "asserts",
 			CTLFLAG_RD, &adapter->stats.iac,
 			"Interrupt Assertion Count");
 
 	SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_pkt_timer",
 			CTLFLAG_RD, &adapter->stats.icrxptc,
 			"Interrupt Cause Rx Pkt Timer Expire Count");
 
 	SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_abs_timer",
 			CTLFLAG_RD, &adapter->stats.icrxatc,
 			"Interrupt Cause Rx Abs Timer Expire Count");
 
 	SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "tx_pkt_timer",
 			CTLFLAG_RD, &adapter->stats.ictxptc,
 			"Interrupt Cause Tx Pkt Timer Expire Count");
 
 	SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "tx_abs_timer",
 			CTLFLAG_RD, &adapter->stats.ictxatc,
 			"Interrupt Cause Tx Abs Timer Expire Count");
 
 	SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "tx_queue_empty",
 			CTLFLAG_RD, &adapter->stats.ictxqec,
 			"Interrupt Cause Tx Queue Empty Count");
 
 	SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "tx_queue_min_thresh",
 			CTLFLAG_RD, &adapter->stats.ictxqmtc,
 			"Interrupt Cause Tx Queue Min Thresh Count");
 
 	SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_desc_min_thresh",
 			CTLFLAG_RD, &adapter->stats.icrxdmtc,
 			"Interrupt Cause Rx Desc Min Thresh Count");
 
 	SYSCTL_ADD_UQUAD(ctx, int_list, OID_AUTO, "rx_overrun",
 			CTLFLAG_RD, &adapter->stats.icrxoc,
 			"Interrupt Cause Receiver Overrun Count");
 }
 
 /**********************************************************************
  *
  *  This routine provides a way to dump out the adapter eeprom,
  *  often a useful debug/service tool. This only dumps the first
  *  32 words, stuff that matters is in that extent.
  *
  **********************************************************************/
 static int
 em_sysctl_nvm_info(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter *adapter = (struct adapter *)arg1;
 	int error;
 	int result;
 
 	result = -1;
 	error = sysctl_handle_int(oidp, &result, 0, req);
 
 	if (error || !req->newptr)
 		return (error);
 
 	/*
 	 * This value will cause a hex dump of the
 	 * first 32 16-bit words of the EEPROM to
 	 * the screen.
 	 */
 	if (result == 1)
 		em_print_nvm_info(adapter);
 
 	return (error);
 }
 
 static void
 em_print_nvm_info(struct adapter *adapter)
 {
 	u16 eeprom_data;
 	int i, j, row = 0;
 
 	/* Its a bit crude, but it gets the job done */
 	printf("\nInterface EEPROM Dump:\n");
 	printf("Offset\n0x0000  ");
 	for (i = 0, j = 0; i < 32; i++, j++) {
 		if (j == 8) { /* Make the offset block */
 			j = 0; ++row;
 			printf("\n0x00%x0  ",row);
 		}
 		e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data);
 		printf("%04x ", eeprom_data);
 	}
 	printf("\n");
 }
 
 static int
 em_sysctl_int_delay(SYSCTL_HANDLER_ARGS)
 {
 	struct em_int_delay_info *info;
 	struct adapter *adapter;
 	u32 regval;
 	int error, usecs, ticks;
 
 	info = (struct em_int_delay_info *) arg1;
 	usecs = info->value;
 	error = sysctl_handle_int(oidp, &usecs, 0, req);
 	if (error != 0 || req->newptr == NULL)
 		return (error);
 	if (usecs < 0 || usecs > EM_TICKS_TO_USECS(65535))
 		return (EINVAL);
 	info->value = usecs;
 	ticks = EM_USECS_TO_TICKS(usecs);
 	if (info->offset == E1000_ITR)	/* units are 256ns here */
 		ticks *= 4;
 
 	adapter = info->adapter;
 
 	regval = E1000_READ_OFFSET(&adapter->hw, info->offset);
 	regval = (regval & ~0xffff) | (ticks & 0xffff);
 	/* Handle a few special cases. */
 	switch (info->offset) {
 	case E1000_RDTR:
 		break;
 	case E1000_TIDV:
 		if (ticks == 0) {
 			adapter->txd_cmd &= ~E1000_TXD_CMD_IDE;
 			/* Don't write 0 into the TIDV register. */
 			regval++;
 		} else
 			adapter->txd_cmd |= E1000_TXD_CMD_IDE;
 		break;
 	}
 	E1000_WRITE_OFFSET(&adapter->hw, info->offset, regval);
 	return (0);
 }
 
 static void
 em_add_int_delay_sysctl(struct adapter *adapter, const char *name,
 	const char *description, struct em_int_delay_info *info,
 	int offset, int value)
 {
 	info->adapter = adapter;
 	info->offset = offset;
 	info->value = value;
 	SYSCTL_ADD_PROC(device_get_sysctl_ctx(adapter->dev),
 	    SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
 	    OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW,
 	    info, 0, em_sysctl_int_delay, "I", description);
 }
 
 /*
  * Set flow control using sysctl:
  * Flow control values:
  *      0 - off
  *      1 - rx pause
  *      2 - tx pause
  *      3 - full
  */
 static int
 em_set_flowcntl(SYSCTL_HANDLER_ARGS)
 {
 	int error;
 	static int input = 3; /* default is full */
 	struct adapter	*adapter = (struct adapter *) arg1;
 
 	error = sysctl_handle_int(oidp, &input, 0, req);
 
 	if ((error) || (req->newptr == NULL))
 		return (error);
 
 	if (input == adapter->fc) /* no change? */
 		return (error);
 
 	switch (input) {
 	case e1000_fc_rx_pause:
 	case e1000_fc_tx_pause:
 	case e1000_fc_full:
 	case e1000_fc_none:
 		adapter->hw.fc.requested_mode = input;
 		adapter->fc = input;
 		break;
 	default:
 		/* Do nothing */
 		return (error);
 	}
 
 	adapter->hw.fc.current_mode = adapter->hw.fc.requested_mode;
 	e1000_force_mac_fc(&adapter->hw);
 	return (error);
 }
 
 /*
  * Manage Energy Efficient Ethernet:
  * Control values:
  *     0/1 - enabled/disabled
  */
 static int
 em_sysctl_eee(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter *adapter = (struct adapter *) arg1;
 	int error, value;
 
 	value = adapter->hw.dev_spec.ich8lan.eee_disable;
 	error = sysctl_handle_int(oidp, &value, 0, req);
 	if (error || req->newptr == NULL)
 		return (error);
 	adapter->hw.dev_spec.ich8lan.eee_disable = (value != 0);
 	em_if_init(adapter->ctx);
 
 	return (0);
 }
 
 static int
 em_sysctl_debug_info(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter *adapter;
 	int error;
 	int result;
 
 	result = -1;
 	error = sysctl_handle_int(oidp, &result, 0, req);
 
 	if (error || !req->newptr)
 		return (error);
 
 	if (result == 1) {
 		adapter = (struct adapter *) arg1;
 		em_print_debug_info(adapter);
 	}
 
 	return (error);
 }
 
 static int
 em_get_rs(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter *adapter = (struct adapter *) arg1;
 	int error;
 	int result;
 
 	result = 0;
 	error = sysctl_handle_int(oidp, &result, 0, req);
 
 	if (error || !req->newptr || result != 1)
 		return (error);
 	em_dump_rs(adapter);
 
 	return (error);
 }
 
 static void
 em_if_debug(if_ctx_t ctx)
 {
 	em_dump_rs(iflib_get_softc(ctx));
 }
 
 /*
  * This routine is meant to be fluid, add whatever is
  * needed for debugging a problem.  -jfv
  */
 static void
 em_print_debug_info(struct adapter *adapter)
 {
 	device_t dev = iflib_get_dev(adapter->ctx);
 	struct ifnet *ifp = iflib_get_ifp(adapter->ctx);
 	struct tx_ring *txr = &adapter->tx_queues->txr;
 	struct rx_ring *rxr = &adapter->rx_queues->rxr;
 
 	if (if_getdrvflags(ifp) & IFF_DRV_RUNNING)
 		printf("Interface is RUNNING ");
 	else
 		printf("Interface is NOT RUNNING\n");
 
 	if (if_getdrvflags(ifp) & IFF_DRV_OACTIVE)
 		printf("and INACTIVE\n");
 	else
 		printf("and ACTIVE\n");
 
 	for (int i = 0; i < adapter->tx_num_queues; i++, txr++) {
 		device_printf(dev, "TX Queue %d ------\n", i);
 		device_printf(dev, "hw tdh = %d, hw tdt = %d\n",
 			E1000_READ_REG(&adapter->hw, E1000_TDH(i)),
 			E1000_READ_REG(&adapter->hw, E1000_TDT(i)));
 
 	}
 	for (int j=0; j < adapter->rx_num_queues; j++, rxr++) {
 		device_printf(dev, "RX Queue %d ------\n", j);
 		device_printf(dev, "hw rdh = %d, hw rdt = %d\n",
 			E1000_READ_REG(&adapter->hw, E1000_RDH(j)),
 			E1000_READ_REG(&adapter->hw, E1000_RDT(j)));
 	}
 }
 
 /*
  * 82574 only:
  * Write a new value to the EEPROM increasing the number of MSI-X
  * vectors from 3 to 5, for proper multiqueue support.
  */
 static void
 em_enable_vectors_82574(if_ctx_t ctx)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	struct e1000_hw *hw = &adapter->hw;
 	device_t dev = iflib_get_dev(ctx);
 	u16 edata;
 
 	e1000_read_nvm(hw, EM_NVM_PCIE_CTRL, 1, &edata);
 	if (bootverbose)
 		device_printf(dev, "EM_NVM_PCIE_CTRL = %#06x\n", edata);
 	if (((edata & EM_NVM_MSIX_N_MASK) >> EM_NVM_MSIX_N_SHIFT) != 4) {
 		device_printf(dev, "Writing to eeprom: increasing "
 		    "reported MSI-X vectors from 3 to 5...\n");
 		edata &= ~(EM_NVM_MSIX_N_MASK);
 		edata |= 4 << EM_NVM_MSIX_N_SHIFT;
 		e1000_write_nvm(hw, EM_NVM_PCIE_CTRL, 1, &edata);
 		e1000_update_nvm_checksum(hw);
 		device_printf(dev, "Writing to eeprom: done\n");
 	}
 }
Index: head/sys/dev/ixgbe/if_ix.c
===================================================================
--- head/sys/dev/ixgbe/if_ix.c	(revision 345304)
+++ head/sys/dev/ixgbe/if_ix.c	(revision 345305)
@@ -1,4555 +1,4552 @@
 /******************************************************************************
 
   Copyright (c) 2001-2017, Intel Corporation
   All rights reserved.
 
   Redistribution and use in source and binary forms, with or without
   modification, are permitted provided that the following conditions are met:
 
    1. Redistributions of source code must retain the above copyright notice,
       this list of conditions and the following disclaimer.
 
    2. Redistributions in binary form must reproduce the above copyright
       notice, this list of conditions and the following disclaimer in the
       documentation and/or other materials provided with the distribution.
 
    3. Neither the name of the Intel Corporation nor the names of its
       contributors may be used to endorse or promote products derived from
       this software without specific prior written permission.
 
   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
   POSSIBILITY OF SUCH DAMAGE.
 
 ******************************************************************************/
 /*$FreeBSD$*/
 
 
 #include "opt_inet.h"
 #include "opt_inet6.h"
 #include "opt_rss.h"
 
 #include "ixgbe.h"
 #include "ixgbe_sriov.h"
 #include "ifdi_if.h"
 
 #include <net/netmap.h>
 #include <dev/netmap/netmap_kern.h>
 
 /************************************************************************
  * Driver version
  ************************************************************************/
 char ixgbe_driver_version[] = "4.0.1-k";
 
 
 /************************************************************************
  * PCI Device ID Table
  *
  *   Used by probe to select devices to load on
  *   Last field stores an index into ixgbe_strings
  *   Last entry must be all 0s
  *
  *   { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  ************************************************************************/
 static pci_vendor_info_t ixgbe_vendor_info_array[] =
 {
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT,  "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT,  "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4,  "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT,  "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2,  "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598,  "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT,  "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT,  "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR,  "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM,  "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM,  "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4,  "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ,  "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP,  "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM,  "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4,  "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM,  "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE,  "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE,  "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2,  "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE,  "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP,  "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF_QP,  "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_QSFP_SF_QP,  "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T,  "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1,  "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550T,  "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550T1, "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_KR,  "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_KX4,  "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_10G_T,  "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_1G_T,  "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_SFP, "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_KR, "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_KR_L, "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SFP, "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SFP_N, "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SGMII, "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SGMII_L, "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_10G_T, "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_1G_T, "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_1G_T_L, "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_BYPASS, "Intel(R) PRO/10GbE PCI-Express Network Driver"),
   PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BYPASS, "Intel(R) PRO/10GbE PCI-Express Network Driver"),
 	/* required last entry */
   PVID_END
 };
 
 static void *ixgbe_register(device_t dev);
 static int  ixgbe_if_attach_pre(if_ctx_t ctx);
 static int  ixgbe_if_attach_post(if_ctx_t ctx);
 static int  ixgbe_if_detach(if_ctx_t ctx);
 static int  ixgbe_if_shutdown(if_ctx_t ctx);
 static int  ixgbe_if_suspend(if_ctx_t ctx);
 static int  ixgbe_if_resume(if_ctx_t ctx);
 
 static void ixgbe_if_stop(if_ctx_t ctx);
 void ixgbe_if_enable_intr(if_ctx_t ctx);
 static void ixgbe_if_disable_intr(if_ctx_t ctx);
 static void ixgbe_link_intr_enable(if_ctx_t ctx);
 static int  ixgbe_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t qid);
 static void ixgbe_if_media_status(if_ctx_t ctx, struct ifmediareq * ifmr);
 static int  ixgbe_if_media_change(if_ctx_t ctx);
 static int  ixgbe_if_msix_intr_assign(if_ctx_t, int);
 static int  ixgbe_if_mtu_set(if_ctx_t ctx, uint32_t mtu);
 static void ixgbe_if_crcstrip_set(if_ctx_t ctx, int onoff, int strip);
 static void ixgbe_if_multi_set(if_ctx_t ctx);
 static int  ixgbe_if_promisc_set(if_ctx_t ctx, int flags);
 static int  ixgbe_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs,
                                      uint64_t *paddrs, int nrxqs, int nrxqsets);
 static int  ixgbe_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs,
                                      uint64_t *paddrs, int nrxqs, int nrxqsets);
 static void ixgbe_if_queues_free(if_ctx_t ctx);
 static void ixgbe_if_timer(if_ctx_t ctx, uint16_t);
 static void ixgbe_if_update_admin_status(if_ctx_t ctx);
 static void ixgbe_if_vlan_register(if_ctx_t ctx, u16 vtag);
 static void ixgbe_if_vlan_unregister(if_ctx_t ctx, u16 vtag);
 static int  ixgbe_if_i2c_req(if_ctx_t ctx, struct ifi2creq *req);
 int ixgbe_intr(void *arg);
 
 /************************************************************************
  * Function prototypes
  ************************************************************************/
 #if __FreeBSD_version >= 1100036
 static uint64_t ixgbe_if_get_counter(if_ctx_t, ift_counter);
 #endif
 
 static void ixgbe_enable_queue(struct adapter *adapter, u32 vector);
 static void ixgbe_disable_queue(struct adapter *adapter, u32 vector);
 static void ixgbe_add_device_sysctls(if_ctx_t ctx);
 static int  ixgbe_allocate_pci_resources(if_ctx_t ctx);
 static int  ixgbe_setup_low_power_mode(if_ctx_t ctx);
 
 static void ixgbe_config_dmac(struct adapter *adapter);
 static void ixgbe_configure_ivars(struct adapter *adapter);
 static void ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector,
                            s8 type);
 static u8   *ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
 static bool ixgbe_sfp_probe(if_ctx_t ctx);
 
 static void ixgbe_free_pci_resources(if_ctx_t ctx);
 
 static int  ixgbe_msix_link(void *arg);
 static int  ixgbe_msix_que(void *arg);
 static void ixgbe_initialize_rss_mapping(struct adapter *adapter);
 static void ixgbe_initialize_receive_units(if_ctx_t ctx);
 static void ixgbe_initialize_transmit_units(if_ctx_t ctx);
 
 static int  ixgbe_setup_interface(if_ctx_t ctx);
 static void ixgbe_init_device_features(struct adapter *adapter);
 static void ixgbe_check_fan_failure(struct adapter *, u32, bool);
 static void ixgbe_add_media_types(if_ctx_t ctx);
 static void ixgbe_update_stats_counters(struct adapter *adapter);
 static void ixgbe_config_link(if_ctx_t ctx);
 static void ixgbe_get_slot_info(struct adapter *);
 static void ixgbe_check_wol_support(struct adapter *adapter);
 static void ixgbe_enable_rx_drop(struct adapter *);
 static void ixgbe_disable_rx_drop(struct adapter *);
 
 static void ixgbe_add_hw_stats(struct adapter *adapter);
 static int  ixgbe_set_flowcntl(struct adapter *, int);
 static int  ixgbe_set_advertise(struct adapter *, int);
 static int  ixgbe_get_advertise(struct adapter *);
 static void ixgbe_setup_vlan_hw_support(if_ctx_t ctx);
 static void ixgbe_config_gpie(struct adapter *adapter);
 static void ixgbe_config_delay_values(struct adapter *adapter);
 
 /* Sysctl handlers */
 static int  ixgbe_sysctl_flowcntl(SYSCTL_HANDLER_ARGS);
 static int  ixgbe_sysctl_advertise(SYSCTL_HANDLER_ARGS);
 static int  ixgbe_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS);
 static int  ixgbe_sysctl_dmac(SYSCTL_HANDLER_ARGS);
 static int  ixgbe_sysctl_phy_temp(SYSCTL_HANDLER_ARGS);
 static int  ixgbe_sysctl_phy_overtemp_occurred(SYSCTL_HANDLER_ARGS);
 #ifdef IXGBE_DEBUG
 static int  ixgbe_sysctl_power_state(SYSCTL_HANDLER_ARGS);
 static int  ixgbe_sysctl_print_rss_config(SYSCTL_HANDLER_ARGS);
 #endif
 static int  ixgbe_sysctl_rdh_handler(SYSCTL_HANDLER_ARGS);
 static int  ixgbe_sysctl_rdt_handler(SYSCTL_HANDLER_ARGS);
 static int  ixgbe_sysctl_tdt_handler(SYSCTL_HANDLER_ARGS);
 static int  ixgbe_sysctl_tdh_handler(SYSCTL_HANDLER_ARGS);
 static int  ixgbe_sysctl_eee_state(SYSCTL_HANDLER_ARGS);
 static int  ixgbe_sysctl_wol_enable(SYSCTL_HANDLER_ARGS);
 static int  ixgbe_sysctl_wufc(SYSCTL_HANDLER_ARGS);
 
 /* Deferred interrupt tasklets */
 static void ixgbe_handle_msf(void *);
 static void ixgbe_handle_mod(void *);
 static void ixgbe_handle_phy(void *);
 
 /************************************************************************
  *  FreeBSD Device Interface Entry Points
  ************************************************************************/
 static device_method_t ix_methods[] = {
 	/* Device interface */
 	DEVMETHOD(device_register, ixgbe_register),
 	DEVMETHOD(device_probe, iflib_device_probe),
 	DEVMETHOD(device_attach, iflib_device_attach),
 	DEVMETHOD(device_detach, iflib_device_detach),
 	DEVMETHOD(device_shutdown, iflib_device_shutdown),
 	DEVMETHOD(device_suspend, iflib_device_suspend),
 	DEVMETHOD(device_resume, iflib_device_resume),
 #ifdef PCI_IOV
 	DEVMETHOD(pci_iov_init, iflib_device_iov_init),
 	DEVMETHOD(pci_iov_uninit, iflib_device_iov_uninit),
 	DEVMETHOD(pci_iov_add_vf, iflib_device_iov_add_vf),
 #endif /* PCI_IOV */
 	DEVMETHOD_END
 };
 
 static driver_t ix_driver = {
 	"ix", ix_methods, sizeof(struct adapter),
 };
 
 devclass_t ix_devclass;
 DRIVER_MODULE(ix, pci, ix_driver, ix_devclass, 0, 0);
 IFLIB_PNP_INFO(pci, ix_driver, ixgbe_vendor_info_array);
 MODULE_DEPEND(ix, pci, 1, 1, 1);
 MODULE_DEPEND(ix, ether, 1, 1, 1);
 MODULE_DEPEND(ix, iflib, 1, 1, 1);
 
 static device_method_t ixgbe_if_methods[] = {
 	DEVMETHOD(ifdi_attach_pre, ixgbe_if_attach_pre),
 	DEVMETHOD(ifdi_attach_post, ixgbe_if_attach_post),
 	DEVMETHOD(ifdi_detach, ixgbe_if_detach),
 	DEVMETHOD(ifdi_shutdown, ixgbe_if_shutdown),
 	DEVMETHOD(ifdi_suspend, ixgbe_if_suspend),
 	DEVMETHOD(ifdi_resume, ixgbe_if_resume),
 	DEVMETHOD(ifdi_init, ixgbe_if_init),
 	DEVMETHOD(ifdi_stop, ixgbe_if_stop),
 	DEVMETHOD(ifdi_msix_intr_assign, ixgbe_if_msix_intr_assign),
 	DEVMETHOD(ifdi_intr_enable, ixgbe_if_enable_intr),
 	DEVMETHOD(ifdi_intr_disable, ixgbe_if_disable_intr),
 	DEVMETHOD(ifdi_link_intr_enable, ixgbe_link_intr_enable),
 	DEVMETHOD(ifdi_tx_queue_intr_enable, ixgbe_if_rx_queue_intr_enable),
 	DEVMETHOD(ifdi_rx_queue_intr_enable, ixgbe_if_rx_queue_intr_enable),
 	DEVMETHOD(ifdi_tx_queues_alloc, ixgbe_if_tx_queues_alloc),
 	DEVMETHOD(ifdi_rx_queues_alloc, ixgbe_if_rx_queues_alloc),
 	DEVMETHOD(ifdi_queues_free, ixgbe_if_queues_free),
 	DEVMETHOD(ifdi_update_admin_status, ixgbe_if_update_admin_status),
 	DEVMETHOD(ifdi_multi_set, ixgbe_if_multi_set),
 	DEVMETHOD(ifdi_mtu_set, ixgbe_if_mtu_set),
 	DEVMETHOD(ifdi_crcstrip_set, ixgbe_if_crcstrip_set),
 	DEVMETHOD(ifdi_media_status, ixgbe_if_media_status),
 	DEVMETHOD(ifdi_media_change, ixgbe_if_media_change),
 	DEVMETHOD(ifdi_promisc_set, ixgbe_if_promisc_set),
 	DEVMETHOD(ifdi_timer, ixgbe_if_timer),
 	DEVMETHOD(ifdi_vlan_register, ixgbe_if_vlan_register),
 	DEVMETHOD(ifdi_vlan_unregister, ixgbe_if_vlan_unregister),
 	DEVMETHOD(ifdi_get_counter, ixgbe_if_get_counter),
 	DEVMETHOD(ifdi_i2c_req, ixgbe_if_i2c_req),
 #ifdef PCI_IOV
 	DEVMETHOD(ifdi_iov_init, ixgbe_if_iov_init),
 	DEVMETHOD(ifdi_iov_uninit, ixgbe_if_iov_uninit),
 	DEVMETHOD(ifdi_iov_vf_add, ixgbe_if_iov_vf_add),
 #endif /* PCI_IOV */
 	DEVMETHOD_END
 };
 
 /*
  * TUNEABLE PARAMETERS:
  */
 
 static SYSCTL_NODE(_hw, OID_AUTO, ix, CTLFLAG_RD, 0, "IXGBE driver parameters");
 static driver_t ixgbe_if_driver = {
   "ixgbe_if", ixgbe_if_methods, sizeof(struct adapter)
 };
 
 static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY);
 SYSCTL_INT(_hw_ix, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN,
     &ixgbe_max_interrupt_rate, 0, "Maximum interrupts per second");
 
 /* Flow control setting, default to full */
 static int ixgbe_flow_control = ixgbe_fc_full;
 SYSCTL_INT(_hw_ix, OID_AUTO, flow_control, CTLFLAG_RDTUN,
     &ixgbe_flow_control, 0, "Default flow control used for all adapters");
 
 /* Advertise Speed, default to 0 (auto) */
 static int ixgbe_advertise_speed = 0;
 SYSCTL_INT(_hw_ix, OID_AUTO, advertise_speed, CTLFLAG_RDTUN,
     &ixgbe_advertise_speed, 0, "Default advertised speed for all adapters");
 
 /*
  * Smart speed setting, default to on
  * this only works as a compile option
  * right now as its during attach, set
  * this to 'ixgbe_smart_speed_off' to
  * disable.
  */
 static int ixgbe_smart_speed = ixgbe_smart_speed_on;
 
 /*
  * MSI-X should be the default for best performance,
  * but this allows it to be forced off for testing.
  */
 static int ixgbe_enable_msix = 1;
 SYSCTL_INT(_hw_ix, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &ixgbe_enable_msix, 0,
     "Enable MSI-X interrupts");
 
 /*
  * Defining this on will allow the use
  * of unsupported SFP+ modules, note that
  * doing so you are on your own :)
  */
 static int allow_unsupported_sfp = FALSE;
 SYSCTL_INT(_hw_ix, OID_AUTO, unsupported_sfp, CTLFLAG_RDTUN,
     &allow_unsupported_sfp, 0,
     "Allow unsupported SFP modules...use at your own risk");
 
 /*
  * Not sure if Flow Director is fully baked,
  * so we'll default to turning it off.
  */
 static int ixgbe_enable_fdir = 0;
 SYSCTL_INT(_hw_ix, OID_AUTO, enable_fdir, CTLFLAG_RDTUN, &ixgbe_enable_fdir, 0,
     "Enable Flow Director");
 
 /* Receive-Side Scaling */
 static int ixgbe_enable_rss = 1;
 SYSCTL_INT(_hw_ix, OID_AUTO, enable_rss, CTLFLAG_RDTUN, &ixgbe_enable_rss, 0,
     "Enable Receive-Side Scaling (RSS)");
 
 #if 0
 /* Keep running tab on them for sanity check */
 static int ixgbe_total_ports;
 #endif
 
 MALLOC_DEFINE(M_IXGBE, "ix", "ix driver allocations");
 
 /*
  * For Flow Director: this is the number of TX packets we sample
  * for the filter pool, this means every 20th packet will be probed.
  *
  * This feature can be disabled by setting this to 0.
  */
 static int atr_sample_rate = 20;
 
 extern struct if_txrx ixgbe_txrx;
 
 static struct if_shared_ctx ixgbe_sctx_init = {
 	.isc_magic = IFLIB_MAGIC,
 	.isc_q_align = PAGE_SIZE,/* max(DBA_ALIGN, PAGE_SIZE) */
 	.isc_tx_maxsize = IXGBE_TSO_SIZE + sizeof(struct ether_vlan_header),
 	.isc_tx_maxsegsize = PAGE_SIZE,
 	.isc_tso_maxsize = IXGBE_TSO_SIZE + sizeof(struct ether_vlan_header),
 	.isc_tso_maxsegsize = PAGE_SIZE,
 	.isc_rx_maxsize = PAGE_SIZE*4,
 	.isc_rx_nsegments = 1,
 	.isc_rx_maxsegsize = PAGE_SIZE*4,
 	.isc_nfl = 1,
 	.isc_ntxqs = 1,
 	.isc_nrxqs = 1,
 
 	.isc_admin_intrcnt = 1,
 	.isc_vendor_info = ixgbe_vendor_info_array,
 	.isc_driver_version = ixgbe_driver_version,
 	.isc_driver = &ixgbe_if_driver,
 	.isc_flags = IFLIB_TSO_INIT_IP,
 
 	.isc_nrxd_min = {MIN_RXD},
 	.isc_ntxd_min = {MIN_TXD},
 	.isc_nrxd_max = {MAX_RXD},
 	.isc_ntxd_max = {MAX_TXD},
 	.isc_nrxd_default = {DEFAULT_RXD},
 	.isc_ntxd_default = {DEFAULT_TXD},
 };
 
 if_shared_ctx_t ixgbe_sctx = &ixgbe_sctx_init;
 
 /************************************************************************
  * ixgbe_if_tx_queues_alloc
  ************************************************************************/
 static int
 ixgbe_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs,
                          int ntxqs, int ntxqsets)
 {
 	struct adapter     *adapter = iflib_get_softc(ctx);
 	if_softc_ctx_t     scctx = adapter->shared;
 	struct ix_tx_queue *que;
 	int                i, j, error;
 
 	MPASS(adapter->num_tx_queues > 0);
 	MPASS(adapter->num_tx_queues == ntxqsets);
 	MPASS(ntxqs == 1);
 
 	/* Allocate queue structure memory */
 	adapter->tx_queues =
 	    (struct ix_tx_queue *)malloc(sizeof(struct ix_tx_queue) * ntxqsets,
 	                                 M_IXGBE, M_NOWAIT | M_ZERO);
 	if (!adapter->tx_queues) {
 		device_printf(iflib_get_dev(ctx),
 		    "Unable to allocate TX ring memory\n");
 		return (ENOMEM);
 	}
 
 	for (i = 0, que = adapter->tx_queues; i < ntxqsets; i++, que++) {
 		struct tx_ring *txr = &que->txr;
 
 		/* In case SR-IOV is enabled, align the index properly */
 		txr->me = ixgbe_vf_que_index(adapter->iov_mode, adapter->pool,
 		    i);
 
 		txr->adapter = que->adapter = adapter;
 		adapter->active_queues |= (u64)1 << txr->me;
 
 		/* Allocate report status array */
 		txr->tx_rsq = (qidx_t *)malloc(sizeof(qidx_t) * scctx->isc_ntxd[0], M_IXGBE, M_NOWAIT | M_ZERO);
 		if (txr->tx_rsq == NULL) {
 			error = ENOMEM;
 			goto fail;
 		}
 		for (j = 0; j < scctx->isc_ntxd[0]; j++)
 			txr->tx_rsq[j] = QIDX_INVALID;
 		/* get the virtual and physical address of the hardware queues */
 		txr->tail = IXGBE_TDT(txr->me);
 		txr->tx_base = (union ixgbe_adv_tx_desc *)vaddrs[i];
 		txr->tx_paddr = paddrs[i];
 
 		txr->bytes = 0;
 		txr->total_packets = 0;
 
 		/* Set the rate at which we sample packets */
 		if (adapter->feat_en & IXGBE_FEATURE_FDIR)
 			txr->atr_sample = atr_sample_rate;
 
 	}
 
 	device_printf(iflib_get_dev(ctx), "allocated for %d queues\n",
 	    adapter->num_tx_queues);
 
 	return (0);
 
 fail:
 	ixgbe_if_queues_free(ctx);
 
 	return (error);
 } /* ixgbe_if_tx_queues_alloc */
 
 /************************************************************************
  * ixgbe_if_rx_queues_alloc
  ************************************************************************/
 static int
 ixgbe_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs,
                          int nrxqs, int nrxqsets)
 {
 	struct adapter     *adapter = iflib_get_softc(ctx);
 	struct ix_rx_queue *que;
 	int                i;
 
 	MPASS(adapter->num_rx_queues > 0);
 	MPASS(adapter->num_rx_queues == nrxqsets);
 	MPASS(nrxqs == 1);
 
 	/* Allocate queue structure memory */
 	adapter->rx_queues =
 	    (struct ix_rx_queue *)malloc(sizeof(struct ix_rx_queue)*nrxqsets,
 	                                 M_IXGBE, M_NOWAIT | M_ZERO);
 	if (!adapter->rx_queues) {
 		device_printf(iflib_get_dev(ctx),
 		    "Unable to allocate TX ring memory\n");
 		return (ENOMEM);
 	}
 
 	for (i = 0, que = adapter->rx_queues; i < nrxqsets; i++, que++) {
 		struct rx_ring *rxr = &que->rxr;
 
 		/* In case SR-IOV is enabled, align the index properly */
 		rxr->me = ixgbe_vf_que_index(adapter->iov_mode, adapter->pool,
 		    i);
 
 		rxr->adapter = que->adapter = adapter;
 
 		/* get the virtual and physical address of the hw queues */
 		rxr->tail = IXGBE_RDT(rxr->me);
 		rxr->rx_base = (union ixgbe_adv_rx_desc *)vaddrs[i];
 		rxr->rx_paddr = paddrs[i];
 		rxr->bytes = 0;
 		rxr->que = que;
 	}
 
 	device_printf(iflib_get_dev(ctx), "allocated for %d rx queues\n",
 	    adapter->num_rx_queues);
 
 	return (0);
 } /* ixgbe_if_rx_queues_alloc */
 
 /************************************************************************
  * ixgbe_if_queues_free
  ************************************************************************/
 static void
 ixgbe_if_queues_free(if_ctx_t ctx)
 {
 	struct adapter     *adapter = iflib_get_softc(ctx);
 	struct ix_tx_queue *tx_que = adapter->tx_queues;
 	struct ix_rx_queue *rx_que = adapter->rx_queues;
 	int                i;
 
 	if (tx_que != NULL) {
 		for (i = 0; i < adapter->num_tx_queues; i++, tx_que++) {
 			struct tx_ring *txr = &tx_que->txr;
 			if (txr->tx_rsq == NULL)
 				break;
 
 			free(txr->tx_rsq, M_IXGBE);
 			txr->tx_rsq = NULL;
 		}
 
 		free(adapter->tx_queues, M_IXGBE);
 		adapter->tx_queues = NULL;
 	}
 	if (rx_que != NULL) {
 		free(adapter->rx_queues, M_IXGBE);
 		adapter->rx_queues = NULL;
 	}
 } /* ixgbe_if_queues_free */
 
 /************************************************************************
  * ixgbe_initialize_rss_mapping
  ************************************************************************/
 static void
 ixgbe_initialize_rss_mapping(struct adapter *adapter)
 {
 	struct ixgbe_hw *hw = &adapter->hw;
 	u32             reta = 0, mrqc, rss_key[10];
 	int             queue_id, table_size, index_mult;
 	int             i, j;
 	u32             rss_hash_config;
 
 	if (adapter->feat_en & IXGBE_FEATURE_RSS) {
 		/* Fetch the configured RSS key */
 		rss_getkey((uint8_t *)&rss_key);
 	} else {
 		/* set up random bits */
 		arc4rand(&rss_key, sizeof(rss_key), 0);
 	}
 
 	/* Set multiplier for RETA setup and table size based on MAC */
 	index_mult = 0x1;
 	table_size = 128;
 	switch (adapter->hw.mac.type) {
 	case ixgbe_mac_82598EB:
 		index_mult = 0x11;
 		break;
 	case ixgbe_mac_X550:
 	case ixgbe_mac_X550EM_x:
 	case ixgbe_mac_X550EM_a:
 		table_size = 512;
 		break;
 	default:
 		break;
 	}
 
 	/* Set up the redirection table */
 	for (i = 0, j = 0; i < table_size; i++, j++) {
 		if (j == adapter->num_rx_queues)
 			j = 0;
 
 		if (adapter->feat_en & IXGBE_FEATURE_RSS) {
 			/*
 			 * Fetch the RSS bucket id for the given indirection
 			 * entry. Cap it at the number of configured buckets
 			 * (which is num_rx_queues.)
 			 */
 			queue_id = rss_get_indirection_to_bucket(i);
 			queue_id = queue_id % adapter->num_rx_queues;
 		} else
 			queue_id = (j * index_mult);
 
 		/*
 		 * The low 8 bits are for hash value (n+0);
 		 * The next 8 bits are for hash value (n+1), etc.
 		 */
 		reta = reta >> 8;
 		reta = reta | (((uint32_t)queue_id) << 24);
 		if ((i & 3) == 3) {
 			if (i < 128)
 				IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
 			else
 				IXGBE_WRITE_REG(hw, IXGBE_ERETA((i >> 2) - 32),
 				    reta);
 			reta = 0;
 		}
 	}
 
 	/* Now fill our hash function seeds */
 	for (i = 0; i < 10; i++)
 		IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), rss_key[i]);
 
 	/* Perform hash on these packet types */
 	if (adapter->feat_en & IXGBE_FEATURE_RSS)
 		rss_hash_config = rss_gethashconfig();
 	else {
 		/*
 		 * Disable UDP - IP fragments aren't currently being handled
 		 * and so we end up with a mix of 2-tuple and 4-tuple
 		 * traffic.
 		 */
 		rss_hash_config = RSS_HASHTYPE_RSS_IPV4
 		                | RSS_HASHTYPE_RSS_TCP_IPV4
 		                | RSS_HASHTYPE_RSS_IPV6
 		                | RSS_HASHTYPE_RSS_TCP_IPV6
 		                | RSS_HASHTYPE_RSS_IPV6_EX
 		                | RSS_HASHTYPE_RSS_TCP_IPV6_EX;
 	}
 
 	mrqc = IXGBE_MRQC_RSSEN;
 	if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4)
 		mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4;
 	if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4)
 		mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP;
 	if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6)
 		mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6;
 	if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6)
 		mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP;
 	if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX)
 		mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX;
 	if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6_EX)
 		mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP;
 	if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4)
 		mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP;
 	if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6)
 		mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP;
 	if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6_EX)
 		mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
 	mrqc |= ixgbe_get_mrqc(adapter->iov_mode);
 	IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
 } /* ixgbe_initialize_rss_mapping */
 
 /************************************************************************
  * ixgbe_initialize_receive_units - Setup receive registers and features.
  ************************************************************************/
 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1)
 
 static void
 ixgbe_initialize_receive_units(if_ctx_t ctx)
 {
 	struct adapter     *adapter = iflib_get_softc(ctx);
 	if_softc_ctx_t     scctx = adapter->shared;
 	struct ixgbe_hw    *hw = &adapter->hw;
 	struct ifnet       *ifp = iflib_get_ifp(ctx);
 	struct ix_rx_queue *que;
 	int                i, j;
 	u32                bufsz, fctrl, srrctl, rxcsum;
 	u32                hlreg;
 
 	/*
 	 * Make sure receives are disabled while
 	 * setting up the descriptor ring
 	 */
 	ixgbe_disable_rx(hw);
 
 	/* Enable broadcasts */
 	fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
 	fctrl |= IXGBE_FCTRL_BAM;
 	if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
 		fctrl |= IXGBE_FCTRL_DPF;
 		fctrl |= IXGBE_FCTRL_PMCF;
 	}
 	IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
 
 	/* Set for Jumbo Frames? */
 	hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
 	if (ifp->if_mtu > ETHERMTU)
 		hlreg |= IXGBE_HLREG0_JUMBOEN;
 	else
 		hlreg &= ~IXGBE_HLREG0_JUMBOEN;
 	IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
 
 	bufsz = (adapter->rx_mbuf_sz + BSIZEPKT_ROUNDUP) >>
 	    IXGBE_SRRCTL_BSIZEPKT_SHIFT;
 
 	/* Setup the Base and Length of the Rx Descriptor Ring */
 	for (i = 0, que = adapter->rx_queues; i < adapter->num_rx_queues; i++, que++) {
 		struct rx_ring *rxr = &que->rxr;
 		u64            rdba = rxr->rx_paddr;
 
 		j = rxr->me;
 
 		/* Setup the Base and Length of the Rx Descriptor Ring */
 		IXGBE_WRITE_REG(hw, IXGBE_RDBAL(j),
 		    (rdba & 0x00000000ffffffffULL));
 		IXGBE_WRITE_REG(hw, IXGBE_RDBAH(j), (rdba >> 32));
 		IXGBE_WRITE_REG(hw, IXGBE_RDLEN(j),
 		     scctx->isc_nrxd[0] * sizeof(union ixgbe_adv_rx_desc));
 
 		/* Set up the SRRCTL register */
 		srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(j));
 		srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
 		srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
 		srrctl |= bufsz;
 		srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
 
 		/*
 		 * Set DROP_EN iff we have no flow control and >1 queue.
 		 * Note that srrctl was cleared shortly before during reset,
 		 * so we do not need to clear the bit, but do it just in case
 		 * this code is moved elsewhere.
 		 */
 		if (adapter->num_rx_queues > 1 &&
 		    adapter->hw.fc.requested_mode == ixgbe_fc_none) {
 			srrctl |= IXGBE_SRRCTL_DROP_EN;
 		} else {
 			srrctl &= ~IXGBE_SRRCTL_DROP_EN;
 		}
 
 		IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(j), srrctl);
 
 		/* Setup the HW Rx Head and Tail Descriptor Pointers */
 		IXGBE_WRITE_REG(hw, IXGBE_RDH(j), 0);
 		IXGBE_WRITE_REG(hw, IXGBE_RDT(j), 0);
 
 		/* Set the driver rx tail address */
 		rxr->tail =  IXGBE_RDT(rxr->me);
 	}
 
 	if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
 		u32 psrtype = IXGBE_PSRTYPE_TCPHDR
 		            | IXGBE_PSRTYPE_UDPHDR
 		            | IXGBE_PSRTYPE_IPV4HDR
 		            | IXGBE_PSRTYPE_IPV6HDR;
 		IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype);
 	}
 
 	rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
 
 	ixgbe_initialize_rss_mapping(adapter);
 
 	if (adapter->num_rx_queues > 1) {
 		/* RSS and RX IPP Checksum are mutually exclusive */
 		rxcsum |= IXGBE_RXCSUM_PCSD;
 	}
 
 	if (ifp->if_capenable & IFCAP_RXCSUM)
 		rxcsum |= IXGBE_RXCSUM_PCSD;
 
 	/* This is useful for calculating UDP/IP fragment checksums */
 	if (!(rxcsum & IXGBE_RXCSUM_PCSD))
 		rxcsum |= IXGBE_RXCSUM_IPPCSE;
 
 	IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
 
 } /* ixgbe_initialize_receive_units */
 
 /************************************************************************
  * ixgbe_initialize_transmit_units - Enable transmit units.
  ************************************************************************/
 static void
 ixgbe_initialize_transmit_units(if_ctx_t ctx)
 {
 	struct adapter     *adapter = iflib_get_softc(ctx);
 	struct ixgbe_hw    *hw = &adapter->hw;
 	if_softc_ctx_t     scctx = adapter->shared;
 	struct ix_tx_queue *que;
 	int i;
 
 	/* Setup the Base and Length of the Tx Descriptor Ring */
 	for (i = 0, que = adapter->tx_queues; i < adapter->num_tx_queues;
 	    i++, que++) {
 		struct tx_ring	   *txr = &que->txr;
 		u64 tdba = txr->tx_paddr;
 		u32 txctrl = 0;
 		int j = txr->me;
 
 		IXGBE_WRITE_REG(hw, IXGBE_TDBAL(j),
 		    (tdba & 0x00000000ffffffffULL));
 		IXGBE_WRITE_REG(hw, IXGBE_TDBAH(j), (tdba >> 32));
 		IXGBE_WRITE_REG(hw, IXGBE_TDLEN(j),
 		    scctx->isc_ntxd[0] * sizeof(union ixgbe_adv_tx_desc));
 
 		/* Setup the HW Tx Head and Tail descriptor pointers */
 		IXGBE_WRITE_REG(hw, IXGBE_TDH(j), 0);
 		IXGBE_WRITE_REG(hw, IXGBE_TDT(j), 0);
 
 		/* Cache the tail address */
 		txr->tx_rs_cidx = txr->tx_rs_pidx;
 		txr->tx_cidx_processed = scctx->isc_ntxd[0] - 1;
 		for (int k = 0; k < scctx->isc_ntxd[0]; k++)
 			txr->tx_rsq[k] = QIDX_INVALID;
 
 		/* Disable Head Writeback */
 		/*
 		 * Note: for X550 series devices, these registers are actually
 		 * prefixed with TPH_ isntead of DCA_, but the addresses and
 		 * fields remain the same.
 		 */
 		switch (hw->mac.type) {
 		case ixgbe_mac_82598EB:
 			txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(j));
 			break;
 		default:
 			txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(j));
 			break;
 		}
 		txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
 		switch (hw->mac.type) {
 		case ixgbe_mac_82598EB:
 			IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(j), txctrl);
 			break;
 		default:
 			IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(j), txctrl);
 			break;
 		}
 
 	}
 
 	if (hw->mac.type != ixgbe_mac_82598EB) {
 		u32 dmatxctl, rttdcs;
 
 		dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
 		dmatxctl |= IXGBE_DMATXCTL_TE;
 		IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
 		/* Disable arbiter to set MTQC */
 		rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
 		rttdcs |= IXGBE_RTTDCS_ARBDIS;
 		IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
 		IXGBE_WRITE_REG(hw, IXGBE_MTQC,
 		    ixgbe_get_mtqc(adapter->iov_mode));
 		rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
 		IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
 	}
 
 } /* ixgbe_initialize_transmit_units */
 
 /************************************************************************
  * ixgbe_register
  ************************************************************************/
 static void *
 ixgbe_register(device_t dev)
 {
 	return (ixgbe_sctx);
 } /* ixgbe_register */
 
 /************************************************************************
  * ixgbe_if_attach_pre - Device initialization routine, part 1
  *
  *   Called when the driver is being loaded.
  *   Identifies the type of hardware, initializes the hardware,
  *   and initializes iflib structures.
  *
  *   return 0 on success, positive on failure
  ************************************************************************/
 static int
 ixgbe_if_attach_pre(if_ctx_t ctx)
 {
 	struct adapter  *adapter;
 	device_t        dev;
 	if_softc_ctx_t  scctx;
 	struct ixgbe_hw *hw;
 	int             error = 0;
 	u32             ctrl_ext;
 
 	INIT_DEBUGOUT("ixgbe_attach: begin");
 
 	/* Allocate, clear, and link in our adapter structure */
 	dev = iflib_get_dev(ctx);
 	adapter = iflib_get_softc(ctx);
 	adapter->hw.back = adapter;
 	adapter->ctx = ctx;
 	adapter->dev = dev;
 	scctx = adapter->shared = iflib_get_softc_ctx(ctx);
 	adapter->media = iflib_get_media(ctx);
 	hw = &adapter->hw;
 
 	/* Determine hardware revision */
 	hw->vendor_id = pci_get_vendor(dev);
 	hw->device_id = pci_get_device(dev);
 	hw->revision_id = pci_get_revid(dev);
 	hw->subsystem_vendor_id = pci_get_subvendor(dev);
 	hw->subsystem_device_id = pci_get_subdevice(dev);
 
 	/* Do base PCI setup - map BAR0 */
 	if (ixgbe_allocate_pci_resources(ctx)) {
 		device_printf(dev, "Allocation of PCI resources failed\n");
 		return (ENXIO);
 	}
 
 	/* let hardware know driver is loaded */
 	ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 	ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
 	IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 
 	/*
 	 * Initialize the shared code
 	 */
 	if (ixgbe_init_shared_code(hw) != 0) {
 		device_printf(dev, "Unable to initialize the shared code\n");
 		error = ENXIO;
 		goto err_pci;
 	}
 
 	if (hw->mbx.ops.init_params)
 		hw->mbx.ops.init_params(hw);
 
 	hw->allow_unsupported_sfp = allow_unsupported_sfp;
 
 	if (hw->mac.type != ixgbe_mac_82598EB)
 		hw->phy.smart_speed = ixgbe_smart_speed;
 
 	ixgbe_init_device_features(adapter);
 
 	/* Enable WoL (if supported) */
 	ixgbe_check_wol_support(adapter);
 
 	/* Verify adapter fan is still functional (if applicable) */
 	if (adapter->feat_en & IXGBE_FEATURE_FAN_FAIL) {
 		u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP);
 		ixgbe_check_fan_failure(adapter, esdp, FALSE);
 	}
 
 	/* Ensure SW/FW semaphore is free */
 	ixgbe_init_swfw_semaphore(hw);
 
 	/* Set an initial default flow control value */
 	hw->fc.requested_mode = ixgbe_flow_control;
 
 	hw->phy.reset_if_overtemp = TRUE;
 	error = ixgbe_reset_hw(hw);
 	hw->phy.reset_if_overtemp = FALSE;
 	if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
 		/*
 		 * No optics in this port, set up
 		 * so the timer routine will probe
 		 * for later insertion.
 		 */
 		adapter->sfp_probe = TRUE;
 		error = 0;
 	} else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
 		device_printf(dev, "Unsupported SFP+ module detected!\n");
 		error = EIO;
 		goto err_pci;
 	} else if (error) {
 		device_printf(dev, "Hardware initialization failed\n");
 		error = EIO;
 		goto err_pci;
 	}
 
 	/* Make sure we have a good EEPROM before we read from it */
 	if (ixgbe_validate_eeprom_checksum(&adapter->hw, NULL) < 0) {
 		device_printf(dev, "The EEPROM Checksum Is Not Valid\n");
 		error = EIO;
 		goto err_pci;
 	}
 
 	error = ixgbe_start_hw(hw);
 	switch (error) {
 	case IXGBE_ERR_EEPROM_VERSION:
 		device_printf(dev, "This device is a pre-production adapter/LOM.  Please be aware there may be issues associated with your hardware.\nIf you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n");
 		break;
 	case IXGBE_ERR_SFP_NOT_SUPPORTED:
 		device_printf(dev, "Unsupported SFP+ Module\n");
 		error = EIO;
 		goto err_pci;
 	case IXGBE_ERR_SFP_NOT_PRESENT:
 		device_printf(dev, "No SFP+ Module found\n");
 		/* falls thru */
 	default:
 		break;
 	}
 
 	/* Most of the iflib initialization... */
 
 	iflib_set_mac(ctx, hw->mac.addr);
 	switch (adapter->hw.mac.type) {
 	case ixgbe_mac_X550:
 	case ixgbe_mac_X550EM_x:
 	case ixgbe_mac_X550EM_a:
 		scctx->isc_rss_table_size = 512;
 		scctx->isc_ntxqsets_max = scctx->isc_nrxqsets_max = 64;
 		break;
 	default:
 		scctx->isc_rss_table_size = 128;
 		scctx->isc_ntxqsets_max = scctx->isc_nrxqsets_max = 16;
 	}
 
 	/* Allow legacy interrupts */
 	ixgbe_txrx.ift_legacy_intr = ixgbe_intr;
 
 	scctx->isc_txqsizes[0] =
 	    roundup2(scctx->isc_ntxd[0] * sizeof(union ixgbe_adv_tx_desc) +
 	    sizeof(u32), DBA_ALIGN),
 	scctx->isc_rxqsizes[0] =
 	    roundup2(scctx->isc_nrxd[0] * sizeof(union ixgbe_adv_rx_desc),
 	    DBA_ALIGN);
 
 	/* XXX */
 	scctx->isc_tx_csum_flags = CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_TSO |
 	    CSUM_IP6_TCP | CSUM_IP6_UDP | CSUM_IP6_TSO;
 	if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
 		scctx->isc_tx_nsegments = IXGBE_82598_SCATTER;
 		scctx->isc_msix_bar = PCIR_BAR(MSIX_82598_BAR);
 	} else {
 		scctx->isc_tx_csum_flags |= CSUM_SCTP |CSUM_IP6_SCTP;
 		scctx->isc_tx_nsegments = IXGBE_82599_SCATTER;
 		scctx->isc_msix_bar = PCIR_BAR(MSIX_82599_BAR);
 	}
 	scctx->isc_tx_tso_segments_max = scctx->isc_tx_nsegments;
 	scctx->isc_tx_tso_size_max = IXGBE_TSO_SIZE;
 	scctx->isc_tx_tso_segsize_max = PAGE_SIZE;
 
 	scctx->isc_txrx = &ixgbe_txrx;
 
 	scctx->isc_capabilities = scctx->isc_capenable = IXGBE_CAPS;
 
 	return (0);
 
 err_pci:
 	ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
 	ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
 	IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
 	ixgbe_free_pci_resources(ctx);
 
 	return (error);
 } /* ixgbe_if_attach_pre */
 
  /*********************************************************************
  * ixgbe_if_attach_post - Device initialization routine, part 2
  *
  *   Called during driver load, but after interrupts and
  *   resources have been allocated and configured.
  *   Sets up some data structures not relevant to iflib.
  *
  *   return 0 on success, positive on failure
  *********************************************************************/
 static int
 ixgbe_if_attach_post(if_ctx_t ctx)
 {
 	device_t dev;
 	struct adapter  *adapter;
 	struct ixgbe_hw *hw;
 	int             error = 0;
 
 	dev = iflib_get_dev(ctx);
 	adapter = iflib_get_softc(ctx);
 	hw = &adapter->hw;
 
 
 	if (adapter->intr_type == IFLIB_INTR_LEGACY &&
 		(adapter->feat_cap & IXGBE_FEATURE_LEGACY_IRQ) == 0) {
 		device_printf(dev, "Device does not support legacy interrupts");
 		error = ENXIO;
 		goto err;
 	}
 
 	/* Allocate multicast array memory. */
 	adapter->mta = malloc(sizeof(*adapter->mta) *
 	                      MAX_NUM_MULTICAST_ADDRESSES, M_IXGBE, M_NOWAIT);
 	if (adapter->mta == NULL) {
 		device_printf(dev, "Can not allocate multicast setup array\n");
 		error = ENOMEM;
 		goto err;
 	}
 
 	/* hw.ix defaults init */
 	ixgbe_set_advertise(adapter, ixgbe_advertise_speed);
 
 	/* Enable the optics for 82599 SFP+ fiber */
 	ixgbe_enable_tx_laser(hw);
 
 	/* Enable power to the phy. */
 	ixgbe_set_phy_power(hw, TRUE);
 
 	ixgbe_initialize_iov(adapter);
 
 	error = ixgbe_setup_interface(ctx);
 	if (error) {
 		device_printf(dev, "Interface setup failed: %d\n", error);
 		goto err;
 	}
 
 	ixgbe_if_update_admin_status(ctx);
 
 	/* Initialize statistics */
 	ixgbe_update_stats_counters(adapter);
 	ixgbe_add_hw_stats(adapter);
 
 	/* Check PCIE slot type/speed/width */
 	ixgbe_get_slot_info(adapter);
 
 	/*
 	 * Do time init and sysctl init here, but
 	 * only on the first port of a bypass adapter.
 	 */
 	ixgbe_bypass_init(adapter);
 
 	/* Set an initial dmac value */
 	adapter->dmac = 0;
 	/* Set initial advertised speeds (if applicable) */
 	adapter->advertise = ixgbe_get_advertise(adapter);
 
 	if (adapter->feat_cap & IXGBE_FEATURE_SRIOV)
 		ixgbe_define_iov_schemas(dev, &error);
 
 	/* Add sysctls */
 	ixgbe_add_device_sysctls(ctx);
 
 	return (0);
 err:
 	return (error);
 } /* ixgbe_if_attach_post */
 
 /************************************************************************
  * ixgbe_check_wol_support
  *
  *   Checks whether the adapter's ports are capable of
  *   Wake On LAN by reading the adapter's NVM.
  *
  *   Sets each port's hw->wol_enabled value depending
  *   on the value read here.
  ************************************************************************/
 static void
 ixgbe_check_wol_support(struct adapter *adapter)
 {
 	struct ixgbe_hw *hw = &adapter->hw;
 	u16             dev_caps = 0;
 
 	/* Find out WoL support for port */
 	adapter->wol_support = hw->wol_enabled = 0;
 	ixgbe_get_device_caps(hw, &dev_caps);
 	if ((dev_caps & IXGBE_DEVICE_CAPS_WOL_PORT0_1) ||
 	    ((dev_caps & IXGBE_DEVICE_CAPS_WOL_PORT0) &&
 	     hw->bus.func == 0))
 		adapter->wol_support = hw->wol_enabled = 1;
 
 	/* Save initial wake up filter configuration */
 	adapter->wufc = IXGBE_READ_REG(hw, IXGBE_WUFC);
 
 	return;
 } /* ixgbe_check_wol_support */
 
 /************************************************************************
  * ixgbe_setup_interface
  *
  *   Setup networking device structure and register an interface.
  ************************************************************************/
 static int
 ixgbe_setup_interface(if_ctx_t ctx)
 {
 	struct ifnet   *ifp = iflib_get_ifp(ctx);
 	struct adapter *adapter = iflib_get_softc(ctx);
 
 	INIT_DEBUGOUT("ixgbe_setup_interface: begin");
 
 	if_setbaudrate(ifp, IF_Gbps(10));
 
 	adapter->max_frame_size = ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 
 	adapter->phy_layer = ixgbe_get_supported_physical_layer(&adapter->hw);
 
 	ixgbe_add_media_types(ctx);
 
 	/* Autoselect media by default */
 	ifmedia_set(adapter->media, IFM_ETHER | IFM_AUTO);
 
 	return (0);
 } /* ixgbe_setup_interface */
 
 /************************************************************************
  * ixgbe_if_get_counter
  ************************************************************************/
 static uint64_t
 ixgbe_if_get_counter(if_ctx_t ctx, ift_counter cnt)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	if_t           ifp = iflib_get_ifp(ctx);
 
 	switch (cnt) {
 	case IFCOUNTER_IPACKETS:
 		return (adapter->ipackets);
 	case IFCOUNTER_OPACKETS:
 		return (adapter->opackets);
 	case IFCOUNTER_IBYTES:
 		return (adapter->ibytes);
 	case IFCOUNTER_OBYTES:
 		return (adapter->obytes);
 	case IFCOUNTER_IMCASTS:
 		return (adapter->imcasts);
 	case IFCOUNTER_OMCASTS:
 		return (adapter->omcasts);
 	case IFCOUNTER_COLLISIONS:
 		return (0);
 	case IFCOUNTER_IQDROPS:
 		return (adapter->iqdrops);
 	case IFCOUNTER_OQDROPS:
 		return (0);
 	case IFCOUNTER_IERRORS:
 		return (adapter->ierrors);
 	default:
 		return (if_get_counter_default(ifp, cnt));
 	}
 } /* ixgbe_if_get_counter */
 
 /************************************************************************
  * ixgbe_if_i2c_req
  ************************************************************************/
 static int
 ixgbe_if_i2c_req(if_ctx_t ctx, struct ifi2creq *req)
 {
 	struct adapter		*adapter = iflib_get_softc(ctx);
 	struct ixgbe_hw 	*hw = &adapter->hw;
 	int 			i;
 
 
 	if (hw->phy.ops.read_i2c_byte == NULL)
 		return (ENXIO);
 	for (i = 0; i < req->len; i++)
 		hw->phy.ops.read_i2c_byte(hw, req->offset + i,
 		    req->dev_addr, &req->data[i]);
 	return (0);
 } /* ixgbe_if_i2c_req */
 
 /************************************************************************
  * ixgbe_add_media_types
  ************************************************************************/
 static void
 ixgbe_add_media_types(if_ctx_t ctx)
 {
 	struct adapter  *adapter = iflib_get_softc(ctx);
 	struct ixgbe_hw *hw = &adapter->hw;
 	device_t        dev = iflib_get_dev(ctx);
 	u64             layer;
 
 	layer = adapter->phy_layer = ixgbe_get_supported_physical_layer(hw);
 
 	/* Media types with matching FreeBSD media defines */
 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T)
 		ifmedia_add(adapter->media, IFM_ETHER | IFM_10G_T, 0, NULL);
 	if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T)
 		ifmedia_add(adapter->media, IFM_ETHER | IFM_1000_T, 0, NULL);
 	if (layer & IXGBE_PHYSICAL_LAYER_100BASE_TX)
 		ifmedia_add(adapter->media, IFM_ETHER | IFM_100_TX, 0, NULL);
 	if (layer & IXGBE_PHYSICAL_LAYER_10BASE_T)
 		ifmedia_add(adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
 
 	if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU ||
 	    layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA)
 		ifmedia_add(adapter->media, IFM_ETHER | IFM_10G_TWINAX, 0,
 		    NULL);
 
 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR) {
 		ifmedia_add(adapter->media, IFM_ETHER | IFM_10G_LR, 0, NULL);
 		if (hw->phy.multispeed_fiber)
 			ifmedia_add(adapter->media, IFM_ETHER | IFM_1000_LX, 0,
 			    NULL);
 	}
 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
 		ifmedia_add(adapter->media, IFM_ETHER | IFM_10G_SR, 0, NULL);
 		if (hw->phy.multispeed_fiber)
 			ifmedia_add(adapter->media, IFM_ETHER | IFM_1000_SX, 0,
 			    NULL);
 	} else if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX)
 		ifmedia_add(adapter->media, IFM_ETHER | IFM_1000_SX, 0, NULL);
 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4)
 		ifmedia_add(adapter->media, IFM_ETHER | IFM_10G_CX4, 0, NULL);
 
 #ifdef IFM_ETH_XTYPE
 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR)
 		ifmedia_add(adapter->media, IFM_ETHER | IFM_10G_KR, 0, NULL);
 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4)
 		ifmedia_add( adapter->media, IFM_ETHER | IFM_10G_KX4, 0, NULL);
 	if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX)
 		ifmedia_add(adapter->media, IFM_ETHER | IFM_1000_KX, 0, NULL);
 	if (layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX)
 		ifmedia_add(adapter->media, IFM_ETHER | IFM_2500_KX, 0, NULL);
 #else
 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR) {
 		device_printf(dev, "Media supported: 10GbaseKR\n");
 		device_printf(dev, "10GbaseKR mapped to 10GbaseSR\n");
 		ifmedia_add(adapter->media, IFM_ETHER | IFM_10G_SR, 0, NULL);
 	}
 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4) {
 		device_printf(dev, "Media supported: 10GbaseKX4\n");
 		device_printf(dev, "10GbaseKX4 mapped to 10GbaseCX4\n");
 		ifmedia_add(adapter->media, IFM_ETHER | IFM_10G_CX4, 0, NULL);
 	}
 	if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX) {
 		device_printf(dev, "Media supported: 1000baseKX\n");
 		device_printf(dev, "1000baseKX mapped to 1000baseCX\n");
 		ifmedia_add(adapter->media, IFM_ETHER | IFM_1000_CX, 0, NULL);
 	}
 	if (layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX) {
 		device_printf(dev, "Media supported: 2500baseKX\n");
 		device_printf(dev, "2500baseKX mapped to 2500baseSX\n");
 		ifmedia_add(adapter->media, IFM_ETHER | IFM_2500_SX, 0, NULL);
 	}
 #endif
 	if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_BX)
 		device_printf(dev, "Media supported: 1000baseBX\n");
 
 	if (hw->device_id == IXGBE_DEV_ID_82598AT) {
 		ifmedia_add(adapter->media, IFM_ETHER | IFM_1000_T | IFM_FDX,
 		    0, NULL);
 		ifmedia_add(adapter->media, IFM_ETHER | IFM_1000_T, 0, NULL);
 	}
 
 	ifmedia_add(adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
 } /* ixgbe_add_media_types */
 
 /************************************************************************
  * ixgbe_is_sfp
  ************************************************************************/
 static inline bool
 ixgbe_is_sfp(struct ixgbe_hw *hw)
 {
 	switch (hw->mac.type) {
 	case ixgbe_mac_82598EB:
 		if (hw->phy.type == ixgbe_phy_nl)
 			return (TRUE);
 		return (FALSE);
 	case ixgbe_mac_82599EB:
 		switch (hw->mac.ops.get_media_type(hw)) {
 		case ixgbe_media_type_fiber:
 		case ixgbe_media_type_fiber_qsfp:
 			return (TRUE);
 		default:
 			return (FALSE);
 		}
 	case ixgbe_mac_X550EM_x:
 	case ixgbe_mac_X550EM_a:
 		if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_fiber)
 			return (TRUE);
 		return (FALSE);
 	default:
 		return (FALSE);
 	}
 } /* ixgbe_is_sfp */
 
 /************************************************************************
  * ixgbe_config_link
  ************************************************************************/
 static void
 ixgbe_config_link(if_ctx_t ctx)
 {
 	struct adapter  *adapter = iflib_get_softc(ctx);
 	struct ixgbe_hw *hw = &adapter->hw;
 	u32             autoneg, err = 0;
 	bool            sfp, negotiate;
 
 	sfp = ixgbe_is_sfp(hw);
 
 	if (sfp) {
 		adapter->task_requests |= IXGBE_REQUEST_TASK_MOD;
 		iflib_admin_intr_deferred(ctx);
 	} else {
 		if (hw->mac.ops.check_link)
 			err = ixgbe_check_link(hw, &adapter->link_speed,
 			    &adapter->link_up, FALSE);
 		if (err)
 			return;
 		autoneg = hw->phy.autoneg_advertised;
 		if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
 			err = hw->mac.ops.get_link_capabilities(hw, &autoneg,
 			    &negotiate);
 		if (err)
 			return;
 		if (hw->mac.ops.setup_link)
 			err = hw->mac.ops.setup_link(hw, autoneg,
 			    adapter->link_up);
 	}
 } /* ixgbe_config_link */
 
 /************************************************************************
  * ixgbe_update_stats_counters - Update board statistics counters.
  ************************************************************************/
 static void
 ixgbe_update_stats_counters(struct adapter *adapter)
 {
 	struct ixgbe_hw       *hw = &adapter->hw;
 	struct ixgbe_hw_stats *stats = &adapter->stats.pf;
 	u32                   missed_rx = 0, bprc, lxon, lxoff, total;
 	u64                   total_missed_rx = 0;
 
 	stats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
 	stats->illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
 	stats->errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
 	stats->mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
 	stats->mpc[0] += IXGBE_READ_REG(hw, IXGBE_MPC(0));
 
 	for (int i = 0; i < 16; i++) {
 		stats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
 		stats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
 		stats->qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
 	}
 	stats->mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
 	stats->mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
 	stats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
 
 	/* Hardware workaround, gprc counts missed packets */
 	stats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
 	stats->gprc -= missed_rx;
 
 	if (hw->mac.type != ixgbe_mac_82598EB) {
 		stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) +
 		    ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
 		stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) +
 		    ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
 		stats->tor += IXGBE_READ_REG(hw, IXGBE_TORL) +
 		    ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
 		stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
 		stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
 	} else {
 		stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
 		stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
 		/* 82598 only has a counter in the high register */
 		stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
 		stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
 		stats->tor += IXGBE_READ_REG(hw, IXGBE_TORH);
 	}
 
 	/*
 	 * Workaround: mprc hardware is incorrectly counting
 	 * broadcasts, so for now we subtract those.
 	 */
 	bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
 	stats->bprc += bprc;
 	stats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
 	if (hw->mac.type == ixgbe_mac_82598EB)
 		stats->mprc -= bprc;
 
 	stats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
 	stats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
 	stats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
 	stats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
 	stats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
 	stats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
 
 	lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
 	stats->lxontxc += lxon;
 	lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
 	stats->lxofftxc += lxoff;
 	total = lxon + lxoff;
 
 	stats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
 	stats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
 	stats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
 	stats->gptc -= total;
 	stats->mptc -= total;
 	stats->ptc64 -= total;
 	stats->gotc -= total * ETHER_MIN_LEN;
 
 	stats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
 	stats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
 	stats->roc += IXGBE_READ_REG(hw, IXGBE_ROC);
 	stats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
 	stats->mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
 	stats->mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
 	stats->mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
 	stats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
 	stats->tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
 	stats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
 	stats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
 	stats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
 	stats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
 	stats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
 	stats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
 	stats->xec += IXGBE_READ_REG(hw, IXGBE_XEC);
 	stats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
 	stats->fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
 	/* Only read FCOE on 82599 */
 	if (hw->mac.type != ixgbe_mac_82598EB) {
 		stats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
 		stats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
 		stats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
 		stats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
 		stats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
 	}
 
 	/* Fill out the OS statistics structure */
 	IXGBE_SET_IPACKETS(adapter, stats->gprc);
 	IXGBE_SET_OPACKETS(adapter, stats->gptc);
 	IXGBE_SET_IBYTES(adapter, stats->gorc);
 	IXGBE_SET_OBYTES(adapter, stats->gotc);
 	IXGBE_SET_IMCASTS(adapter, stats->mprc);
 	IXGBE_SET_OMCASTS(adapter, stats->mptc);
 	IXGBE_SET_COLLISIONS(adapter, 0);
 	IXGBE_SET_IQDROPS(adapter, total_missed_rx);
 	IXGBE_SET_IERRORS(adapter, stats->crcerrs + stats->rlec);
 } /* ixgbe_update_stats_counters */
 
 /************************************************************************
  * ixgbe_add_hw_stats
  *
  *   Add sysctl variables, one per statistic, to the system.
  ************************************************************************/
 static void
 ixgbe_add_hw_stats(struct adapter *adapter)
 {
 	device_t               dev = iflib_get_dev(adapter->ctx);
 	struct ix_rx_queue     *rx_que;
 	struct ix_tx_queue     *tx_que;
 	struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
 	struct sysctl_oid      *tree = device_get_sysctl_tree(dev);
 	struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
 	struct ixgbe_hw_stats  *stats = &adapter->stats.pf;
 	struct sysctl_oid      *stat_node, *queue_node;
 	struct sysctl_oid_list *stat_list, *queue_list;
 	int                    i;
 
 #define QUEUE_NAME_LEN 32
 	char                   namebuf[QUEUE_NAME_LEN];
 
 	/* Driver Statistics */
 	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
 	    CTLFLAG_RD, &adapter->dropped_pkts, "Driver dropped packets");
 	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
 	    CTLFLAG_RD, &adapter->watchdog_events, "Watchdog timeouts");
 	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
 	    CTLFLAG_RD, &adapter->link_irq, "Link MSI-X IRQ Handled");
 
 	for (i = 0, tx_que = adapter->tx_queues; i < adapter->num_tx_queues; i++, tx_que++) {
 		struct tx_ring *txr = &tx_que->txr;
 		snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
 		queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
 		    CTLFLAG_RD, NULL, "Queue Name");
 		queue_list = SYSCTL_CHILDREN(queue_node);
 
 		SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
 		    CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
 		    ixgbe_sysctl_tdh_handler, "IU", "Transmit Descriptor Head");
 		SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
 		    CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
 		    ixgbe_sysctl_tdt_handler, "IU", "Transmit Descriptor Tail");
 		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tso_tx",
 		    CTLFLAG_RD, &txr->tso_tx, "TSO");
 		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
 		    CTLFLAG_RD, &txr->total_packets,
 		    "Queue Packets Transmitted");
 	}
 
 	for (i = 0, rx_que = adapter->rx_queues; i < adapter->num_rx_queues; i++, rx_que++) {
 		struct rx_ring *rxr = &rx_que->rxr;
 		snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
 		queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
 		    CTLFLAG_RD, NULL, "Queue Name");
 		queue_list = SYSCTL_CHILDREN(queue_node);
 
 		SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate",
 		    CTLTYPE_UINT | CTLFLAG_RW, &adapter->rx_queues[i],
 		    sizeof(&adapter->rx_queues[i]),
 		    ixgbe_sysctl_interrupt_rate_handler, "IU",
 		    "Interrupt Rate");
 		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
 		    CTLFLAG_RD, &(adapter->rx_queues[i].irqs),
 		    "irqs on this queue");
 		SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
 		    CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
 		    ixgbe_sysctl_rdh_handler, "IU", "Receive Descriptor Head");
 		SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
 		    CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
 		    ixgbe_sysctl_rdt_handler, "IU", "Receive Descriptor Tail");
 		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
 		    CTLFLAG_RD, &rxr->rx_packets, "Queue Packets Received");
 		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
 		    CTLFLAG_RD, &rxr->rx_bytes, "Queue Bytes Received");
 		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_copies",
 		    CTLFLAG_RD, &rxr->rx_copies, "Copied RX Frames");
 		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_discarded",
 		    CTLFLAG_RD, &rxr->rx_discarded, "Discarded RX packets");
 	}
 
 	/* MAC stats get their own sub node */
 
 	stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
 	    CTLFLAG_RD, NULL, "MAC Statistics");
 	stat_list = SYSCTL_CHILDREN(stat_node);
 
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
 	    CTLFLAG_RD, &stats->crcerrs, "CRC Errors");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "ill_errs",
 	    CTLFLAG_RD, &stats->illerrc, "Illegal Byte Errors");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "byte_errs",
 	    CTLFLAG_RD, &stats->errbc, "Byte Errors");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "short_discards",
 	    CTLFLAG_RD, &stats->mspdc, "MAC Short Packets Discarded");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "local_faults",
 	    CTLFLAG_RD, &stats->mlfc, "MAC Local Faults");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "remote_faults",
 	    CTLFLAG_RD, &stats->mrfc, "MAC Remote Faults");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rec_len_errs",
 	    CTLFLAG_RD, &stats->rlec, "Receive Length Errors");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_missed_packets",
 	    CTLFLAG_RD, &stats->mpc[0], "RX Missed Packet Count");
 
 	/* Flow Control stats */
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_txd",
 	    CTLFLAG_RD, &stats->lxontxc, "Link XON Transmitted");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_recvd",
 	    CTLFLAG_RD, &stats->lxonrxc, "Link XON Received");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_txd",
 	    CTLFLAG_RD, &stats->lxofftxc, "Link XOFF Transmitted");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_recvd",
 	    CTLFLAG_RD, &stats->lxoffrxc, "Link XOFF Received");
 
 	/* Packet Reception Stats */
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_octets_rcvd",
 	    CTLFLAG_RD, &stats->tor, "Total Octets Received");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd",
 	    CTLFLAG_RD, &stats->gorc, "Good Octets Received");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_rcvd",
 	    CTLFLAG_RD, &stats->tpr, "Total Packets Received");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd",
 	    CTLFLAG_RD, &stats->gprc, "Good Packets Received");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd",
 	    CTLFLAG_RD, &stats->mprc, "Multicast Packets Received");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_rcvd",
 	    CTLFLAG_RD, &stats->bprc, "Broadcast Packets Received");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
 	    CTLFLAG_RD, &stats->prc64, "64 byte frames received ");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
 	    CTLFLAG_RD, &stats->prc127, "65-127 byte frames received");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
 	    CTLFLAG_RD, &stats->prc255, "128-255 byte frames received");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
 	    CTLFLAG_RD, &stats->prc511, "256-511 byte frames received");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
 	    CTLFLAG_RD, &stats->prc1023, "512-1023 byte frames received");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
 	    CTLFLAG_RD, &stats->prc1522, "1023-1522 byte frames received");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersized",
 	    CTLFLAG_RD, &stats->ruc, "Receive Undersized");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
 	    CTLFLAG_RD, &stats->rfc, "Fragmented Packets Received ");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversized",
 	    CTLFLAG_RD, &stats->roc, "Oversized Packets Received");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabberd",
 	    CTLFLAG_RD, &stats->rjc, "Received Jabber");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_rcvd",
 	    CTLFLAG_RD, &stats->mngprc, "Management Packets Received");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_drpd",
 	    CTLFLAG_RD, &stats->mngptc, "Management Packets Dropped");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "checksum_errs",
 	    CTLFLAG_RD, &stats->xec, "Checksum Errors");
 
 	/* Packet Transmission Stats */
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
 	    CTLFLAG_RD, &stats->gotc, "Good Octets Transmitted");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
 	    CTLFLAG_RD, &stats->tpt, "Total Packets Transmitted");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
 	    CTLFLAG_RD, &stats->gptc, "Good Packets Transmitted");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
 	    CTLFLAG_RD, &stats->bptc, "Broadcast Packets Transmitted");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
 	    CTLFLAG_RD, &stats->mptc, "Multicast Packets Transmitted");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_txd",
 	    CTLFLAG_RD, &stats->mngptc, "Management Packets Transmitted");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
 	    CTLFLAG_RD, &stats->ptc64, "64 byte frames transmitted ");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
 	    CTLFLAG_RD, &stats->ptc127, "65-127 byte frames transmitted");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
 	    CTLFLAG_RD, &stats->ptc255, "128-255 byte frames transmitted");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
 	    CTLFLAG_RD, &stats->ptc511, "256-511 byte frames transmitted");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
 	    CTLFLAG_RD, &stats->ptc1023, "512-1023 byte frames transmitted");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
 	    CTLFLAG_RD, &stats->ptc1522, "1024-1522 byte frames transmitted");
 } /* ixgbe_add_hw_stats */
 
 /************************************************************************
  * ixgbe_sysctl_tdh_handler - Transmit Descriptor Head handler function
  *
  *   Retrieves the TDH value from the hardware
  ************************************************************************/
 static int
 ixgbe_sysctl_tdh_handler(SYSCTL_HANDLER_ARGS)
 {
 	struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
 	int            error;
 	unsigned int   val;
 
 	if (!txr)
 		return (0);
 
 	val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDH(txr->me));
 	error = sysctl_handle_int(oidp, &val, 0, req);
 	if (error || !req->newptr)
 		return error;
 
 	return (0);
 } /* ixgbe_sysctl_tdh_handler */
 
 /************************************************************************
  * ixgbe_sysctl_tdt_handler - Transmit Descriptor Tail handler function
  *
  *   Retrieves the TDT value from the hardware
  ************************************************************************/
 static int
 ixgbe_sysctl_tdt_handler(SYSCTL_HANDLER_ARGS)
 {
 	struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
 	int            error;
 	unsigned int   val;
 
 	if (!txr)
 		return (0);
 
 	val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDT(txr->me));
 	error = sysctl_handle_int(oidp, &val, 0, req);
 	if (error || !req->newptr)
 		return error;
 
 	return (0);
 } /* ixgbe_sysctl_tdt_handler */
 
 /************************************************************************
  * ixgbe_sysctl_rdh_handler - Receive Descriptor Head handler function
  *
  *   Retrieves the RDH value from the hardware
  ************************************************************************/
 static int
 ixgbe_sysctl_rdh_handler(SYSCTL_HANDLER_ARGS)
 {
 	struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
 	int            error;
 	unsigned int   val;
 
 	if (!rxr)
 		return (0);
 
 	val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDH(rxr->me));
 	error = sysctl_handle_int(oidp, &val, 0, req);
 	if (error || !req->newptr)
 		return error;
 
 	return (0);
 } /* ixgbe_sysctl_rdh_handler */
 
 /************************************************************************
  * ixgbe_sysctl_rdt_handler - Receive Descriptor Tail handler function
  *
  *   Retrieves the RDT value from the hardware
  ************************************************************************/
 static int
 ixgbe_sysctl_rdt_handler(SYSCTL_HANDLER_ARGS)
 {
 	struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
 	int            error;
 	unsigned int   val;
 
 	if (!rxr)
 		return (0);
 
 	val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDT(rxr->me));
 	error = sysctl_handle_int(oidp, &val, 0, req);
 	if (error || !req->newptr)
 		return error;
 
 	return (0);
 } /* ixgbe_sysctl_rdt_handler */
 
 /************************************************************************
  * ixgbe_if_vlan_register
  *
  *   Run via vlan config EVENT, it enables us to use the
  *   HW Filter table since we can get the vlan id. This
  *   just creates the entry in the soft version of the
  *   VFTA, init will repopulate the real table.
  ************************************************************************/
 static void
 ixgbe_if_vlan_register(if_ctx_t ctx, u16 vtag)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	u16            index, bit;
 
 	index = (vtag >> 5) & 0x7F;
 	bit = vtag & 0x1F;
 	adapter->shadow_vfta[index] |= (1 << bit);
 	++adapter->num_vlans;
 	ixgbe_setup_vlan_hw_support(ctx);
 } /* ixgbe_if_vlan_register */
 
 /************************************************************************
  * ixgbe_if_vlan_unregister
  *
  *   Run via vlan unconfig EVENT, remove our entry in the soft vfta.
  ************************************************************************/
 static void
 ixgbe_if_vlan_unregister(if_ctx_t ctx, u16 vtag)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	u16            index, bit;
 
 	index = (vtag >> 5) & 0x7F;
 	bit = vtag & 0x1F;
 	adapter->shadow_vfta[index] &= ~(1 << bit);
 	--adapter->num_vlans;
 	/* Re-init to load the changes */
 	ixgbe_setup_vlan_hw_support(ctx);
 } /* ixgbe_if_vlan_unregister */
 
 /************************************************************************
  * ixgbe_setup_vlan_hw_support
  ************************************************************************/
 static void
 ixgbe_setup_vlan_hw_support(if_ctx_t ctx)
 {
 	struct ifnet	*ifp = iflib_get_ifp(ctx);
 	struct adapter  *adapter = iflib_get_softc(ctx);
 	struct ixgbe_hw *hw = &adapter->hw;
 	struct rx_ring  *rxr;
 	int             i;
 	u32             ctrl;
 
 
 	/*
 	 * We get here thru init_locked, meaning
 	 * a soft reset, this has already cleared
 	 * the VFTA and other state, so if there
 	 * have been no vlan's registered do nothing.
 	 */
 	if (adapter->num_vlans == 0)
 		return;
 
 	/* Setup the queues for vlans */
 	if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) {
 		for (i = 0; i < adapter->num_rx_queues; i++) {
 			rxr = &adapter->rx_queues[i].rxr;
 			/* On 82599 the VLAN enable is per/queue in RXDCTL */
 			if (hw->mac.type != ixgbe_mac_82598EB) {
 				ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxr->me));
 				ctrl |= IXGBE_RXDCTL_VME;
 				IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxr->me), ctrl);
 			}
 			rxr->vtag_strip = TRUE;
 		}
 	}
 
 	if ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0)
 		return;
 	/*
 	 * A soft reset zero's out the VFTA, so
 	 * we need to repopulate it now.
 	 */
 	for (i = 0; i < IXGBE_VFTA_SIZE; i++)
 		if (adapter->shadow_vfta[i] != 0)
 			IXGBE_WRITE_REG(hw, IXGBE_VFTA(i),
 			    adapter->shadow_vfta[i]);
 
 	ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
 	/* Enable the Filter Table if enabled */
 	if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
 		ctrl &= ~IXGBE_VLNCTRL_CFIEN;
 		ctrl |= IXGBE_VLNCTRL_VFE;
 	}
 	if (hw->mac.type == ixgbe_mac_82598EB)
 		ctrl |= IXGBE_VLNCTRL_VME;
 	IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
 } /* ixgbe_setup_vlan_hw_support */
 
 /************************************************************************
  * ixgbe_get_slot_info
  *
  *   Get the width and transaction speed of
  *   the slot this adapter is plugged into.
  ************************************************************************/
 static void
 ixgbe_get_slot_info(struct adapter *adapter)
 {
 	device_t        dev = iflib_get_dev(adapter->ctx);
 	struct ixgbe_hw *hw = &adapter->hw;
 	int             bus_info_valid = TRUE;
 	u32             offset;
 	u16             link;
 
 	/* Some devices are behind an internal bridge */
 	switch (hw->device_id) {
 	case IXGBE_DEV_ID_82599_SFP_SF_QP:
 	case IXGBE_DEV_ID_82599_QSFP_SF_QP:
 		goto get_parent_info;
 	default:
 		break;
 	}
 
 	ixgbe_get_bus_info(hw);
 
 	/*
 	 * Some devices don't use PCI-E, but there is no need
 	 * to display "Unknown" for bus speed and width.
 	 */
 	switch (hw->mac.type) {
 	case ixgbe_mac_X550EM_x:
 	case ixgbe_mac_X550EM_a:
 		return;
 	default:
 		goto display;
 	}
 
 get_parent_info:
 	/*
 	 * For the Quad port adapter we need to parse back
 	 * up the PCI tree to find the speed of the expansion
 	 * slot into which this adapter is plugged. A bit more work.
 	 */
 	dev = device_get_parent(device_get_parent(dev));
 #ifdef IXGBE_DEBUG
 	device_printf(dev, "parent pcib = %x,%x,%x\n", pci_get_bus(dev),
 	    pci_get_slot(dev), pci_get_function(dev));
 #endif
 	dev = device_get_parent(device_get_parent(dev));
 #ifdef IXGBE_DEBUG
 	device_printf(dev, "slot pcib = %x,%x,%x\n", pci_get_bus(dev),
 	    pci_get_slot(dev), pci_get_function(dev));
 #endif
 	/* Now get the PCI Express Capabilities offset */
 	if (pci_find_cap(dev, PCIY_EXPRESS, &offset)) {
 		/*
 		 * Hmm...can't get PCI-Express capabilities.
 		 * Falling back to default method.
 		 */
 		bus_info_valid = FALSE;
 		ixgbe_get_bus_info(hw);
 		goto display;
 	}
 	/* ...and read the Link Status Register */
 	link = pci_read_config(dev, offset + PCIER_LINK_STA, 2);
 	ixgbe_set_pci_config_data_generic(hw, link);
 
 display:
 	device_printf(dev, "PCI Express Bus: Speed %s %s\n",
 	    ((hw->bus.speed == ixgbe_bus_speed_8000)    ? "8.0GT/s"  :
 	     (hw->bus.speed == ixgbe_bus_speed_5000)    ? "5.0GT/s"  :
 	     (hw->bus.speed == ixgbe_bus_speed_2500)    ? "2.5GT/s"  :
 	     "Unknown"),
 	    ((hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
 	     (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
 	     (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
 	     "Unknown"));
 
 	if (bus_info_valid) {
 		if ((hw->device_id != IXGBE_DEV_ID_82599_SFP_SF_QP) &&
 		    ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
 		    (hw->bus.speed == ixgbe_bus_speed_2500))) {
 			device_printf(dev, "PCI-Express bandwidth available for this card\n     is not sufficient for optimal performance.\n");
 			device_printf(dev, "For optimal performance a x8 PCIE, or x4 PCIE Gen2 slot is required.\n");
 		}
 		if ((hw->device_id == IXGBE_DEV_ID_82599_SFP_SF_QP) &&
 		    ((hw->bus.width <= ixgbe_bus_width_pcie_x8) &&
 		    (hw->bus.speed < ixgbe_bus_speed_8000))) {
 			device_printf(dev, "PCI-Express bandwidth available for this card\n     is not sufficient for optimal performance.\n");
 			device_printf(dev, "For optimal performance a x8 PCIE Gen3 slot is required.\n");
 		}
 	} else
 		device_printf(dev, "Unable to determine slot speed/width. The speed/width reported are that of the internal switch.\n");
 
 	return;
 } /* ixgbe_get_slot_info */
 
 /************************************************************************
  * ixgbe_if_msix_intr_assign
  *
  *   Setup MSI-X Interrupt resources and handlers
  ************************************************************************/
 static int
 ixgbe_if_msix_intr_assign(if_ctx_t ctx, int msix)
 {
 	struct adapter     *adapter = iflib_get_softc(ctx);
 	struct ix_rx_queue *rx_que = adapter->rx_queues;
 	struct ix_tx_queue *tx_que;
 	int                error, rid, vector = 0;
 	int                cpu_id = 0;
 	char               buf[16];
 
 	/* Admin Que is vector 0*/
 	rid = vector + 1;
 	for (int i = 0; i < adapter->num_rx_queues; i++, vector++, rx_que++) {
 		rid = vector + 1;
 
 		snprintf(buf, sizeof(buf), "rxq%d", i);
 		error = iflib_irq_alloc_generic(ctx, &rx_que->que_irq, rid,
 		    IFLIB_INTR_RX, ixgbe_msix_que, rx_que, rx_que->rxr.me, buf);
 
 		if (error) {
 			device_printf(iflib_get_dev(ctx),
 			    "Failed to allocate que int %d err: %d", i, error);
 			adapter->num_rx_queues = i + 1;
 			goto fail;
 		}
 
 		rx_que->msix = vector;
 		adapter->active_queues |= (u64)(1 << rx_que->msix);
 		if (adapter->feat_en & IXGBE_FEATURE_RSS) {
 			/*
 			 * The queue ID is used as the RSS layer bucket ID.
 			 * We look up the queue ID -> RSS CPU ID and select
 			 * that.
 			 */
 			cpu_id = rss_getcpu(i % rss_getnumbuckets());
 		} else {
 			/*
 			 * Bind the MSI-X vector, and thus the
 			 * rings to the corresponding cpu.
 			 *
 			 * This just happens to match the default RSS
 			 * round-robin bucket -> queue -> CPU allocation.
 			 */
 			if (adapter->num_rx_queues > 1)
 				cpu_id = i;
 		}
 
 	}
 	for (int i = 0; i < adapter->num_tx_queues; i++) {
 		snprintf(buf, sizeof(buf), "txq%d", i);
 		tx_que = &adapter->tx_queues[i];
 		tx_que->msix = i % adapter->num_rx_queues;
 		iflib_softirq_alloc_generic(ctx,
 		    &adapter->rx_queues[tx_que->msix].que_irq,
 		    IFLIB_INTR_TX, tx_que, tx_que->txr.me, buf);
 	}
 	rid = vector + 1;
 	error = iflib_irq_alloc_generic(ctx, &adapter->irq, rid,
 	    IFLIB_INTR_ADMIN, ixgbe_msix_link, adapter, 0, "aq");
 	if (error) {
 		device_printf(iflib_get_dev(ctx),
 		    "Failed to register admin handler");
 		return (error);
 	}
 
 	adapter->vector = vector;
 
 	return (0);
 fail:
 	iflib_irq_free(ctx, &adapter->irq);
 	rx_que = adapter->rx_queues;
 	for (int i = 0; i < adapter->num_rx_queues; i++, rx_que++)
 		iflib_irq_free(ctx, &rx_que->que_irq);
 
 	return (error);
 } /* ixgbe_if_msix_intr_assign */
 
 /*********************************************************************
  * ixgbe_msix_que - MSI-X Queue Interrupt Service routine
  **********************************************************************/
 static int
 ixgbe_msix_que(void *arg)
 {
 	struct ix_rx_queue *que = arg;
 	struct adapter     *adapter = que->adapter;
 	struct ifnet       *ifp = iflib_get_ifp(que->adapter->ctx);
 
 	/* Protect against spurious interrupts */
 	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 		return (FILTER_HANDLED);
 
 	ixgbe_disable_queue(adapter, que->msix);
 	++que->irqs;
 
 	return (FILTER_SCHEDULE_THREAD);
 } /* ixgbe_msix_que */
 
 /************************************************************************
  * ixgbe_media_status - Media Ioctl callback
  *
  *   Called whenever the user queries the status of
  *   the interface using ifconfig.
  ************************************************************************/
 static void
 ixgbe_if_media_status(if_ctx_t ctx, struct ifmediareq * ifmr)
 {
 	struct adapter  *adapter = iflib_get_softc(ctx);
 	struct ixgbe_hw *hw = &adapter->hw;
 	int             layer;
 
 	INIT_DEBUGOUT("ixgbe_if_media_status: begin");
 
 	ifmr->ifm_status = IFM_AVALID;
 	ifmr->ifm_active = IFM_ETHER;
 
 	if (!adapter->link_active)
 		return;
 
 	ifmr->ifm_status |= IFM_ACTIVE;
 	layer = adapter->phy_layer;
 
 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T ||
 	    layer & IXGBE_PHYSICAL_LAYER_1000BASE_T ||
 	    layer & IXGBE_PHYSICAL_LAYER_100BASE_TX ||
 	    layer & IXGBE_PHYSICAL_LAYER_10BASE_T)
 		switch (adapter->link_speed) {
 		case IXGBE_LINK_SPEED_10GB_FULL:
 			ifmr->ifm_active |= IFM_10G_T | IFM_FDX;
 			break;
 		case IXGBE_LINK_SPEED_1GB_FULL:
 			ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
 			break;
 		case IXGBE_LINK_SPEED_100_FULL:
 			ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
 			break;
 		case IXGBE_LINK_SPEED_10_FULL:
 			ifmr->ifm_active |= IFM_10_T | IFM_FDX;
 			break;
 		}
 	if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU ||
 	    layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA)
 		switch (adapter->link_speed) {
 		case IXGBE_LINK_SPEED_10GB_FULL:
 			ifmr->ifm_active |= IFM_10G_TWINAX | IFM_FDX;
 			break;
 		}
 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR)
 		switch (adapter->link_speed) {
 		case IXGBE_LINK_SPEED_10GB_FULL:
 			ifmr->ifm_active |= IFM_10G_LR | IFM_FDX;
 			break;
 		case IXGBE_LINK_SPEED_1GB_FULL:
 			ifmr->ifm_active |= IFM_1000_LX | IFM_FDX;
 			break;
 		}
 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LRM)
 		switch (adapter->link_speed) {
 		case IXGBE_LINK_SPEED_10GB_FULL:
 			ifmr->ifm_active |= IFM_10G_LRM | IFM_FDX;
 			break;
 		case IXGBE_LINK_SPEED_1GB_FULL:
 			ifmr->ifm_active |= IFM_1000_LX | IFM_FDX;
 			break;
 		}
 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR ||
 	    layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX)
 		switch (adapter->link_speed) {
 		case IXGBE_LINK_SPEED_10GB_FULL:
 			ifmr->ifm_active |= IFM_10G_SR | IFM_FDX;
 			break;
 		case IXGBE_LINK_SPEED_1GB_FULL:
 			ifmr->ifm_active |= IFM_1000_SX | IFM_FDX;
 			break;
 		}
 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4)
 		switch (adapter->link_speed) {
 		case IXGBE_LINK_SPEED_10GB_FULL:
 			ifmr->ifm_active |= IFM_10G_CX4 | IFM_FDX;
 			break;
 		}
 	/*
 	 * XXX: These need to use the proper media types once
 	 * they're added.
 	 */
 #ifndef IFM_ETH_XTYPE
 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR)
 		switch (adapter->link_speed) {
 		case IXGBE_LINK_SPEED_10GB_FULL:
 			ifmr->ifm_active |= IFM_10G_SR | IFM_FDX;
 			break;
 		case IXGBE_LINK_SPEED_2_5GB_FULL:
 			ifmr->ifm_active |= IFM_2500_SX | IFM_FDX;
 			break;
 		case IXGBE_LINK_SPEED_1GB_FULL:
 			ifmr->ifm_active |= IFM_1000_CX | IFM_FDX;
 			break;
 		}
 	else if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4 ||
 	    layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX ||
 	    layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX)
 		switch (adapter->link_speed) {
 		case IXGBE_LINK_SPEED_10GB_FULL:
 			ifmr->ifm_active |= IFM_10G_CX4 | IFM_FDX;
 			break;
 		case IXGBE_LINK_SPEED_2_5GB_FULL:
 			ifmr->ifm_active |= IFM_2500_SX | IFM_FDX;
 			break;
 		case IXGBE_LINK_SPEED_1GB_FULL:
 			ifmr->ifm_active |= IFM_1000_CX | IFM_FDX;
 			break;
 		}
 #else
 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR)
 		switch (adapter->link_speed) {
 		case IXGBE_LINK_SPEED_10GB_FULL:
 			ifmr->ifm_active |= IFM_10G_KR | IFM_FDX;
 			break;
 		case IXGBE_LINK_SPEED_2_5GB_FULL:
 			ifmr->ifm_active |= IFM_2500_KX | IFM_FDX;
 			break;
 		case IXGBE_LINK_SPEED_1GB_FULL:
 			ifmr->ifm_active |= IFM_1000_KX | IFM_FDX;
 			break;
 		}
 	else if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4 ||
 	    layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX ||
 	    layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX)
 		switch (adapter->link_speed) {
 		case IXGBE_LINK_SPEED_10GB_FULL:
 			ifmr->ifm_active |= IFM_10G_KX4 | IFM_FDX;
 			break;
 		case IXGBE_LINK_SPEED_2_5GB_FULL:
 			ifmr->ifm_active |= IFM_2500_KX | IFM_FDX;
 			break;
 		case IXGBE_LINK_SPEED_1GB_FULL:
 			ifmr->ifm_active |= IFM_1000_KX | IFM_FDX;
 			break;
 		}
 #endif
 
 	/* If nothing is recognized... */
 	if (IFM_SUBTYPE(ifmr->ifm_active) == 0)
 		ifmr->ifm_active |= IFM_UNKNOWN;
 
 	/* Display current flow control setting used on link */
 	if (hw->fc.current_mode == ixgbe_fc_rx_pause ||
 	    hw->fc.current_mode == ixgbe_fc_full)
 		ifmr->ifm_active |= IFM_ETH_RXPAUSE;
 	if (hw->fc.current_mode == ixgbe_fc_tx_pause ||
 	    hw->fc.current_mode == ixgbe_fc_full)
 		ifmr->ifm_active |= IFM_ETH_TXPAUSE;
 } /* ixgbe_media_status */
 
 /************************************************************************
  * ixgbe_media_change - Media Ioctl callback
  *
  *   Called when the user changes speed/duplex using
  *   media/mediopt option with ifconfig.
  ************************************************************************/
 static int
 ixgbe_if_media_change(if_ctx_t ctx)
 {
 	struct adapter   *adapter = iflib_get_softc(ctx);
 	struct ifmedia   *ifm = iflib_get_media(ctx);
 	struct ixgbe_hw  *hw = &adapter->hw;
 	ixgbe_link_speed speed = 0;
 
 	INIT_DEBUGOUT("ixgbe_if_media_change: begin");
 
 	if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
 		return (EINVAL);
 
 	if (hw->phy.media_type == ixgbe_media_type_backplane)
 		return (EPERM);
 
 	/*
 	 * We don't actually need to check against the supported
 	 * media types of the adapter; ifmedia will take care of
 	 * that for us.
 	 */
 	switch (IFM_SUBTYPE(ifm->ifm_media)) {
 	case IFM_AUTO:
 	case IFM_10G_T:
 		speed |= IXGBE_LINK_SPEED_100_FULL;
 		speed |= IXGBE_LINK_SPEED_1GB_FULL;
 		speed |= IXGBE_LINK_SPEED_10GB_FULL;
 		break;
 	case IFM_10G_LRM:
 	case IFM_10G_LR:
 #ifndef IFM_ETH_XTYPE
 	case IFM_10G_SR: /* KR, too */
 	case IFM_10G_CX4: /* KX4 */
 #else
 	case IFM_10G_KR:
 	case IFM_10G_KX4:
 #endif
 		speed |= IXGBE_LINK_SPEED_1GB_FULL;
 		speed |= IXGBE_LINK_SPEED_10GB_FULL;
 		break;
 #ifndef IFM_ETH_XTYPE
 	case IFM_1000_CX: /* KX */
 #else
 	case IFM_1000_KX:
 #endif
 	case IFM_1000_LX:
 	case IFM_1000_SX:
 		speed |= IXGBE_LINK_SPEED_1GB_FULL;
 		break;
 	case IFM_1000_T:
 		speed |= IXGBE_LINK_SPEED_100_FULL;
 		speed |= IXGBE_LINK_SPEED_1GB_FULL;
 		break;
 	case IFM_10G_TWINAX:
 		speed |= IXGBE_LINK_SPEED_10GB_FULL;
 		break;
 	case IFM_100_TX:
 		speed |= IXGBE_LINK_SPEED_100_FULL;
 		break;
 	case IFM_10_T:
 		speed |= IXGBE_LINK_SPEED_10_FULL;
 		break;
 	default:
 		goto invalid;
 	}
 
 	hw->mac.autotry_restart = TRUE;
 	hw->mac.ops.setup_link(hw, speed, TRUE);
 	adapter->advertise =
 	    ((speed & IXGBE_LINK_SPEED_10GB_FULL) ? 4 : 0) |
 	    ((speed & IXGBE_LINK_SPEED_1GB_FULL)  ? 2 : 0) |
 	    ((speed & IXGBE_LINK_SPEED_100_FULL)  ? 1 : 0) |
 	    ((speed & IXGBE_LINK_SPEED_10_FULL)   ? 8 : 0);
 
 	return (0);
 
 invalid:
 	device_printf(iflib_get_dev(ctx), "Invalid media type!\n");
 
 	return (EINVAL);
 } /* ixgbe_if_media_change */
 
 /************************************************************************
  * ixgbe_set_promisc
  ************************************************************************/
 static int
 ixgbe_if_promisc_set(if_ctx_t ctx, int flags)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	struct ifnet   *ifp = iflib_get_ifp(ctx);
 	u32            rctl;
 	int            mcnt = 0;
 
 	rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
 	rctl &= (~IXGBE_FCTRL_UPE);
 	if (ifp->if_flags & IFF_ALLMULTI)
 		mcnt = MAX_NUM_MULTICAST_ADDRESSES;
 	else {
 		mcnt = if_multiaddr_count(ifp, MAX_NUM_MULTICAST_ADDRESSES);
 	}
 	if (mcnt < MAX_NUM_MULTICAST_ADDRESSES)
 		rctl &= (~IXGBE_FCTRL_MPE);
 	IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, rctl);
 
 	if (ifp->if_flags & IFF_PROMISC) {
 		rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
 		IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, rctl);
 	} else if (ifp->if_flags & IFF_ALLMULTI) {
 		rctl |= IXGBE_FCTRL_MPE;
 		rctl &= ~IXGBE_FCTRL_UPE;
 		IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, rctl);
 	}
 	return (0);
 } /* ixgbe_if_promisc_set */
 
 /************************************************************************
  * ixgbe_msix_link - Link status change ISR (MSI/MSI-X)
  ************************************************************************/
 static int
 ixgbe_msix_link(void *arg)
 {
 	struct adapter  *adapter = arg;
 	struct ixgbe_hw *hw = &adapter->hw;
 	u32             eicr, eicr_mask;
 	s32             retval;
 
 	++adapter->link_irq;
 
 	/* Pause other interrupts */
 	IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_OTHER);
 
 	/* First get the cause */
 	eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
 	/* Be sure the queue bits are not cleared */
 	eicr &= ~IXGBE_EICR_RTX_QUEUE;
 	/* Clear interrupt with write */
 	IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr);
 
 	/* Link status change */
 	if (eicr & IXGBE_EICR_LSC) {
 		IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_LSC);
 		adapter->task_requests |= IXGBE_REQUEST_TASK_LSC;
 	}
 
 	if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
 		if ((adapter->feat_en & IXGBE_FEATURE_FDIR) &&
 		    (eicr & IXGBE_EICR_FLOW_DIR)) {
 			/* This is probably overkill :) */
 			if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1))
 				return (FILTER_HANDLED);
 			/* Disable the interrupt */
 			IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_FLOW_DIR);
 			adapter->task_requests |= IXGBE_REQUEST_TASK_FDIR;
 		} else
 			if (eicr & IXGBE_EICR_ECC) {
 				device_printf(iflib_get_dev(adapter->ctx),
 				   "\nCRITICAL: ECC ERROR!! Please Reboot!!\n");
 				IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
 			}
 
 		/* Check for over temp condition */
 		if (adapter->feat_en & IXGBE_FEATURE_TEMP_SENSOR) {
 			switch (adapter->hw.mac.type) {
 			case ixgbe_mac_X550EM_a:
 				if (!(eicr & IXGBE_EICR_GPI_SDP0_X550EM_a))
 					break;
 				IXGBE_WRITE_REG(hw, IXGBE_EIMC,
 				    IXGBE_EICR_GPI_SDP0_X550EM_a);
 				IXGBE_WRITE_REG(hw, IXGBE_EICR,
 				    IXGBE_EICR_GPI_SDP0_X550EM_a);
 				retval = hw->phy.ops.check_overtemp(hw);
 				if (retval != IXGBE_ERR_OVERTEMP)
 					break;
 				device_printf(iflib_get_dev(adapter->ctx),
 				    "\nCRITICAL: OVER TEMP!! PHY IS SHUT DOWN!!\n");
 				device_printf(iflib_get_dev(adapter->ctx),
 				    "System shutdown required!\n");
 				break;
 			default:
 				if (!(eicr & IXGBE_EICR_TS))
 					break;
 				retval = hw->phy.ops.check_overtemp(hw);
 				if (retval != IXGBE_ERR_OVERTEMP)
 					break;
 				device_printf(iflib_get_dev(adapter->ctx),
 				    "\nCRITICAL: OVER TEMP!! PHY IS SHUT DOWN!!\n");
 				device_printf(iflib_get_dev(adapter->ctx),
 				    "System shutdown required!\n");
 				IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_TS);
 				break;
 			}
 		}
 
 		/* Check for VF message */
 		if ((adapter->feat_en & IXGBE_FEATURE_SRIOV) &&
 		    (eicr & IXGBE_EICR_MAILBOX))
 			adapter->task_requests |= IXGBE_REQUEST_TASK_MBX;
 	}
 
 	if (ixgbe_is_sfp(hw)) {
 		/* Pluggable optics-related interrupt */
 		if (hw->mac.type >= ixgbe_mac_X540)
 			eicr_mask = IXGBE_EICR_GPI_SDP0_X540;
 		else
 			eicr_mask = IXGBE_EICR_GPI_SDP2_BY_MAC(hw);
 
 		if (eicr & eicr_mask) {
 			IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr_mask);
 			adapter->task_requests |= IXGBE_REQUEST_TASK_MOD;
 		}
 
 		if ((hw->mac.type == ixgbe_mac_82599EB) &&
 		    (eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw))) {
 			IXGBE_WRITE_REG(hw, IXGBE_EICR,
 			    IXGBE_EICR_GPI_SDP1_BY_MAC(hw));
 			adapter->task_requests |= IXGBE_REQUEST_TASK_MSF;
 		}
 	}
 
 	/* Check for fan failure */
 	if (adapter->feat_en & IXGBE_FEATURE_FAN_FAIL) {
 		ixgbe_check_fan_failure(adapter, eicr, TRUE);
 		IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1_BY_MAC(hw));
 	}
 
 	/* External PHY interrupt */
 	if ((hw->phy.type == ixgbe_phy_x550em_ext_t) &&
 	    (eicr & IXGBE_EICR_GPI_SDP0_X540)) {
 		IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0_X540);
 		adapter->task_requests |= IXGBE_REQUEST_TASK_PHY;
 	}
 
 	return (adapter->task_requests != 0) ? FILTER_SCHEDULE_THREAD : FILTER_HANDLED;
 } /* ixgbe_msix_link */
 
 /************************************************************************
  * ixgbe_sysctl_interrupt_rate_handler
  ************************************************************************/
 static int
 ixgbe_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS)
 {
 	struct ix_rx_queue *que = ((struct ix_rx_queue *)oidp->oid_arg1);
 	int                error;
 	unsigned int       reg, usec, rate;
 
 	reg = IXGBE_READ_REG(&que->adapter->hw, IXGBE_EITR(que->msix));
 	usec = ((reg & 0x0FF8) >> 3);
 	if (usec > 0)
 		rate = 500000 / usec;
 	else
 		rate = 0;
 	error = sysctl_handle_int(oidp, &rate, 0, req);
 	if (error || !req->newptr)
 		return error;
 	reg &= ~0xfff; /* default, no limitation */
 	ixgbe_max_interrupt_rate = 0;
 	if (rate > 0 && rate < 500000) {
 		if (rate < 1000)
 			rate = 1000;
 		ixgbe_max_interrupt_rate = rate;
 		reg |= ((4000000/rate) & 0xff8);
 	}
 	IXGBE_WRITE_REG(&que->adapter->hw, IXGBE_EITR(que->msix), reg);
 
 	return (0);
 } /* ixgbe_sysctl_interrupt_rate_handler */
 
 /************************************************************************
  * ixgbe_add_device_sysctls
  ************************************************************************/
 static void
 ixgbe_add_device_sysctls(if_ctx_t ctx)
 {
 	struct adapter         *adapter = iflib_get_softc(ctx);
 	device_t               dev = iflib_get_dev(ctx);
 	struct ixgbe_hw        *hw = &adapter->hw;
 	struct sysctl_oid_list *child;
 	struct sysctl_ctx_list *ctx_list;
 
 	ctx_list = device_get_sysctl_ctx(dev);
 	child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev));
 
 	/* Sysctls for all devices */
 	SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "fc",
 	    CTLTYPE_INT | CTLFLAG_RW, adapter, 0, ixgbe_sysctl_flowcntl, "I",
 	    IXGBE_SYSCTL_DESC_SET_FC);
 
 	SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "advertise_speed",
 	    CTLTYPE_INT | CTLFLAG_RW, adapter, 0, ixgbe_sysctl_advertise, "I",
 	    IXGBE_SYSCTL_DESC_ADV_SPEED);
 
 #ifdef IXGBE_DEBUG
 	/* testing sysctls (for all devices) */
 	SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "power_state",
 	    CTLTYPE_INT | CTLFLAG_RW, adapter, 0, ixgbe_sysctl_power_state,
 	    "I", "PCI Power State");
 
 	SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "print_rss_config",
 	    CTLTYPE_STRING | CTLFLAG_RD, adapter, 0,
 	    ixgbe_sysctl_print_rss_config, "A", "Prints RSS Configuration");
 #endif
 	/* for X550 series devices */
 	if (hw->mac.type >= ixgbe_mac_X550)
 		SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "dmac",
 		    CTLTYPE_U16 | CTLFLAG_RW, adapter, 0, ixgbe_sysctl_dmac,
 		    "I", "DMA Coalesce");
 
 	/* for WoL-capable devices */
 	if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) {
 		SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "wol_enable",
 		    CTLTYPE_INT | CTLFLAG_RW, adapter, 0,
 		    ixgbe_sysctl_wol_enable, "I", "Enable/Disable Wake on LAN");
 
 		SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "wufc",
 		    CTLTYPE_U32 | CTLFLAG_RW, adapter, 0, ixgbe_sysctl_wufc,
 		    "I", "Enable/Disable Wake Up Filters");
 	}
 
 	/* for X552/X557-AT devices */
 	if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) {
 		struct sysctl_oid *phy_node;
 		struct sysctl_oid_list *phy_list;
 
 		phy_node = SYSCTL_ADD_NODE(ctx_list, child, OID_AUTO, "phy",
 		    CTLFLAG_RD, NULL, "External PHY sysctls");
 		phy_list = SYSCTL_CHILDREN(phy_node);
 
 		SYSCTL_ADD_PROC(ctx_list, phy_list, OID_AUTO, "temp",
 		    CTLTYPE_U16 | CTLFLAG_RD, adapter, 0, ixgbe_sysctl_phy_temp,
 		    "I", "Current External PHY Temperature (Celsius)");
 
 		SYSCTL_ADD_PROC(ctx_list, phy_list, OID_AUTO,
 		    "overtemp_occurred", CTLTYPE_U16 | CTLFLAG_RD, adapter, 0,
 		    ixgbe_sysctl_phy_overtemp_occurred, "I",
 		    "External PHY High Temperature Event Occurred");
 	}
 
 	if (adapter->feat_cap & IXGBE_FEATURE_EEE) {
 		SYSCTL_ADD_PROC(ctx_list, child, OID_AUTO, "eee_state",
 		    CTLTYPE_INT | CTLFLAG_RW, adapter, 0,
 		    ixgbe_sysctl_eee_state, "I", "EEE Power Save State");
 	}
 } /* ixgbe_add_device_sysctls */
 
 /************************************************************************
  * ixgbe_allocate_pci_resources
  ************************************************************************/
 static int
 ixgbe_allocate_pci_resources(if_ctx_t ctx)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	device_t        dev = iflib_get_dev(ctx);
 	int             rid;
 
 	rid = PCIR_BAR(0);
 	adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 	    RF_ACTIVE);
 
 	if (!(adapter->pci_mem)) {
 		device_printf(dev, "Unable to allocate bus resource: memory\n");
 		return (ENXIO);
 	}
 
 	/* Save bus_space values for READ/WRITE_REG macros */
 	adapter->osdep.mem_bus_space_tag = rman_get_bustag(adapter->pci_mem);
 	adapter->osdep.mem_bus_space_handle =
 	    rman_get_bushandle(adapter->pci_mem);
 	/* Set hw values for shared code */
 	adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle;
 
 	return (0);
 } /* ixgbe_allocate_pci_resources */
 
 /************************************************************************
  * ixgbe_detach - Device removal routine
  *
  *   Called when the driver is being removed.
  *   Stops the adapter and deallocates all the resources
  *   that were allocated for driver operation.
  *
  *   return 0 on success, positive on failure
  ************************************************************************/
 static int
 ixgbe_if_detach(if_ctx_t ctx)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	device_t       dev = iflib_get_dev(ctx);
 	u32            ctrl_ext;
 
 	INIT_DEBUGOUT("ixgbe_detach: begin");
 
 	if (ixgbe_pci_iov_detach(dev) != 0) {
 		device_printf(dev, "SR-IOV in use; detach first.\n");
 		return (EBUSY);
 	}
 
 	ixgbe_setup_low_power_mode(ctx);
 
 	/* let hardware know driver is unloading */
 	ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
 	ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
 	IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
 
 	ixgbe_free_pci_resources(ctx);
 	free(adapter->mta, M_IXGBE);
 
 	return (0);
 } /* ixgbe_if_detach */
 
 /************************************************************************
  * ixgbe_setup_low_power_mode - LPLU/WoL preparation
  *
  *   Prepare the adapter/port for LPLU and/or WoL
  ************************************************************************/
 static int
 ixgbe_setup_low_power_mode(if_ctx_t ctx)
 {
 	struct adapter  *adapter = iflib_get_softc(ctx);
 	struct ixgbe_hw *hw = &adapter->hw;
 	device_t        dev = iflib_get_dev(ctx);
 	s32             error = 0;
 
 	if (!hw->wol_enabled)
 		ixgbe_set_phy_power(hw, FALSE);
 
 	/* Limit power management flow to X550EM baseT */
 	if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T &&
 	    hw->phy.ops.enter_lplu) {
 		/* Turn off support for APM wakeup. (Using ACPI instead) */
 		IXGBE_WRITE_REG(hw, IXGBE_GRC,
 		    IXGBE_READ_REG(hw, IXGBE_GRC) & ~(u32)2);
 
 		/*
 		 * Clear Wake Up Status register to prevent any previous wakeup
 		 * events from waking us up immediately after we suspend.
 		 */
 		IXGBE_WRITE_REG(hw, IXGBE_WUS, 0xffffffff);
 
 		/*
 		 * Program the Wakeup Filter Control register with user filter
 		 * settings
 		 */
 		IXGBE_WRITE_REG(hw, IXGBE_WUFC, adapter->wufc);
 
 		/* Enable wakeups and power management in Wakeup Control */
 		IXGBE_WRITE_REG(hw, IXGBE_WUC,
 		    IXGBE_WUC_WKEN | IXGBE_WUC_PME_EN);
 
 		/* X550EM baseT adapters need a special LPLU flow */
 		hw->phy.reset_disable = TRUE;
 		ixgbe_if_stop(ctx);
 		error = hw->phy.ops.enter_lplu(hw);
 		if (error)
 			device_printf(dev, "Error entering LPLU: %d\n", error);
 		hw->phy.reset_disable = FALSE;
 	} else {
 		/* Just stop for other adapters */
 		ixgbe_if_stop(ctx);
 	}
 
 	return error;
 } /* ixgbe_setup_low_power_mode */
 
 /************************************************************************
  * ixgbe_shutdown - Shutdown entry point
  ************************************************************************/
 static int
 ixgbe_if_shutdown(if_ctx_t ctx)
 {
 	int error = 0;
 
 	INIT_DEBUGOUT("ixgbe_shutdown: begin");
 
 	error = ixgbe_setup_low_power_mode(ctx);
 
 	return (error);
 } /* ixgbe_if_shutdown */
 
 /************************************************************************
  * ixgbe_suspend
  *
  *   From D0 to D3
  ************************************************************************/
 static int
 ixgbe_if_suspend(if_ctx_t ctx)
 {
 	int error = 0;
 
 	INIT_DEBUGOUT("ixgbe_suspend: begin");
 
 	error = ixgbe_setup_low_power_mode(ctx);
 
 	return (error);
 } /* ixgbe_if_suspend */
 
 /************************************************************************
  * ixgbe_resume
  *
  *   From D3 to D0
  ************************************************************************/
 static int
 ixgbe_if_resume(if_ctx_t ctx)
 {
 	struct adapter  *adapter = iflib_get_softc(ctx);
 	device_t        dev = iflib_get_dev(ctx);
 	struct ifnet    *ifp = iflib_get_ifp(ctx);
 	struct ixgbe_hw *hw = &adapter->hw;
 	u32             wus;
 
 	INIT_DEBUGOUT("ixgbe_resume: begin");
 
 	/* Read & clear WUS register */
 	wus = IXGBE_READ_REG(hw, IXGBE_WUS);
 	if (wus)
 		device_printf(dev, "Woken up by (WUS): %#010x\n",
 		    IXGBE_READ_REG(hw, IXGBE_WUS));
 	IXGBE_WRITE_REG(hw, IXGBE_WUS, 0xffffffff);
 	/* And clear WUFC until next low-power transition */
 	IXGBE_WRITE_REG(hw, IXGBE_WUFC, 0);
 
 	/*
 	 * Required after D3->D0 transition;
 	 * will re-advertise all previous advertised speeds
 	 */
 	if (ifp->if_flags & IFF_UP)
 		ixgbe_if_init(ctx);
 
 	return (0);
 } /* ixgbe_if_resume */
 
 /************************************************************************
  * ixgbe_if_mtu_set - Ioctl mtu entry point
  *
  *   Return 0 on success, EINVAL on failure
  ************************************************************************/
 static int
 ixgbe_if_mtu_set(if_ctx_t ctx, uint32_t mtu)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	int error = 0;
 
 	IOCTL_DEBUGOUT("ioctl: SIOCIFMTU (Set Interface MTU)");
 
 	if (mtu > IXGBE_MAX_MTU) {
 		error = EINVAL;
 	} else {
 		adapter->max_frame_size = mtu + IXGBE_MTU_HDR;
 	}
 
 	return error;
 } /* ixgbe_if_mtu_set */
 
 /************************************************************************
  * ixgbe_if_crcstrip_set
  ************************************************************************/
 static void
 ixgbe_if_crcstrip_set(if_ctx_t ctx, int onoff, int crcstrip)
 {
 	struct adapter *sc = iflib_get_softc(ctx);
 	struct ixgbe_hw *hw = &sc->hw;
 	/* crc stripping is set in two places:
 	 * IXGBE_HLREG0 (modified on init_locked and hw reset)
 	 * IXGBE_RDRXCTL (set by the original driver in
 	 *	ixgbe_setup_hw_rsc() called in init_locked.
 	 *	We disable the setting when netmap is compiled in).
 	 * We update the values here, but also in ixgbe.c because
 	 * init_locked sometimes is called outside our control.
 	 */
 	uint32_t hl, rxc;
 
 	hl = IXGBE_READ_REG(hw, IXGBE_HLREG0);
 	rxc = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
 #ifdef NETMAP
 	if (netmap_verbose)
 		D("%s read  HLREG 0x%x rxc 0x%x",
 			onoff ? "enter" : "exit", hl, rxc);
 #endif
 	/* hw requirements ... */
 	rxc &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
 	rxc |= IXGBE_RDRXCTL_RSCACKC;
 	if (onoff && !crcstrip) {
 		/* keep the crc. Fast rx */
 		hl &= ~IXGBE_HLREG0_RXCRCSTRP;
 		rxc &= ~IXGBE_RDRXCTL_CRCSTRIP;
 	} else {
 		/* reset default mode */
 		hl |= IXGBE_HLREG0_RXCRCSTRP;
 		rxc |= IXGBE_RDRXCTL_CRCSTRIP;
 	}
 #ifdef NETMAP
 	if (netmap_verbose)
 		D("%s write HLREG 0x%x rxc 0x%x",
 			onoff ? "enter" : "exit", hl, rxc);
 #endif
 	IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hl);
 	IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rxc);
 } /* ixgbe_if_crcstrip_set */
 
 /*********************************************************************
  * ixgbe_if_init - Init entry point
  *
  *   Used in two ways: It is used by the stack as an init
  *   entry point in network interface structure. It is also
  *   used by the driver as a hw/sw initialization routine to
  *   get to a consistent state.
  *
  *   Return 0 on success, positive on failure
  **********************************************************************/
 void
 ixgbe_if_init(if_ctx_t ctx)
 {
 	struct adapter     *adapter = iflib_get_softc(ctx);
 	struct ifnet       *ifp = iflib_get_ifp(ctx);
 	device_t           dev = iflib_get_dev(ctx);
 	struct ixgbe_hw *hw = &adapter->hw;
 	struct ix_rx_queue *rx_que;
 	struct ix_tx_queue *tx_que;
 	u32             txdctl, mhadd;
 	u32             rxdctl, rxctrl;
 	u32             ctrl_ext;
 
 	int             i, j, err;
 
 	INIT_DEBUGOUT("ixgbe_if_init: begin");
 
 	/* Queue indices may change with IOV mode */
 	ixgbe_align_all_queue_indices(adapter);
 
 	/* reprogram the RAR[0] in case user changed it. */
 	ixgbe_set_rar(hw, 0, hw->mac.addr, adapter->pool, IXGBE_RAH_AV);
 
 	/* Get the latest mac address, User can use a LAA */
 	bcopy(IF_LLADDR(ifp), hw->mac.addr, IXGBE_ETH_LENGTH_OF_ADDRESS);
 	ixgbe_set_rar(hw, 0, hw->mac.addr, adapter->pool, 1);
 	hw->addr_ctrl.rar_used_count = 1;
 
 	ixgbe_init_hw(hw);
 
 	ixgbe_initialize_iov(adapter);
 
 	ixgbe_initialize_transmit_units(ctx);
 
 	/* Setup Multicast table */
 	ixgbe_if_multi_set(ctx);
 
 	/* Determine the correct mbuf pool, based on frame size */
-	if (adapter->max_frame_size <= MCLBYTES)
-		adapter->rx_mbuf_sz = MCLBYTES;
-	else
-		adapter->rx_mbuf_sz = MJUMPAGESIZE;
+	adapter->rx_mbuf_sz = iflib_get_rx_mbuf_sz(ctx);
 
 	/* Configure RX settings */
 	ixgbe_initialize_receive_units(ctx);
 
 	/*
 	 * Initialize variable holding task enqueue requests
 	 * from MSI-X interrupts
 	 */
 	adapter->task_requests = 0;
 
 	/* Enable SDP & MSI-X interrupts based on adapter */
 	ixgbe_config_gpie(adapter);
 
 	/* Set MTU size */
 	if (ifp->if_mtu > ETHERMTU) {
 		/* aka IXGBE_MAXFRS on 82599 and newer */
 		mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
 		mhadd &= ~IXGBE_MHADD_MFS_MASK;
 		mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
 		IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
 	}
 
 	/* Now enable all the queues */
 	for (i = 0, tx_que = adapter->tx_queues; i < adapter->num_tx_queues; i++, tx_que++) {
 		struct tx_ring *txr = &tx_que->txr;
 
 		txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(txr->me));
 		txdctl |= IXGBE_TXDCTL_ENABLE;
 		/* Set WTHRESH to 8, burst writeback */
 		txdctl |= (8 << 16);
 		/*
 		 * When the internal queue falls below PTHRESH (32),
 		 * start prefetching as long as there are at least
 		 * HTHRESH (1) buffers ready. The values are taken
 		 * from the Intel linux driver 3.8.21.
 		 * Prefetching enables tx line rate even with 1 queue.
 		 */
 		txdctl |= (32 << 0) | (1 << 8);
 		IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(txr->me), txdctl);
 	}
 
 	for (i = 0, rx_que = adapter->rx_queues; i < adapter->num_rx_queues; i++, rx_que++) {
 		struct rx_ring *rxr = &rx_que->rxr;
 
 		rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxr->me));
 		if (hw->mac.type == ixgbe_mac_82598EB) {
 			/*
 			 * PTHRESH = 21
 			 * HTHRESH = 4
 			 * WTHRESH = 8
 			 */
 			rxdctl &= ~0x3FFFFF;
 			rxdctl |= 0x080420;
 		}
 		rxdctl |= IXGBE_RXDCTL_ENABLE;
 		IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxr->me), rxdctl);
 		for (j = 0; j < 10; j++) {
 			if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxr->me)) &
 			    IXGBE_RXDCTL_ENABLE)
 				break;
 			else
 				msec_delay(1);
 		}
 		wmb();
 	}
 
 	/* Enable Receive engine */
 	rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
 	if (hw->mac.type == ixgbe_mac_82598EB)
 		rxctrl |= IXGBE_RXCTRL_DMBYPS;
 	rxctrl |= IXGBE_RXCTRL_RXEN;
 	ixgbe_enable_rx_dma(hw, rxctrl);
 
 	/* Set up MSI/MSI-X routing */
 	if (ixgbe_enable_msix)  {
 		ixgbe_configure_ivars(adapter);
 		/* Set up auto-mask */
 		if (hw->mac.type == ixgbe_mac_82598EB)
 			IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
 		else {
 			IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
 			IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
 		}
 	} else {  /* Simple settings for Legacy/MSI */
 		ixgbe_set_ivar(adapter, 0, 0, 0);
 		ixgbe_set_ivar(adapter, 0, 0, 1);
 		IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
 	}
 
 	ixgbe_init_fdir(adapter);
 
 	/*
 	 * Check on any SFP devices that
 	 * need to be kick-started
 	 */
 	if (hw->phy.type == ixgbe_phy_none) {
 		err = hw->phy.ops.identify(hw);
 		if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
 			device_printf(dev,
 			    "Unsupported SFP+ module type was detected.\n");
 			return;
 		}
 	}
 
 	/* Set moderation on the Link interrupt */
 	IXGBE_WRITE_REG(hw, IXGBE_EITR(adapter->vector), IXGBE_LINK_ITR);
 
 	/* Enable power to the phy. */
 	ixgbe_set_phy_power(hw, TRUE);
 
 	/* Config/Enable Link */
 	ixgbe_config_link(ctx);
 
 	/* Hardware Packet Buffer & Flow Control setup */
 	ixgbe_config_delay_values(adapter);
 
 	/* Initialize the FC settings */
 	ixgbe_start_hw(hw);
 
 	/* Set up VLAN support and filter */
 	ixgbe_setup_vlan_hw_support(ctx);
 
 	/* Setup DMA Coalescing */
 	ixgbe_config_dmac(adapter);
 
 	/* And now turn on interrupts */
 	ixgbe_if_enable_intr(ctx);
 
 	/* Enable the use of the MBX by the VF's */
 	if (adapter->feat_en & IXGBE_FEATURE_SRIOV) {
 		ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 		ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
 		IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 	}
 
 } /* ixgbe_init_locked */
 
 /************************************************************************
  * ixgbe_set_ivar
  *
  *   Setup the correct IVAR register for a particular MSI-X interrupt
  *     (yes this is all very magic and confusing :)
  *    - entry is the register array entry
  *    - vector is the MSI-X vector for this queue
  *    - type is RX/TX/MISC
  ************************************************************************/
 static void
 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type)
 {
 	struct ixgbe_hw *hw = &adapter->hw;
 	u32 ivar, index;
 
 	vector |= IXGBE_IVAR_ALLOC_VAL;
 
 	switch (hw->mac.type) {
 	case ixgbe_mac_82598EB:
 		if (type == -1)
 			entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
 		else
 			entry += (type * 64);
 		index = (entry >> 2) & 0x1F;
 		ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
 		ivar &= ~(0xFF << (8 * (entry & 0x3)));
 		ivar |= (vector << (8 * (entry & 0x3)));
 		IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar);
 		break;
 	case ixgbe_mac_82599EB:
 	case ixgbe_mac_X540:
 	case ixgbe_mac_X550:
 	case ixgbe_mac_X550EM_x:
 	case ixgbe_mac_X550EM_a:
 		if (type == -1) { /* MISC IVAR */
 			index = (entry & 1) * 8;
 			ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
 			ivar &= ~(0xFF << index);
 			ivar |= (vector << index);
 			IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
 		} else {          /* RX/TX IVARS */
 			index = (16 * (entry & 1)) + (8 * type);
 			ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
 			ivar &= ~(0xFF << index);
 			ivar |= (vector << index);
 			IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
 		}
 	default:
 		break;
 	}
 } /* ixgbe_set_ivar */
 
 /************************************************************************
  * ixgbe_configure_ivars
  ************************************************************************/
 static void
 ixgbe_configure_ivars(struct adapter *adapter)
 {
 	struct ix_rx_queue *rx_que = adapter->rx_queues;
 	struct ix_tx_queue *tx_que = adapter->tx_queues;
 	u32                newitr;
 
 	if (ixgbe_max_interrupt_rate > 0)
 		newitr = (4000000 / ixgbe_max_interrupt_rate) & 0x0FF8;
 	else {
 		/*
 		 * Disable DMA coalescing if interrupt moderation is
 		 * disabled.
 		 */
 		adapter->dmac = 0;
 		newitr = 0;
 	}
 
 	for (int i = 0; i < adapter->num_rx_queues; i++, rx_que++) {
 		struct rx_ring *rxr = &rx_que->rxr;
 
 		/* First the RX queue entry */
 		ixgbe_set_ivar(adapter, rxr->me, rx_que->msix, 0);
 
 		/* Set an Initial EITR value */
 		IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(rx_que->msix), newitr);
 	}
 	for (int i = 0; i < adapter->num_tx_queues; i++, tx_que++) {
 		struct tx_ring *txr = &tx_que->txr;
 
 		/* ... and the TX */
 		ixgbe_set_ivar(adapter, txr->me, tx_que->msix, 1);
 	}
 	/* For the Link interrupt */
 	ixgbe_set_ivar(adapter, 1, adapter->vector, -1);
 } /* ixgbe_configure_ivars */
 
 /************************************************************************
  * ixgbe_config_gpie
  ************************************************************************/
 static void
 ixgbe_config_gpie(struct adapter *adapter)
 {
 	struct ixgbe_hw *hw = &adapter->hw;
 	u32             gpie;
 
 	gpie = IXGBE_READ_REG(hw, IXGBE_GPIE);
 
 	if (adapter->intr_type == IFLIB_INTR_MSIX) {
 		/* Enable Enhanced MSI-X mode */
 		gpie |= IXGBE_GPIE_MSIX_MODE
 		     |  IXGBE_GPIE_EIAME
 		     |  IXGBE_GPIE_PBA_SUPPORT
 		     |  IXGBE_GPIE_OCD;
 	}
 
 	/* Fan Failure Interrupt */
 	if (adapter->feat_en & IXGBE_FEATURE_FAN_FAIL)
 		gpie |= IXGBE_SDP1_GPIEN;
 
 	/* Thermal Sensor Interrupt */
 	if (adapter->feat_en & IXGBE_FEATURE_TEMP_SENSOR)
 		gpie |= IXGBE_SDP0_GPIEN_X540;
 
 	/* Link detection */
 	switch (hw->mac.type) {
 	case ixgbe_mac_82599EB:
 		gpie |= IXGBE_SDP1_GPIEN | IXGBE_SDP2_GPIEN;
 		break;
 	case ixgbe_mac_X550EM_x:
 	case ixgbe_mac_X550EM_a:
 		gpie |= IXGBE_SDP0_GPIEN_X540;
 		break;
 	default:
 		break;
 	}
 
 	IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
 
 } /* ixgbe_config_gpie */
 
 /************************************************************************
  * ixgbe_config_delay_values
  *
  *   Requires adapter->max_frame_size to be set.
  ************************************************************************/
 static void
 ixgbe_config_delay_values(struct adapter *adapter)
 {
 	struct ixgbe_hw *hw = &adapter->hw;
 	u32             rxpb, frame, size, tmp;
 
 	frame = adapter->max_frame_size;
 
 	/* Calculate High Water */
 	switch (hw->mac.type) {
 	case ixgbe_mac_X540:
 	case ixgbe_mac_X550:
 	case ixgbe_mac_X550EM_x:
 	case ixgbe_mac_X550EM_a:
 		tmp = IXGBE_DV_X540(frame, frame);
 		break;
 	default:
 		tmp = IXGBE_DV(frame, frame);
 		break;
 	}
 	size = IXGBE_BT2KB(tmp);
 	rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10;
 	hw->fc.high_water[0] = rxpb - size;
 
 	/* Now calculate Low Water */
 	switch (hw->mac.type) {
 	case ixgbe_mac_X540:
 	case ixgbe_mac_X550:
 	case ixgbe_mac_X550EM_x:
 	case ixgbe_mac_X550EM_a:
 		tmp = IXGBE_LOW_DV_X540(frame);
 		break;
 	default:
 		tmp = IXGBE_LOW_DV(frame);
 		break;
 	}
 	hw->fc.low_water[0] = IXGBE_BT2KB(tmp);
 
 	hw->fc.pause_time = IXGBE_FC_PAUSE;
 	hw->fc.send_xon = TRUE;
 } /* ixgbe_config_delay_values */
 
 /************************************************************************
  * ixgbe_set_multi - Multicast Update
  *
  *   Called whenever multicast address list is updated.
  ************************************************************************/
 static int
 ixgbe_mc_filter_apply(void *arg, struct ifmultiaddr *ifma, int count)
 {
 	struct adapter *adapter = arg;
 	struct ixgbe_mc_addr *mta = adapter->mta;
 
 	if (ifma->ifma_addr->sa_family != AF_LINK)
 		return (0);
 	if (count == MAX_NUM_MULTICAST_ADDRESSES)
 		return (0);
 	bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
 	    mta[count].addr, IXGBE_ETH_LENGTH_OF_ADDRESS);
 	mta[count].vmdq = adapter->pool;
 
 	return (1);
 } /* ixgbe_mc_filter_apply */
 
 static void
 ixgbe_if_multi_set(if_ctx_t ctx)
 {
 	struct adapter       *adapter = iflib_get_softc(ctx);
 	struct ixgbe_mc_addr *mta;
 	struct ifnet         *ifp = iflib_get_ifp(ctx);
 	u8                   *update_ptr;
 	int                  mcnt = 0;
 	u32                  fctrl;
 
 	IOCTL_DEBUGOUT("ixgbe_if_multi_set: begin");
 
 	mta = adapter->mta;
 	bzero(mta, sizeof(*mta) * MAX_NUM_MULTICAST_ADDRESSES);
 
 	mcnt = if_multi_apply(iflib_get_ifp(ctx), ixgbe_mc_filter_apply, adapter);
 
 	fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
 	fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
 	if (ifp->if_flags & IFF_PROMISC)
 		fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
 	else if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES ||
 	    ifp->if_flags & IFF_ALLMULTI) {
 		fctrl |= IXGBE_FCTRL_MPE;
 		fctrl &= ~IXGBE_FCTRL_UPE;
 	} else
 		fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
 
 	IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
 
 	if (mcnt < MAX_NUM_MULTICAST_ADDRESSES) {
 		update_ptr = (u8 *)mta;
 		ixgbe_update_mc_addr_list(&adapter->hw, update_ptr, mcnt,
 		    ixgbe_mc_array_itr, TRUE);
 	}
 
 } /* ixgbe_if_multi_set */
 
 /************************************************************************
  * ixgbe_mc_array_itr
  *
  *   An iterator function needed by the multicast shared code.
  *   It feeds the shared code routine the addresses in the
  *   array of ixgbe_set_multi() one by one.
  ************************************************************************/
 static u8 *
 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
 {
 	struct ixgbe_mc_addr *mta;
 
 	mta = (struct ixgbe_mc_addr *)*update_ptr;
 	*vmdq = mta->vmdq;
 
 	*update_ptr = (u8*)(mta + 1);
 
 	return (mta->addr);
 } /* ixgbe_mc_array_itr */
 
 /************************************************************************
  * ixgbe_local_timer - Timer routine
  *
  *   Checks for link status, updates statistics,
  *   and runs the watchdog check.
  ************************************************************************/
 static void
 ixgbe_if_timer(if_ctx_t ctx, uint16_t qid)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 
 	if (qid != 0)
 		return;
 
 	/* Check for pluggable optics */
 	if (adapter->sfp_probe)
 		if (!ixgbe_sfp_probe(ctx))
 			return; /* Nothing to do */
 
 	ixgbe_check_link(&adapter->hw, &adapter->link_speed,
 	    &adapter->link_up, 0);
 
 	/* Fire off the adminq task */
 	iflib_admin_intr_deferred(ctx);
 
 } /* ixgbe_if_timer */
 
 /************************************************************************
  * ixgbe_sfp_probe
  *
  *   Determine if a port had optics inserted.
  ************************************************************************/
 static bool
 ixgbe_sfp_probe(if_ctx_t ctx)
 {
 	struct adapter  *adapter = iflib_get_softc(ctx);
 	struct ixgbe_hw *hw = &adapter->hw;
 	device_t        dev = iflib_get_dev(ctx);
 	bool            result = FALSE;
 
 	if ((hw->phy.type == ixgbe_phy_nl) &&
 	    (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) {
 		s32 ret = hw->phy.ops.identify_sfp(hw);
 		if (ret)
 			goto out;
 		ret = hw->phy.ops.reset(hw);
 		adapter->sfp_probe = FALSE;
 		if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) {
 			device_printf(dev, "Unsupported SFP+ module detected!");
 			device_printf(dev,
 			    "Reload driver with supported module.\n");
 			goto out;
 		} else
 			device_printf(dev, "SFP+ module detected!\n");
 		/* We now have supported optics */
 		result = TRUE;
 	}
 out:
 
 	return (result);
 } /* ixgbe_sfp_probe */
 
 /************************************************************************
  * ixgbe_handle_mod - Tasklet for SFP module interrupts
  ************************************************************************/
 static void
 ixgbe_handle_mod(void *context)
 {
 	if_ctx_t        ctx = context;
 	struct adapter  *adapter = iflib_get_softc(ctx);
 	struct ixgbe_hw *hw = &adapter->hw;
 	device_t        dev = iflib_get_dev(ctx);
 	u32             err, cage_full = 0;
 
 	if (adapter->hw.need_crosstalk_fix) {
 		switch (hw->mac.type) {
 		case ixgbe_mac_82599EB:
 			cage_full = IXGBE_READ_REG(hw, IXGBE_ESDP) &
 			    IXGBE_ESDP_SDP2;
 			break;
 		case ixgbe_mac_X550EM_x:
 		case ixgbe_mac_X550EM_a:
 			cage_full = IXGBE_READ_REG(hw, IXGBE_ESDP) &
 			    IXGBE_ESDP_SDP0;
 			break;
 		default:
 			break;
 		}
 
 		if (!cage_full)
 			goto handle_mod_out;
 	}
 
 	err = hw->phy.ops.identify_sfp(hw);
 	if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
 		device_printf(dev,
 		    "Unsupported SFP+ module type was detected.\n");
 		goto handle_mod_out;
 	}
 
 	if (hw->mac.type == ixgbe_mac_82598EB)
 		err = hw->phy.ops.reset(hw);
 	else
 		err = hw->mac.ops.setup_sfp(hw);
 
 	if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
 		device_printf(dev,
 		    "Setup failure - unsupported SFP+ module type.\n");
 		goto handle_mod_out;
 	}
 	adapter->task_requests |= IXGBE_REQUEST_TASK_MSF;
 	return;
 
 handle_mod_out:
 	adapter->task_requests &= ~(IXGBE_REQUEST_TASK_MSF);
 } /* ixgbe_handle_mod */
 
 
 /************************************************************************
  * ixgbe_handle_msf - Tasklet for MSF (multispeed fiber) interrupts
  ************************************************************************/
 static void
 ixgbe_handle_msf(void *context)
 {
 	if_ctx_t        ctx = context;
 	struct adapter  *adapter = iflib_get_softc(ctx);
 	struct ixgbe_hw *hw = &adapter->hw;
 	u32             autoneg;
 	bool            negotiate;
 
 	/* get_supported_phy_layer will call hw->phy.ops.identify_sfp() */
 	adapter->phy_layer = ixgbe_get_supported_physical_layer(hw);
 
 	autoneg = hw->phy.autoneg_advertised;
 	if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
 		hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate);
 	if (hw->mac.ops.setup_link)
 		hw->mac.ops.setup_link(hw, autoneg, TRUE);
 
 	/* Adjust media types shown in ifconfig */
 	ifmedia_removeall(adapter->media);
 	ixgbe_add_media_types(adapter->ctx);
 	ifmedia_set(adapter->media, IFM_ETHER | IFM_AUTO);
 } /* ixgbe_handle_msf */
 
 /************************************************************************
  * ixgbe_handle_phy - Tasklet for external PHY interrupts
  ************************************************************************/
 static void
 ixgbe_handle_phy(void *context)
 {
 	if_ctx_t        ctx = context;
 	struct adapter  *adapter = iflib_get_softc(ctx);
 	struct ixgbe_hw *hw = &adapter->hw;
 	int             error;
 
 	error = hw->phy.ops.handle_lasi(hw);
 	if (error == IXGBE_ERR_OVERTEMP)
 		device_printf(adapter->dev, "CRITICAL: EXTERNAL PHY OVER TEMP!!  PHY will downshift to lower power state!\n");
 	else if (error)
 		device_printf(adapter->dev,
 		    "Error handling LASI interrupt: %d\n", error);
 } /* ixgbe_handle_phy */
 
 /************************************************************************
  * ixgbe_if_stop - Stop the hardware
  *
  *   Disables all traffic on the adapter by issuing a
  *   global reset on the MAC and deallocates TX/RX buffers.
  ************************************************************************/
 static void
 ixgbe_if_stop(if_ctx_t ctx)
 {
 	struct adapter  *adapter = iflib_get_softc(ctx);
 	struct ixgbe_hw *hw = &adapter->hw;
 
 	INIT_DEBUGOUT("ixgbe_if_stop: begin\n");
 
 	ixgbe_reset_hw(hw);
 	hw->adapter_stopped = FALSE;
 	ixgbe_stop_adapter(hw);
 	if (hw->mac.type == ixgbe_mac_82599EB)
 		ixgbe_stop_mac_link_on_d3_82599(hw);
 	/* Turn off the laser - noop with no optics */
 	ixgbe_disable_tx_laser(hw);
 
 	/* Update the stack */
 	adapter->link_up = FALSE;
 	ixgbe_if_update_admin_status(ctx);
 
 	/* reprogram the RAR[0] in case user changed it. */
 	ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
 
 	return;
 } /* ixgbe_if_stop */
 
 /************************************************************************
  * ixgbe_update_link_status - Update OS on link state
  *
  * Note: Only updates the OS on the cached link state.
  *       The real check of the hardware only happens with
  *       a link interrupt.
  ************************************************************************/
 static void
 ixgbe_if_update_admin_status(if_ctx_t ctx)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	device_t       dev = iflib_get_dev(ctx);
 
 	if (adapter->link_up) {
 		if (adapter->link_active == FALSE) {
 			if (bootverbose)
 				device_printf(dev, "Link is up %d Gbps %s \n",
 				    ((adapter->link_speed == 128) ? 10 : 1),
 				    "Full Duplex");
 			adapter->link_active = TRUE;
 			/* Update any Flow Control changes */
 			ixgbe_fc_enable(&adapter->hw);
 			/* Update DMA coalescing config */
 			ixgbe_config_dmac(adapter);
 			/* should actually be negotiated value */
 			iflib_link_state_change(ctx, LINK_STATE_UP, IF_Gbps(10));
 
 			if (adapter->feat_en & IXGBE_FEATURE_SRIOV)
 				ixgbe_ping_all_vfs(adapter);
 		}
 	} else { /* Link down */
 		if (adapter->link_active == TRUE) {
 			if (bootverbose)
 				device_printf(dev, "Link is Down\n");
 			iflib_link_state_change(ctx, LINK_STATE_DOWN, 0);
 			adapter->link_active = FALSE;
 			if (adapter->feat_en & IXGBE_FEATURE_SRIOV)
 				ixgbe_ping_all_vfs(adapter);
 		}
 	}
 
 	/* Handle task requests from msix_link() */
 	if (adapter->task_requests & IXGBE_REQUEST_TASK_MOD)
 		ixgbe_handle_mod(ctx);
 	if (adapter->task_requests & IXGBE_REQUEST_TASK_MSF)
 		ixgbe_handle_msf(ctx);
 	if (adapter->task_requests & IXGBE_REQUEST_TASK_MBX)
 		ixgbe_handle_mbx(ctx);
 	if (adapter->task_requests & IXGBE_REQUEST_TASK_FDIR)
 		ixgbe_reinit_fdir(ctx);
 	if (adapter->task_requests & IXGBE_REQUEST_TASK_PHY)
 		ixgbe_handle_phy(ctx);
 	adapter->task_requests = 0;
 
 	ixgbe_update_stats_counters(adapter);
 } /* ixgbe_if_update_admin_status */
 
 /************************************************************************
  * ixgbe_config_dmac - Configure DMA Coalescing
  ************************************************************************/
 static void
 ixgbe_config_dmac(struct adapter *adapter)
 {
 	struct ixgbe_hw          *hw = &adapter->hw;
 	struct ixgbe_dmac_config *dcfg = &hw->mac.dmac_config;
 
 	if (hw->mac.type < ixgbe_mac_X550 || !hw->mac.ops.dmac_config)
 		return;
 
 	if (dcfg->watchdog_timer ^ adapter->dmac ||
 	    dcfg->link_speed ^ adapter->link_speed) {
 		dcfg->watchdog_timer = adapter->dmac;
 		dcfg->fcoe_en = FALSE;
 		dcfg->link_speed = adapter->link_speed;
 		dcfg->num_tcs = 1;
 
 		INIT_DEBUGOUT2("dmac settings: watchdog %d, link speed %d\n",
 		    dcfg->watchdog_timer, dcfg->link_speed);
 
 		hw->mac.ops.dmac_config(hw);
 	}
 } /* ixgbe_config_dmac */
 
 /************************************************************************
  * ixgbe_if_enable_intr
  ************************************************************************/
 void
 ixgbe_if_enable_intr(if_ctx_t ctx)
 {
 	struct adapter     *adapter = iflib_get_softc(ctx);
 	struct ixgbe_hw    *hw = &adapter->hw;
 	struct ix_rx_queue *que = adapter->rx_queues;
 	u32                mask, fwsm;
 
 	mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
 
 	switch (adapter->hw.mac.type) {
 	case ixgbe_mac_82599EB:
 		mask |= IXGBE_EIMS_ECC;
 		/* Temperature sensor on some adapters */
 		mask |= IXGBE_EIMS_GPI_SDP0;
 		/* SFP+ (RX_LOS_N & MOD_ABS_N) */
 		mask |= IXGBE_EIMS_GPI_SDP1;
 		mask |= IXGBE_EIMS_GPI_SDP2;
 		break;
 	case ixgbe_mac_X540:
 		/* Detect if Thermal Sensor is enabled */
 		fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM);
 		if (fwsm & IXGBE_FWSM_TS_ENABLED)
 			mask |= IXGBE_EIMS_TS;
 		mask |= IXGBE_EIMS_ECC;
 		break;
 	case ixgbe_mac_X550:
 		/* MAC thermal sensor is automatically enabled */
 		mask |= IXGBE_EIMS_TS;
 		mask |= IXGBE_EIMS_ECC;
 		break;
 	case ixgbe_mac_X550EM_x:
 	case ixgbe_mac_X550EM_a:
 		/* Some devices use SDP0 for important information */
 		if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP ||
 		    hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP ||
 		    hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP_N ||
 		    hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T)
 			mask |= IXGBE_EIMS_GPI_SDP0_BY_MAC(hw);
 		if (hw->phy.type == ixgbe_phy_x550em_ext_t)
 			mask |= IXGBE_EICR_GPI_SDP0_X540;
 		mask |= IXGBE_EIMS_ECC;
 		break;
 	default:
 		break;
 	}
 
 	/* Enable Fan Failure detection */
 	if (adapter->feat_en & IXGBE_FEATURE_FAN_FAIL)
 		mask |= IXGBE_EIMS_GPI_SDP1;
 	/* Enable SR-IOV */
 	if (adapter->feat_en & IXGBE_FEATURE_SRIOV)
 		mask |= IXGBE_EIMS_MAILBOX;
 	/* Enable Flow Director */
 	if (adapter->feat_en & IXGBE_FEATURE_FDIR)
 		mask |= IXGBE_EIMS_FLOW_DIR;
 
 	IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
 
 	/* With MSI-X we use auto clear */
 	if (adapter->intr_type == IFLIB_INTR_MSIX) {
 		mask = IXGBE_EIMS_ENABLE_MASK;
 		/* Don't autoclear Link */
 		mask &= ~IXGBE_EIMS_OTHER;
 		mask &= ~IXGBE_EIMS_LSC;
 		if (adapter->feat_cap & IXGBE_FEATURE_SRIOV)
 			mask &= ~IXGBE_EIMS_MAILBOX;
 		IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask);
 	}
 
 	/*
 	 * Now enable all queues, this is done separately to
 	 * allow for handling the extended (beyond 32) MSI-X
 	 * vectors that can be used by 82599
 	 */
 	for (int i = 0; i < adapter->num_rx_queues; i++, que++)
 		ixgbe_enable_queue(adapter, que->msix);
 
 	IXGBE_WRITE_FLUSH(hw);
 
 } /* ixgbe_if_enable_intr */
 
 /************************************************************************
  * ixgbe_disable_intr
  ************************************************************************/
 static void
 ixgbe_if_disable_intr(if_ctx_t ctx)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 
 	if (adapter->intr_type == IFLIB_INTR_MSIX)
 		IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0);
 	if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
 		IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
 	} else {
 		IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
 		IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
 		IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
 	}
 	IXGBE_WRITE_FLUSH(&adapter->hw);
 
 } /* ixgbe_if_disable_intr */
 
 /************************************************************************
  * ixgbe_link_intr_enable
  ************************************************************************/
 static void
 ixgbe_link_intr_enable(if_ctx_t ctx)
 {
 	struct ixgbe_hw *hw = &((struct adapter *)iflib_get_softc(ctx))->hw;
 
 	/* Re-enable other interrupts */
 	IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_OTHER | IXGBE_EIMS_LSC);
 } /* ixgbe_link_intr_enable */
 
 /************************************************************************
  * ixgbe_if_rx_queue_intr_enable
  ************************************************************************/
 static int
 ixgbe_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid)
 {
 	struct adapter     *adapter = iflib_get_softc(ctx);
 	struct ix_rx_queue *que = &adapter->rx_queues[rxqid];
 
 	ixgbe_enable_queue(adapter, que->rxr.me);
 
 	return (0);
 } /* ixgbe_if_rx_queue_intr_enable */
 
 /************************************************************************
  * ixgbe_enable_queue
  ************************************************************************/
 static void
 ixgbe_enable_queue(struct adapter *adapter, u32 vector)
 {
 	struct ixgbe_hw *hw = &adapter->hw;
 	u64             queue = (u64)(1 << vector);
 	u32             mask;
 
 	if (hw->mac.type == ixgbe_mac_82598EB) {
 		mask = (IXGBE_EIMS_RTX_QUEUE & queue);
 		IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
 	} else {
 		mask = (queue & 0xFFFFFFFF);
 		if (mask)
 			IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
 		mask = (queue >> 32);
 		if (mask)
 			IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
 	}
 } /* ixgbe_enable_queue */
 
 /************************************************************************
  * ixgbe_disable_queue
  ************************************************************************/
 static void
 ixgbe_disable_queue(struct adapter *adapter, u32 vector)
 {
 	struct ixgbe_hw *hw = &adapter->hw;
 	u64             queue = (u64)(1 << vector);
 	u32             mask;
 
 	if (hw->mac.type == ixgbe_mac_82598EB) {
 		mask = (IXGBE_EIMS_RTX_QUEUE & queue);
 		IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
 	} else {
 		mask = (queue & 0xFFFFFFFF);
 		if (mask)
 			IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
 		mask = (queue >> 32);
 		if (mask)
 			IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
 	}
 } /* ixgbe_disable_queue */
 
 /************************************************************************
  * ixgbe_intr - Legacy Interrupt Service Routine
  ************************************************************************/
 int
 ixgbe_intr(void *arg)
 {
 	struct adapter     *adapter = arg;
 	struct ix_rx_queue *que = adapter->rx_queues;
 	struct ixgbe_hw    *hw = &adapter->hw;
 	if_ctx_t           ctx = adapter->ctx;
 	u32                eicr, eicr_mask;
 
 	eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
 
 	++que->irqs;
 	if (eicr == 0) {
 		ixgbe_if_enable_intr(ctx);
 		return (FILTER_HANDLED);
 	}
 
 	/* Check for fan failure */
 	if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
 	    (eicr & IXGBE_EICR_GPI_SDP1)) {
 		device_printf(adapter->dev,
 		    "\nCRITICAL: FAN FAILURE!! REPLACE IMMEDIATELY!!\n");
 		IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1_BY_MAC(hw));
 	}
 
 	/* Link status change */
 	if (eicr & IXGBE_EICR_LSC) {
 		IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_LSC);
 		iflib_admin_intr_deferred(ctx);
 	}
 
 	if (ixgbe_is_sfp(hw)) {
 		/* Pluggable optics-related interrupt */
 		if (hw->mac.type >= ixgbe_mac_X540)
 			eicr_mask = IXGBE_EICR_GPI_SDP0_X540;
 		else
 			eicr_mask = IXGBE_EICR_GPI_SDP2_BY_MAC(hw);
 
 		if (eicr & eicr_mask) {
 			IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr_mask);
 			adapter->task_requests |= IXGBE_REQUEST_TASK_MOD;
 		}
 
 		if ((hw->mac.type == ixgbe_mac_82599EB) &&
 		    (eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw))) {
 			IXGBE_WRITE_REG(hw, IXGBE_EICR,
 			    IXGBE_EICR_GPI_SDP1_BY_MAC(hw));
 			adapter->task_requests |= IXGBE_REQUEST_TASK_MSF;
 		}
 	}
 
 	/* External PHY interrupt */
 	if ((hw->phy.type == ixgbe_phy_x550em_ext_t) &&
 	    (eicr & IXGBE_EICR_GPI_SDP0_X540))
 		adapter->task_requests |= IXGBE_REQUEST_TASK_PHY;
 
 	return (FILTER_SCHEDULE_THREAD);
 } /* ixgbe_intr */
 
 /************************************************************************
  * ixgbe_free_pci_resources
  ************************************************************************/
 static void
 ixgbe_free_pci_resources(if_ctx_t ctx)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	struct         ix_rx_queue *que = adapter->rx_queues;
 	device_t       dev = iflib_get_dev(ctx);
 
 	/* Release all MSI-X queue resources */
 	if (adapter->intr_type == IFLIB_INTR_MSIX)
 		iflib_irq_free(ctx, &adapter->irq);
 
 	if (que != NULL) {
 		for (int i = 0; i < adapter->num_rx_queues; i++, que++) {
 			iflib_irq_free(ctx, &que->que_irq);
 		}
 	}
 
 	if (adapter->pci_mem != NULL)
 		bus_release_resource(dev, SYS_RES_MEMORY,
 		    rman_get_rid(adapter->pci_mem), adapter->pci_mem);
 } /* ixgbe_free_pci_resources */
 
 /************************************************************************
  * ixgbe_sysctl_flowcntl
  *
  *   SYSCTL wrapper around setting Flow Control
  ************************************************************************/
 static int
 ixgbe_sysctl_flowcntl(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter *adapter;
 	int            error, fc;
 
 	adapter = (struct adapter *)arg1;
 	fc = adapter->hw.fc.current_mode;
 
 	error = sysctl_handle_int(oidp, &fc, 0, req);
 	if ((error) || (req->newptr == NULL))
 		return (error);
 
 	/* Don't bother if it's not changed */
 	if (fc == adapter->hw.fc.current_mode)
 		return (0);
 
 	return ixgbe_set_flowcntl(adapter, fc);
 } /* ixgbe_sysctl_flowcntl */
 
 /************************************************************************
  * ixgbe_set_flowcntl - Set flow control
  *
  *   Flow control values:
  *     0 - off
  *     1 - rx pause
  *     2 - tx pause
  *     3 - full
  ************************************************************************/
 static int
 ixgbe_set_flowcntl(struct adapter *adapter, int fc)
 {
 	switch (fc) {
 	case ixgbe_fc_rx_pause:
 	case ixgbe_fc_tx_pause:
 	case ixgbe_fc_full:
 		adapter->hw.fc.requested_mode = fc;
 		if (adapter->num_rx_queues > 1)
 			ixgbe_disable_rx_drop(adapter);
 		break;
 	case ixgbe_fc_none:
 		adapter->hw.fc.requested_mode = ixgbe_fc_none;
 		if (adapter->num_rx_queues > 1)
 			ixgbe_enable_rx_drop(adapter);
 		break;
 	default:
 		return (EINVAL);
 	}
 
 	/* Don't autoneg if forcing a value */
 	adapter->hw.fc.disable_fc_autoneg = TRUE;
 	ixgbe_fc_enable(&adapter->hw);
 
 	return (0);
 } /* ixgbe_set_flowcntl */
 
 /************************************************************************
  * ixgbe_enable_rx_drop
  *
  *   Enable the hardware to drop packets when the buffer is
  *   full. This is useful with multiqueue, so that no single
  *   queue being full stalls the entire RX engine. We only
  *   enable this when Multiqueue is enabled AND Flow Control
  *   is disabled.
  ************************************************************************/
 static void
 ixgbe_enable_rx_drop(struct adapter *adapter)
 {
 	struct ixgbe_hw *hw = &adapter->hw;
 	struct rx_ring  *rxr;
 	u32             srrctl;
 
 	for (int i = 0; i < adapter->num_rx_queues; i++) {
 		rxr = &adapter->rx_queues[i].rxr;
 		srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(rxr->me));
 		srrctl |= IXGBE_SRRCTL_DROP_EN;
 		IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rxr->me), srrctl);
 	}
 
 	/* enable drop for each vf */
 	for (int i = 0; i < adapter->num_vfs; i++) {
 		IXGBE_WRITE_REG(hw, IXGBE_QDE,
 		                (IXGBE_QDE_WRITE | (i << IXGBE_QDE_IDX_SHIFT) |
 		                IXGBE_QDE_ENABLE));
 	}
 } /* ixgbe_enable_rx_drop */
 
 /************************************************************************
  * ixgbe_disable_rx_drop
  ************************************************************************/
 static void
 ixgbe_disable_rx_drop(struct adapter *adapter)
 {
 	struct ixgbe_hw *hw = &adapter->hw;
 	struct rx_ring  *rxr;
 	u32             srrctl;
 
 	for (int i = 0; i < adapter->num_rx_queues; i++) {
 		rxr = &adapter->rx_queues[i].rxr;
 		srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(rxr->me));
 		srrctl &= ~IXGBE_SRRCTL_DROP_EN;
 		IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rxr->me), srrctl);
 	}
 
 	/* disable drop for each vf */
 	for (int i = 0; i < adapter->num_vfs; i++) {
 		IXGBE_WRITE_REG(hw, IXGBE_QDE,
 		    (IXGBE_QDE_WRITE | (i << IXGBE_QDE_IDX_SHIFT)));
 	}
 } /* ixgbe_disable_rx_drop */
 
 /************************************************************************
  * ixgbe_sysctl_advertise
  *
  *   SYSCTL wrapper around setting advertised speed
  ************************************************************************/
 static int
 ixgbe_sysctl_advertise(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter *adapter;
 	int            error, advertise;
 
 	adapter = (struct adapter *)arg1;
 	advertise = adapter->advertise;
 
 	error = sysctl_handle_int(oidp, &advertise, 0, req);
 	if ((error) || (req->newptr == NULL))
 		return (error);
 
 	return ixgbe_set_advertise(adapter, advertise);
 } /* ixgbe_sysctl_advertise */
 
 /************************************************************************
  * ixgbe_set_advertise - Control advertised link speed
  *
  *   Flags:
  *     0x1 - advertise 100 Mb
  *     0x2 - advertise 1G
  *     0x4 - advertise 10G
  *     0x8 - advertise 10 Mb (yes, Mb)
  ************************************************************************/
 static int
 ixgbe_set_advertise(struct adapter *adapter, int advertise)
 {
 	device_t         dev = iflib_get_dev(adapter->ctx);
 	struct ixgbe_hw  *hw;
 	ixgbe_link_speed speed = 0;
 	ixgbe_link_speed link_caps = 0;
 	s32              err = IXGBE_NOT_IMPLEMENTED;
 	bool             negotiate = FALSE;
 
 	/* Checks to validate new value */
 	if (adapter->advertise == advertise) /* no change */
 		return (0);
 
 	hw = &adapter->hw;
 
 	/* No speed changes for backplane media */
 	if (hw->phy.media_type == ixgbe_media_type_backplane)
 		return (ENODEV);
 
 	if (!((hw->phy.media_type == ixgbe_media_type_copper) ||
 	      (hw->phy.multispeed_fiber))) {
 		device_printf(dev, "Advertised speed can only be set on copper or multispeed fiber media types.\n");
 		return (EINVAL);
 	}
 
 	if (advertise < 0x1 || advertise > 0xF) {
 		device_printf(dev, "Invalid advertised speed; valid modes are 0x1 through 0xF\n");
 		return (EINVAL);
 	}
 
 	if (hw->mac.ops.get_link_capabilities) {
 		err = hw->mac.ops.get_link_capabilities(hw, &link_caps,
 		    &negotiate);
 		if (err != IXGBE_SUCCESS) {
 			device_printf(dev, "Unable to determine supported advertise speeds\n");
 			return (ENODEV);
 		}
 	}
 
 	/* Set new value and report new advertised mode */
 	if (advertise & 0x1) {
 		if (!(link_caps & IXGBE_LINK_SPEED_100_FULL)) {
 			device_printf(dev, "Interface does not support 100Mb advertised speed\n");
 			return (EINVAL);
 		}
 		speed |= IXGBE_LINK_SPEED_100_FULL;
 	}
 	if (advertise & 0x2) {
 		if (!(link_caps & IXGBE_LINK_SPEED_1GB_FULL)) {
 			device_printf(dev, "Interface does not support 1Gb advertised speed\n");
 			return (EINVAL);
 		}
 		speed |= IXGBE_LINK_SPEED_1GB_FULL;
 	}
 	if (advertise & 0x4) {
 		if (!(link_caps & IXGBE_LINK_SPEED_10GB_FULL)) {
 			device_printf(dev, "Interface does not support 10Gb advertised speed\n");
 			return (EINVAL);
 		}
 		speed |= IXGBE_LINK_SPEED_10GB_FULL;
 	}
 	if (advertise & 0x8) {
 		if (!(link_caps & IXGBE_LINK_SPEED_10_FULL)) {
 			device_printf(dev, "Interface does not support 10Mb advertised speed\n");
 			return (EINVAL);
 		}
 		speed |= IXGBE_LINK_SPEED_10_FULL;
 	}
 
 	hw->mac.autotry_restart = TRUE;
 	hw->mac.ops.setup_link(hw, speed, TRUE);
 	adapter->advertise = advertise;
 
 	return (0);
 } /* ixgbe_set_advertise */
 
 /************************************************************************
  * ixgbe_get_advertise - Get current advertised speed settings
  *
  *   Formatted for sysctl usage.
  *   Flags:
  *     0x1 - advertise 100 Mb
  *     0x2 - advertise 1G
  *     0x4 - advertise 10G
  *     0x8 - advertise 10 Mb (yes, Mb)
  ************************************************************************/
 static int
 ixgbe_get_advertise(struct adapter *adapter)
 {
 	struct ixgbe_hw  *hw = &adapter->hw;
 	int              speed;
 	ixgbe_link_speed link_caps = 0;
 	s32              err;
 	bool             negotiate = FALSE;
 
 	/*
 	 * Advertised speed means nothing unless it's copper or
 	 * multi-speed fiber
 	 */
 	if (!(hw->phy.media_type == ixgbe_media_type_copper) &&
 	    !(hw->phy.multispeed_fiber))
 		return (0);
 
 	err = hw->mac.ops.get_link_capabilities(hw, &link_caps, &negotiate);
 	if (err != IXGBE_SUCCESS)
 		return (0);
 
 	speed =
 	    ((link_caps & IXGBE_LINK_SPEED_10GB_FULL) ? 4 : 0) |
 	    ((link_caps & IXGBE_LINK_SPEED_1GB_FULL)  ? 2 : 0) |
 	    ((link_caps & IXGBE_LINK_SPEED_100_FULL)  ? 1 : 0) |
 	    ((link_caps & IXGBE_LINK_SPEED_10_FULL)   ? 8 : 0);
 
 	return speed;
 } /* ixgbe_get_advertise */
 
 /************************************************************************
  * ixgbe_sysctl_dmac - Manage DMA Coalescing
  *
  *   Control values:
  *     0/1 - off / on (use default value of 1000)
  *
  *     Legal timer values are:
  *     50,100,250,500,1000,2000,5000,10000
  *
  *     Turning off interrupt moderation will also turn this off.
  ************************************************************************/
 static int
 ixgbe_sysctl_dmac(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter *adapter = (struct adapter *)arg1;
 	struct ifnet   *ifp = iflib_get_ifp(adapter->ctx);
 	int            error;
 	u16            newval;
 
 	newval = adapter->dmac;
 	error = sysctl_handle_16(oidp, &newval, 0, req);
 	if ((error) || (req->newptr == NULL))
 		return (error);
 
 	switch (newval) {
 	case 0:
 		/* Disabled */
 		adapter->dmac = 0;
 		break;
 	case 1:
 		/* Enable and use default */
 		adapter->dmac = 1000;
 		break;
 	case 50:
 	case 100:
 	case 250:
 	case 500:
 	case 1000:
 	case 2000:
 	case 5000:
 	case 10000:
 		/* Legal values - allow */
 		adapter->dmac = newval;
 		break;
 	default:
 		/* Do nothing, illegal value */
 		return (EINVAL);
 	}
 
 	/* Re-initialize hardware if it's already running */
 	if (ifp->if_drv_flags & IFF_DRV_RUNNING)
 		ifp->if_init(ifp);
 
 	return (0);
 } /* ixgbe_sysctl_dmac */
 
 #ifdef IXGBE_DEBUG
 /************************************************************************
  * ixgbe_sysctl_power_state
  *
  *   Sysctl to test power states
  *   Values:
  *     0      - set device to D0
  *     3      - set device to D3
  *     (none) - get current device power state
  ************************************************************************/
 static int
 ixgbe_sysctl_power_state(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter *adapter = (struct adapter *)arg1;
 	device_t       dev = adapter->dev;
 	int            curr_ps, new_ps, error = 0;
 
 	curr_ps = new_ps = pci_get_powerstate(dev);
 
 	error = sysctl_handle_int(oidp, &new_ps, 0, req);
 	if ((error) || (req->newptr == NULL))
 		return (error);
 
 	if (new_ps == curr_ps)
 		return (0);
 
 	if (new_ps == 3 && curr_ps == 0)
 		error = DEVICE_SUSPEND(dev);
 	else if (new_ps == 0 && curr_ps == 3)
 		error = DEVICE_RESUME(dev);
 	else
 		return (EINVAL);
 
 	device_printf(dev, "New state: %d\n", pci_get_powerstate(dev));
 
 	return (error);
 } /* ixgbe_sysctl_power_state */
 #endif
 
 /************************************************************************
  * ixgbe_sysctl_wol_enable
  *
  *   Sysctl to enable/disable the WoL capability,
  *   if supported by the adapter.
  *
  *   Values:
  *     0 - disabled
  *     1 - enabled
  ************************************************************************/
 static int
 ixgbe_sysctl_wol_enable(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter  *adapter = (struct adapter *)arg1;
 	struct ixgbe_hw *hw = &adapter->hw;
 	int             new_wol_enabled;
 	int             error = 0;
 
 	new_wol_enabled = hw->wol_enabled;
 	error = sysctl_handle_int(oidp, &new_wol_enabled, 0, req);
 	if ((error) || (req->newptr == NULL))
 		return (error);
 	new_wol_enabled = !!(new_wol_enabled);
 	if (new_wol_enabled == hw->wol_enabled)
 		return (0);
 
 	if (new_wol_enabled > 0 && !adapter->wol_support)
 		return (ENODEV);
 	else
 		hw->wol_enabled = new_wol_enabled;
 
 	return (0);
 } /* ixgbe_sysctl_wol_enable */
 
 /************************************************************************
  * ixgbe_sysctl_wufc - Wake Up Filter Control
  *
  *   Sysctl to enable/disable the types of packets that the
  *   adapter will wake up on upon receipt.
  *   Flags:
  *     0x1  - Link Status Change
  *     0x2  - Magic Packet
  *     0x4  - Direct Exact
  *     0x8  - Directed Multicast
  *     0x10 - Broadcast
  *     0x20 - ARP/IPv4 Request Packet
  *     0x40 - Direct IPv4 Packet
  *     0x80 - Direct IPv6 Packet
  *
  *   Settings not listed above will cause the sysctl to return an error.
  ************************************************************************/
 static int
 ixgbe_sysctl_wufc(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter *adapter = (struct adapter *)arg1;
 	int            error = 0;
 	u32            new_wufc;
 
 	new_wufc = adapter->wufc;
 
 	error = sysctl_handle_32(oidp, &new_wufc, 0, req);
 	if ((error) || (req->newptr == NULL))
 		return (error);
 	if (new_wufc == adapter->wufc)
 		return (0);
 
 	if (new_wufc & 0xffffff00)
 		return (EINVAL);
 
 	new_wufc &= 0xff;
 	new_wufc |= (0xffffff & adapter->wufc);
 	adapter->wufc = new_wufc;
 
 	return (0);
 } /* ixgbe_sysctl_wufc */
 
 #ifdef IXGBE_DEBUG
 /************************************************************************
  * ixgbe_sysctl_print_rss_config
  ************************************************************************/
 static int
 ixgbe_sysctl_print_rss_config(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter  *adapter = (struct adapter *)arg1;
 	struct ixgbe_hw *hw = &adapter->hw;
 	device_t        dev = adapter->dev;
 	struct sbuf     *buf;
 	int             error = 0, reta_size;
 	u32             reg;
 
 	buf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
 	if (!buf) {
 		device_printf(dev, "Could not allocate sbuf for output.\n");
 		return (ENOMEM);
 	}
 
 	// TODO: use sbufs to make a string to print out
 	/* Set multiplier for RETA setup and table size based on MAC */
 	switch (adapter->hw.mac.type) {
 	case ixgbe_mac_X550:
 	case ixgbe_mac_X550EM_x:
 	case ixgbe_mac_X550EM_a:
 		reta_size = 128;
 		break;
 	default:
 		reta_size = 32;
 		break;
 	}
 
 	/* Print out the redirection table */
 	sbuf_cat(buf, "\n");
 	for (int i = 0; i < reta_size; i++) {
 		if (i < 32) {
 			reg = IXGBE_READ_REG(hw, IXGBE_RETA(i));
 			sbuf_printf(buf, "RETA(%2d): 0x%08x\n", i, reg);
 		} else {
 			reg = IXGBE_READ_REG(hw, IXGBE_ERETA(i - 32));
 			sbuf_printf(buf, "ERETA(%2d): 0x%08x\n", i - 32, reg);
 		}
 	}
 
 	// TODO: print more config
 
 	error = sbuf_finish(buf);
 	if (error)
 		device_printf(dev, "Error finishing sbuf: %d\n", error);
 
 	sbuf_delete(buf);
 
 	return (0);
 } /* ixgbe_sysctl_print_rss_config */
 #endif /* IXGBE_DEBUG */
 
 /************************************************************************
  * ixgbe_sysctl_phy_temp - Retrieve temperature of PHY
  *
  *   For X552/X557-AT devices using an external PHY
  ************************************************************************/
 static int
 ixgbe_sysctl_phy_temp(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter  *adapter = (struct adapter *)arg1;
 	struct ixgbe_hw *hw = &adapter->hw;
 	u16             reg;
 
 	if (hw->device_id != IXGBE_DEV_ID_X550EM_X_10G_T) {
 		device_printf(iflib_get_dev(adapter->ctx),
 		    "Device has no supported external thermal sensor.\n");
 		return (ENODEV);
 	}
 
 	if (hw->phy.ops.read_reg(hw, IXGBE_PHY_CURRENT_TEMP,
 	    IXGBE_MDIO_VENDOR_SPECIFIC_1_DEV_TYPE, &reg)) {
 		device_printf(iflib_get_dev(adapter->ctx),
 		    "Error reading from PHY's current temperature register\n");
 		return (EAGAIN);
 	}
 
 	/* Shift temp for output */
 	reg = reg >> 8;
 
 	return (sysctl_handle_16(oidp, NULL, reg, req));
 } /* ixgbe_sysctl_phy_temp */
 
 /************************************************************************
  * ixgbe_sysctl_phy_overtemp_occurred
  *
  *   Reports (directly from the PHY) whether the current PHY
  *   temperature is over the overtemp threshold.
  ************************************************************************/
 static int
 ixgbe_sysctl_phy_overtemp_occurred(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter  *adapter = (struct adapter *)arg1;
 	struct ixgbe_hw *hw = &adapter->hw;
 	u16             reg;
 
 	if (hw->device_id != IXGBE_DEV_ID_X550EM_X_10G_T) {
 		device_printf(iflib_get_dev(adapter->ctx),
 		    "Device has no supported external thermal sensor.\n");
 		return (ENODEV);
 	}
 
 	if (hw->phy.ops.read_reg(hw, IXGBE_PHY_OVERTEMP_STATUS,
 	    IXGBE_MDIO_VENDOR_SPECIFIC_1_DEV_TYPE, &reg)) {
 		device_printf(iflib_get_dev(adapter->ctx),
 		    "Error reading from PHY's temperature status register\n");
 		return (EAGAIN);
 	}
 
 	/* Get occurrence bit */
 	reg = !!(reg & 0x4000);
 
 	return (sysctl_handle_16(oidp, 0, reg, req));
 } /* ixgbe_sysctl_phy_overtemp_occurred */
 
 /************************************************************************
  * ixgbe_sysctl_eee_state
  *
  *   Sysctl to set EEE power saving feature
  *   Values:
  *     0      - disable EEE
  *     1      - enable EEE
  *     (none) - get current device EEE state
  ************************************************************************/
 static int
 ixgbe_sysctl_eee_state(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter *adapter = (struct adapter *)arg1;
 	device_t       dev = adapter->dev;
 	struct ifnet   *ifp = iflib_get_ifp(adapter->ctx);
 	int            curr_eee, new_eee, error = 0;
 	s32            retval;
 
 	curr_eee = new_eee = !!(adapter->feat_en & IXGBE_FEATURE_EEE);
 
 	error = sysctl_handle_int(oidp, &new_eee, 0, req);
 	if ((error) || (req->newptr == NULL))
 		return (error);
 
 	/* Nothing to do */
 	if (new_eee == curr_eee)
 		return (0);
 
 	/* Not supported */
 	if (!(adapter->feat_cap & IXGBE_FEATURE_EEE))
 		return (EINVAL);
 
 	/* Bounds checking */
 	if ((new_eee < 0) || (new_eee > 1))
 		return (EINVAL);
 
 	retval = adapter->hw.mac.ops.setup_eee(&adapter->hw, new_eee);
 	if (retval) {
 		device_printf(dev, "Error in EEE setup: 0x%08X\n", retval);
 		return (EINVAL);
 	}
 
 	/* Restart auto-neg */
 	ifp->if_init(ifp);
 
 	device_printf(dev, "New EEE state: %d\n", new_eee);
 
 	/* Cache new value */
 	if (new_eee)
 		adapter->feat_en |= IXGBE_FEATURE_EEE;
 	else
 		adapter->feat_en &= ~IXGBE_FEATURE_EEE;
 
 	return (error);
 } /* ixgbe_sysctl_eee_state */
 
 /************************************************************************
  * ixgbe_init_device_features
  ************************************************************************/
 static void
 ixgbe_init_device_features(struct adapter *adapter)
 {
 	adapter->feat_cap = IXGBE_FEATURE_NETMAP
 	                  | IXGBE_FEATURE_RSS
 	                  | IXGBE_FEATURE_MSI
 	                  | IXGBE_FEATURE_MSIX
 	                  | IXGBE_FEATURE_LEGACY_IRQ;
 
 	/* Set capabilities first... */
 	switch (adapter->hw.mac.type) {
 	case ixgbe_mac_82598EB:
 		if (adapter->hw.device_id == IXGBE_DEV_ID_82598AT)
 			adapter->feat_cap |= IXGBE_FEATURE_FAN_FAIL;
 		break;
 	case ixgbe_mac_X540:
 		adapter->feat_cap |= IXGBE_FEATURE_SRIOV;
 		adapter->feat_cap |= IXGBE_FEATURE_FDIR;
 		if ((adapter->hw.device_id == IXGBE_DEV_ID_X540_BYPASS) &&
 		    (adapter->hw.bus.func == 0))
 			adapter->feat_cap |= IXGBE_FEATURE_BYPASS;
 		break;
 	case ixgbe_mac_X550:
 		adapter->feat_cap |= IXGBE_FEATURE_TEMP_SENSOR;
 		adapter->feat_cap |= IXGBE_FEATURE_SRIOV;
 		adapter->feat_cap |= IXGBE_FEATURE_FDIR;
 		break;
 	case ixgbe_mac_X550EM_x:
 		adapter->feat_cap |= IXGBE_FEATURE_SRIOV;
 		adapter->feat_cap |= IXGBE_FEATURE_FDIR;
 		if (adapter->hw.device_id == IXGBE_DEV_ID_X550EM_X_KR)
 			adapter->feat_cap |= IXGBE_FEATURE_EEE;
 		break;
 	case ixgbe_mac_X550EM_a:
 		adapter->feat_cap |= IXGBE_FEATURE_SRIOV;
 		adapter->feat_cap |= IXGBE_FEATURE_FDIR;
 		adapter->feat_cap &= ~IXGBE_FEATURE_LEGACY_IRQ;
 		if ((adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_1G_T) ||
 		    (adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_1G_T_L)) {
 			adapter->feat_cap |= IXGBE_FEATURE_TEMP_SENSOR;
 			adapter->feat_cap |= IXGBE_FEATURE_EEE;
 		}
 		break;
 	case ixgbe_mac_82599EB:
 		adapter->feat_cap |= IXGBE_FEATURE_SRIOV;
 		adapter->feat_cap |= IXGBE_FEATURE_FDIR;
 		if ((adapter->hw.device_id == IXGBE_DEV_ID_82599_BYPASS) &&
 		    (adapter->hw.bus.func == 0))
 			adapter->feat_cap |= IXGBE_FEATURE_BYPASS;
 		if (adapter->hw.device_id == IXGBE_DEV_ID_82599_QSFP_SF_QP)
 			adapter->feat_cap &= ~IXGBE_FEATURE_LEGACY_IRQ;
 		break;
 	default:
 		break;
 	}
 
 	/* Enabled by default... */
 	/* Fan failure detection */
 	if (adapter->feat_cap & IXGBE_FEATURE_FAN_FAIL)
 		adapter->feat_en |= IXGBE_FEATURE_FAN_FAIL;
 	/* Netmap */
 	if (adapter->feat_cap & IXGBE_FEATURE_NETMAP)
 		adapter->feat_en |= IXGBE_FEATURE_NETMAP;
 	/* EEE */
 	if (adapter->feat_cap & IXGBE_FEATURE_EEE)
 		adapter->feat_en |= IXGBE_FEATURE_EEE;
 	/* Thermal Sensor */
 	if (adapter->feat_cap & IXGBE_FEATURE_TEMP_SENSOR)
 		adapter->feat_en |= IXGBE_FEATURE_TEMP_SENSOR;
 
 	/* Enabled via global sysctl... */
 	/* Flow Director */
 	if (ixgbe_enable_fdir) {
 		if (adapter->feat_cap & IXGBE_FEATURE_FDIR)
 			adapter->feat_en |= IXGBE_FEATURE_FDIR;
 		else
 			device_printf(adapter->dev, "Device does not support Flow Director. Leaving disabled.");
 	}
 	/*
 	 * Message Signal Interrupts - Extended (MSI-X)
 	 * Normal MSI is only enabled if MSI-X calls fail.
 	 */
 	if (!ixgbe_enable_msix)
 		adapter->feat_cap &= ~IXGBE_FEATURE_MSIX;
 	/* Receive-Side Scaling (RSS) */
 	if ((adapter->feat_cap & IXGBE_FEATURE_RSS) && ixgbe_enable_rss)
 		adapter->feat_en |= IXGBE_FEATURE_RSS;
 
 	/* Disable features with unmet dependencies... */
 	/* No MSI-X */
 	if (!(adapter->feat_cap & IXGBE_FEATURE_MSIX)) {
 		adapter->feat_cap &= ~IXGBE_FEATURE_RSS;
 		adapter->feat_cap &= ~IXGBE_FEATURE_SRIOV;
 		adapter->feat_en &= ~IXGBE_FEATURE_RSS;
 		adapter->feat_en &= ~IXGBE_FEATURE_SRIOV;
 	}
 } /* ixgbe_init_device_features */
 
 /************************************************************************
  * ixgbe_check_fan_failure
  ************************************************************************/
 static void
 ixgbe_check_fan_failure(struct adapter *adapter, u32 reg, bool in_interrupt)
 {
 	u32 mask;
 
 	mask = (in_interrupt) ? IXGBE_EICR_GPI_SDP1_BY_MAC(&adapter->hw) :
 	    IXGBE_ESDP_SDP1;
 
 	if (reg & mask)
 		device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! REPLACE IMMEDIATELY!!\n");
 } /* ixgbe_check_fan_failure */
Index: head/sys/dev/ixgbe/if_ixv.c
===================================================================
--- head/sys/dev/ixgbe/if_ixv.c	(revision 345304)
+++ head/sys/dev/ixgbe/if_ixv.c	(revision 345305)
@@ -1,1934 +1,1927 @@
 /******************************************************************************
 
   Copyright (c) 2001-2017, Intel Corporation
   All rights reserved.
 
   Redistribution and use in source and binary forms, with or without
   modification, are permitted provided that the following conditions are met:
 
    1. Redistributions of source code must retain the above copyright notice,
       this list of conditions and the following disclaimer.
 
    2. Redistributions in binary form must reproduce the above copyright
       notice, this list of conditions and the following disclaimer in the
       documentation and/or other materials provided with the distribution.
 
    3. Neither the name of the Intel Corporation nor the names of its
       contributors may be used to endorse or promote products derived from
       this software without specific prior written permission.
 
   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
   POSSIBILITY OF SUCH DAMAGE.
 
 ******************************************************************************/
 /*$FreeBSD$*/
 
 
 #include "opt_inet.h"
 #include "opt_inet6.h"
 
 #include "ixgbe.h"
 #include "ifdi_if.h"
 
 #include <net/netmap.h>
 #include <dev/netmap/netmap_kern.h>
 
 /************************************************************************
  * Driver version
  ************************************************************************/
 char ixv_driver_version[] = "2.0.1-k";
 
 /************************************************************************
  * PCI Device ID Table
  *
  *   Used by probe to select devices to load on
  *   Last field stores an index into ixv_strings
  *   Last entry must be all 0s
  *
  *   { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  ************************************************************************/
 static pci_vendor_info_t ixv_vendor_info_array[] =
 {
 	PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF, "Intel(R) PRO/10GbE Virtual Function Network Driver"),
 	PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF, "Intel(R) PRO/10GbE Virtual Function Network Driver"),
 	PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550_VF, "Intel(R) PRO/10GbE Virtual Function Network Driver"),
 	PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_VF, "Intel(R) PRO/10GbE Virtual Function Network Driver"),
 	PVID(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_VF, "Intel(R) PRO/10GbE Virtual Function Network Driver"),
 	/* required last entry */
 PVID_END
 };
 
 /************************************************************************
  * Function prototypes
  ************************************************************************/
 static void     *ixv_register(device_t dev);
 static int      ixv_if_attach_pre(if_ctx_t ctx);
 static int      ixv_if_attach_post(if_ctx_t ctx);
 static int      ixv_if_detach(if_ctx_t ctx);
 
 static int      ixv_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t qid);
 static int      ixv_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nqs, int nqsets);
 static int      ixv_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nqs, int nqsets);
 static void     ixv_if_queues_free(if_ctx_t ctx);
 static void     ixv_identify_hardware(if_ctx_t ctx);
 static void     ixv_init_device_features(struct adapter *);
 static int      ixv_allocate_pci_resources(if_ctx_t ctx);
 static void     ixv_free_pci_resources(if_ctx_t ctx);
 static int      ixv_setup_interface(if_ctx_t ctx);
 static void     ixv_if_media_status(if_ctx_t , struct ifmediareq *);
 static int      ixv_if_media_change(if_ctx_t ctx);
 static void     ixv_if_update_admin_status(if_ctx_t ctx);
 static int      ixv_if_msix_intr_assign(if_ctx_t ctx, int msix);
 
 static int      ixv_if_mtu_set(if_ctx_t ctx, uint32_t mtu);
 static void     ixv_if_init(if_ctx_t ctx);
 static void     ixv_if_local_timer(if_ctx_t ctx, uint16_t qid);
 static void     ixv_if_stop(if_ctx_t ctx);
 static int      ixv_negotiate_api(struct adapter *);
 
 static void     ixv_initialize_transmit_units(if_ctx_t ctx);
 static void     ixv_initialize_receive_units(if_ctx_t ctx);
 static void     ixv_initialize_rss_mapping(struct adapter *);
 
 static void     ixv_setup_vlan_support(if_ctx_t ctx);
 static void     ixv_configure_ivars(struct adapter *);
 static void     ixv_if_enable_intr(if_ctx_t ctx);
 static void     ixv_if_disable_intr(if_ctx_t ctx);
 static void     ixv_if_multi_set(if_ctx_t ctx);
 
 static void     ixv_if_register_vlan(if_ctx_t, u16);
 static void     ixv_if_unregister_vlan(if_ctx_t, u16);
 
 static uint64_t ixv_if_get_counter(if_ctx_t, ift_counter);
 
 static void     ixv_save_stats(struct adapter *);
 static void     ixv_init_stats(struct adapter *);
 static void     ixv_update_stats(struct adapter *);
 static void     ixv_add_stats_sysctls(struct adapter *adapter);
 
 static int      ixv_sysctl_debug(SYSCTL_HANDLER_ARGS);
 static void     ixv_set_ivar(struct adapter *, u8, u8, s8);
 
 static u8       *ixv_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
 
 /* The MSI-X Interrupt handlers */
 static int      ixv_msix_que(void *);
 static int      ixv_msix_mbx(void *);
 
 /************************************************************************
  * FreeBSD Device Interface Entry Points
  ************************************************************************/
 static device_method_t ixv_methods[] = {
 	/* Device interface */
 	DEVMETHOD(device_register, ixv_register),
 	DEVMETHOD(device_probe, iflib_device_probe),
 	DEVMETHOD(device_attach, iflib_device_attach),
 	DEVMETHOD(device_detach, iflib_device_detach),
 	DEVMETHOD(device_shutdown, iflib_device_shutdown),
 	DEVMETHOD_END
 };
 
 static driver_t ixv_driver = {
 	"ixv", ixv_methods, sizeof(struct adapter),
 };
 
 devclass_t ixv_devclass;
 DRIVER_MODULE(ixv, pci, ixv_driver, ixv_devclass, 0, 0);
 IFLIB_PNP_INFO(pci, ixv_driver, ixv_vendor_info_array);
 MODULE_DEPEND(ixv, iflib, 1, 1, 1);
 MODULE_DEPEND(ixv, pci, 1, 1, 1);
 MODULE_DEPEND(ixv, ether, 1, 1, 1);
 
 static device_method_t ixv_if_methods[] = {
 	DEVMETHOD(ifdi_attach_pre, ixv_if_attach_pre),
 	DEVMETHOD(ifdi_attach_post, ixv_if_attach_post),
 	DEVMETHOD(ifdi_detach, ixv_if_detach),
 	DEVMETHOD(ifdi_init, ixv_if_init),
 	DEVMETHOD(ifdi_stop, ixv_if_stop),
 	DEVMETHOD(ifdi_msix_intr_assign, ixv_if_msix_intr_assign),
 	DEVMETHOD(ifdi_intr_enable, ixv_if_enable_intr),
 	DEVMETHOD(ifdi_intr_disable, ixv_if_disable_intr),
 	DEVMETHOD(ifdi_tx_queue_intr_enable, ixv_if_rx_queue_intr_enable),
 	DEVMETHOD(ifdi_rx_queue_intr_enable, ixv_if_rx_queue_intr_enable),
 	DEVMETHOD(ifdi_tx_queues_alloc, ixv_if_tx_queues_alloc),
 	DEVMETHOD(ifdi_rx_queues_alloc, ixv_if_rx_queues_alloc),
 	DEVMETHOD(ifdi_queues_free, ixv_if_queues_free),
 	DEVMETHOD(ifdi_update_admin_status, ixv_if_update_admin_status),
 	DEVMETHOD(ifdi_multi_set, ixv_if_multi_set),
 	DEVMETHOD(ifdi_mtu_set, ixv_if_mtu_set),
 	DEVMETHOD(ifdi_media_status, ixv_if_media_status),
 	DEVMETHOD(ifdi_media_change, ixv_if_media_change),
 	DEVMETHOD(ifdi_timer, ixv_if_local_timer),
 	DEVMETHOD(ifdi_vlan_register, ixv_if_register_vlan),
 	DEVMETHOD(ifdi_vlan_unregister, ixv_if_unregister_vlan),
 	DEVMETHOD(ifdi_get_counter, ixv_if_get_counter),
 	DEVMETHOD_END
 };
 
 static driver_t ixv_if_driver = {
   "ixv_if", ixv_if_methods, sizeof(struct adapter)
 };
 
 /*
  * TUNEABLE PARAMETERS:
  */
 
 /* Flow control setting, default to full */
 static int ixv_flow_control = ixgbe_fc_full;
 TUNABLE_INT("hw.ixv.flow_control", &ixv_flow_control);
 
 /*
  * Header split: this causes the hardware to DMA
  * the header into a separate mbuf from the payload,
  * it can be a performance win in some workloads, but
  * in others it actually hurts, its off by default.
  */
 static int ixv_header_split = FALSE;
 TUNABLE_INT("hw.ixv.hdr_split", &ixv_header_split);
 
 /*
  * Shadow VFTA table, this is needed because
  * the real filter table gets cleared during
  * a soft reset and we need to repopulate it.
  */
 static u32 ixv_shadow_vfta[IXGBE_VFTA_SIZE];
 extern struct if_txrx ixgbe_txrx;
 
 static struct if_shared_ctx ixv_sctx_init = {
 	.isc_magic = IFLIB_MAGIC,
 	.isc_q_align = PAGE_SIZE,/* max(DBA_ALIGN, PAGE_SIZE) */
 	.isc_tx_maxsize = IXGBE_TSO_SIZE + sizeof(struct ether_vlan_header),
 	.isc_tx_maxsegsize = PAGE_SIZE,
 	.isc_tso_maxsize = IXGBE_TSO_SIZE + sizeof(struct ether_vlan_header),
 	.isc_tso_maxsegsize = PAGE_SIZE,
 	.isc_rx_maxsize = MJUM16BYTES,
 	.isc_rx_nsegments = 1,
 	.isc_rx_maxsegsize = MJUM16BYTES,
 	.isc_nfl = 1,
 	.isc_ntxqs = 1,
 	.isc_nrxqs = 1,
 	.isc_admin_intrcnt = 1,
 	.isc_vendor_info = ixv_vendor_info_array,
 	.isc_driver_version = ixv_driver_version,
 	.isc_driver = &ixv_if_driver,
 	.isc_flags = IFLIB_TSO_INIT_IP,
 
 	.isc_nrxd_min = {MIN_RXD},
 	.isc_ntxd_min = {MIN_TXD},
 	.isc_nrxd_max = {MAX_RXD},
 	.isc_ntxd_max = {MAX_TXD},
 	.isc_nrxd_default = {DEFAULT_RXD},
 	.isc_ntxd_default = {DEFAULT_TXD},
 };
 
 if_shared_ctx_t ixv_sctx = &ixv_sctx_init;
 
 static void *
 ixv_register(device_t dev)
 {
 	return (ixv_sctx);
 }
 
 /************************************************************************
  * ixv_if_tx_queues_alloc
  ************************************************************************/
 static int
 ixv_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs,
                        int ntxqs, int ntxqsets)
 {
 	struct adapter     *adapter = iflib_get_softc(ctx);
 	if_softc_ctx_t     scctx = adapter->shared;
 	struct ix_tx_queue *que;
 	int                i, j, error;
 
 	MPASS(adapter->num_tx_queues == ntxqsets);
 	MPASS(ntxqs == 1);
 
 	/* Allocate queue structure memory */
 	adapter->tx_queues =
 	    (struct ix_tx_queue *)malloc(sizeof(struct ix_tx_queue) * ntxqsets,
 	                                 M_DEVBUF, M_NOWAIT | M_ZERO);
 	if (!adapter->tx_queues) {
 		device_printf(iflib_get_dev(ctx),
 		    "Unable to allocate TX ring memory\n");
 		return (ENOMEM);
 	}
 
 	for (i = 0, que = adapter->tx_queues; i < ntxqsets; i++, que++) {
 		struct tx_ring *txr = &que->txr;
 
 		txr->me = i;
 		txr->adapter =  que->adapter = adapter;
 		adapter->active_queues |= (u64)1 << txr->me;
 
 		/* Allocate report status array */
 		if (!(txr->tx_rsq = (qidx_t *)malloc(sizeof(qidx_t) * scctx->isc_ntxd[0], M_DEVBUF, M_NOWAIT | M_ZERO))) {
 			error = ENOMEM;
 			goto fail;
 		}
 		for (j = 0; j < scctx->isc_ntxd[0]; j++)
 			txr->tx_rsq[j] = QIDX_INVALID;
 		/* get the virtual and physical address of the hardware queues */
 		txr->tail = IXGBE_VFTDT(txr->me);
 		txr->tx_base = (union ixgbe_adv_tx_desc *)vaddrs[i*ntxqs];
 		txr->tx_paddr = paddrs[i*ntxqs];
 
 		txr->bytes = 0;
 		txr->total_packets = 0;
 
 	}
 
 	device_printf(iflib_get_dev(ctx), "allocated for %d queues\n",
 	    adapter->num_tx_queues);
 
 	return (0);
 
  fail:
 	ixv_if_queues_free(ctx);
 
 	return (error);
 } /* ixv_if_tx_queues_alloc */
 
 /************************************************************************
  * ixv_if_rx_queues_alloc
  ************************************************************************/
 static int
 ixv_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs,
                        int nrxqs, int nrxqsets)
 {
 	struct adapter     *adapter = iflib_get_softc(ctx);
 	struct ix_rx_queue *que;
 	int                i, error;
 
 	MPASS(adapter->num_rx_queues == nrxqsets);
 	MPASS(nrxqs == 1);
 
 	/* Allocate queue structure memory */
 	adapter->rx_queues =
 	    (struct ix_rx_queue *)malloc(sizeof(struct ix_rx_queue) * nrxqsets,
 	                                 M_DEVBUF, M_NOWAIT | M_ZERO);
 	if (!adapter->rx_queues) {
 		device_printf(iflib_get_dev(ctx),
 		    "Unable to allocate TX ring memory\n");
 		error = ENOMEM;
 		goto fail;
 	}
 
 	for (i = 0, que = adapter->rx_queues; i < nrxqsets; i++, que++) {
 		struct rx_ring *rxr = &que->rxr;
 		rxr->me = i;
 		rxr->adapter = que->adapter = adapter;
 
 
 		/* get the virtual and physical address of the hw queues */
 		rxr->tail = IXGBE_VFRDT(rxr->me);
 		rxr->rx_base = (union ixgbe_adv_rx_desc *)vaddrs[i];
 		rxr->rx_paddr = paddrs[i*nrxqs];
 		rxr->bytes = 0;
 		rxr->que = que;
 	}
 
 	device_printf(iflib_get_dev(ctx), "allocated for %d rx queues\n",
 	    adapter->num_rx_queues);
 
 	return (0);
 
 fail:
 	ixv_if_queues_free(ctx);
 
 	return (error);
 } /* ixv_if_rx_queues_alloc */
 
 /************************************************************************
  * ixv_if_queues_free
  ************************************************************************/
 static void
 ixv_if_queues_free(if_ctx_t ctx)
 {
 	struct adapter     *adapter = iflib_get_softc(ctx);
 	struct ix_tx_queue *que = adapter->tx_queues;
 	int                i;
 
 	if (que == NULL)
 		goto free;
 
 	for (i = 0; i < adapter->num_tx_queues; i++, que++) {
 		struct tx_ring *txr = &que->txr;
 		if (txr->tx_rsq == NULL)
 			break;
 
 		free(txr->tx_rsq, M_DEVBUF);
 		txr->tx_rsq = NULL;
 	}
 	if (adapter->tx_queues != NULL)
 		free(adapter->tx_queues, M_DEVBUF);
 free:
 	if (adapter->rx_queues != NULL)
 		free(adapter->rx_queues, M_DEVBUF);
 	adapter->tx_queues = NULL;
 	adapter->rx_queues = NULL;
 } /* ixv_if_queues_free */
 
 /************************************************************************
  * ixv_if_attach_pre - Device initialization routine
  *
  *   Called when the driver is being loaded.
  *   Identifies the type of hardware, allocates all resources
  *   and initializes the hardware.
  *
  *   return 0 on success, positive on failure
  ************************************************************************/
 static int
 ixv_if_attach_pre(if_ctx_t ctx)
 {
 	struct adapter  *adapter;
 	device_t        dev;
 	if_softc_ctx_t  scctx;
 	struct ixgbe_hw *hw;
 	int             error = 0;
 
 	INIT_DEBUGOUT("ixv_attach: begin");
 
 	/* Allocate, clear, and link in our adapter structure */
 	dev = iflib_get_dev(ctx);
 	adapter = iflib_get_softc(ctx);
 	adapter->dev = dev;
 	adapter->ctx = ctx;
 	adapter->hw.back = adapter;
 	scctx = adapter->shared = iflib_get_softc_ctx(ctx);
 	adapter->media = iflib_get_media(ctx);
 	hw = &adapter->hw;
 
 	/* Do base PCI setup - map BAR0 */
 	if (ixv_allocate_pci_resources(ctx)) {
 		device_printf(dev, "ixv_allocate_pci_resources() failed!\n");
 		error = ENXIO;
 		goto err_out;
 	}
 
 	/* SYSCTL APIs */
 	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "debug",
 	    CTLTYPE_INT | CTLFLAG_RW, adapter, 0, ixv_sysctl_debug, "I",
 	    "Debug Info");
 
 	/* Determine hardware revision */
 	ixv_identify_hardware(ctx);
 	ixv_init_device_features(adapter);
 
 	/* Initialize the shared code */
 	error = ixgbe_init_ops_vf(hw);
 	if (error) {
 		device_printf(dev, "ixgbe_init_ops_vf() failed!\n");
 		error = EIO;
 		goto err_out;
 	}
 
 	/* Setup the mailbox */
 	ixgbe_init_mbx_params_vf(hw);
 
 	error = hw->mac.ops.reset_hw(hw);
 	if (error == IXGBE_ERR_RESET_FAILED)
 		device_printf(dev, "...reset_hw() failure: Reset Failed!\n");
 	else if (error)
 		device_printf(dev, "...reset_hw() failed with error %d\n",
 		    error);
 	if (error) {
 		error = EIO;
 		goto err_out;
 	}
 
 	error = hw->mac.ops.init_hw(hw);
 	if (error) {
 		device_printf(dev, "...init_hw() failed with error %d\n",
 		    error);
 		error = EIO;
 		goto err_out;
 	}
 
 	/* Negotiate mailbox API version */
 	error = ixv_negotiate_api(adapter);
 	if (error) {
 		device_printf(dev,
 		    "Mailbox API negotiation failed during attach!\n");
 		goto err_out;
 	}
 
 	/* If no mac address was assigned, make a random one */
 	if (!ixv_check_ether_addr(hw->mac.addr)) {
 		u8 addr[ETHER_ADDR_LEN];
 		arc4rand(&addr, sizeof(addr), 0);
 		addr[0] &= 0xFE;
 		addr[0] |= 0x02;
 		bcopy(addr, hw->mac.addr, sizeof(addr));
 		bcopy(addr, hw->mac.perm_addr, sizeof(addr));
 	}
 
 	/* Most of the iflib initialization... */
 
 	iflib_set_mac(ctx, hw->mac.addr);
 	switch (adapter->hw.mac.type) {
 	case ixgbe_mac_X550_vf:
 	case ixgbe_mac_X550EM_x_vf:
 	case ixgbe_mac_X550EM_a_vf:
 		scctx->isc_ntxqsets_max = scctx->isc_nrxqsets_max = 2;
 		break;
 	default:
 		scctx->isc_ntxqsets_max = scctx->isc_nrxqsets_max = 1;
 	}
 	scctx->isc_txqsizes[0] =
 	    roundup2(scctx->isc_ntxd[0] * sizeof(union ixgbe_adv_tx_desc) +
 	    sizeof(u32), DBA_ALIGN);
 	scctx->isc_rxqsizes[0] =
 	    roundup2(scctx->isc_nrxd[0] * sizeof(union ixgbe_adv_rx_desc),
 	    DBA_ALIGN);
 	/* XXX */
 	scctx->isc_tx_csum_flags = CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_TSO |
 	    CSUM_IP6_TCP | CSUM_IP6_UDP | CSUM_IP6_TSO;
 	scctx->isc_tx_nsegments = IXGBE_82599_SCATTER;
 	scctx->isc_msix_bar = PCIR_BAR(MSIX_82598_BAR);
 	scctx->isc_tx_tso_segments_max = scctx->isc_tx_nsegments;
 	scctx->isc_tx_tso_size_max = IXGBE_TSO_SIZE;
 	scctx->isc_tx_tso_segsize_max = PAGE_SIZE;
 
 	scctx->isc_txrx = &ixgbe_txrx;
 
 	/*
 	 * Tell the upper layer(s) we support everything the PF
 	 * driver does except...
 	 *   Wake-on-LAN
 	 */
 	scctx->isc_capabilities = IXGBE_CAPS;
 	scctx->isc_capabilities ^= IFCAP_WOL;
 	scctx->isc_capenable = scctx->isc_capabilities;
 
 	INIT_DEBUGOUT("ixv_if_attach_pre: end");
 
 	return (0);
 
 err_out:
 	ixv_free_pci_resources(ctx);
 
 	return (error);
 } /* ixv_if_attach_pre */
 
 static int
 ixv_if_attach_post(if_ctx_t ctx)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	device_t       dev = iflib_get_dev(ctx);
 	int            error = 0;
 
 	/* Setup OS specific network interface */
 	error = ixv_setup_interface(ctx);
 	if (error) {
 		device_printf(dev, "Interface setup failed: %d\n", error);
 		goto end;
 	}
 
 	/* Do the stats setup */
 	ixv_save_stats(adapter);
 	ixv_init_stats(adapter);
 	ixv_add_stats_sysctls(adapter);
 
 end:
 	return error;
 } /* ixv_if_attach_post */
 
 /************************************************************************
  * ixv_detach - Device removal routine
  *
  *   Called when the driver is being removed.
  *   Stops the adapter and deallocates all the resources
  *   that were allocated for driver operation.
  *
  *   return 0 on success, positive on failure
  ************************************************************************/
 static int
 ixv_if_detach(if_ctx_t ctx)
 {
 	INIT_DEBUGOUT("ixv_detach: begin");
 
 	ixv_free_pci_resources(ctx);
 
 	return (0);
 } /* ixv_if_detach */
 
 /************************************************************************
  * ixv_if_mtu_set
  ************************************************************************/
 static int
 ixv_if_mtu_set(if_ctx_t ctx, uint32_t mtu)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	struct ifnet   *ifp = iflib_get_ifp(ctx);
 	int            error = 0;
 
 	IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
 	if (mtu > IXGBE_MAX_FRAME_SIZE - IXGBE_MTU_HDR) {
 		error = EINVAL;
 	} else {
 		ifp->if_mtu = mtu;
 		adapter->max_frame_size = ifp->if_mtu + IXGBE_MTU_HDR;
 	}
 
 	return error;
 } /* ixv_if_mtu_set */
 
 /************************************************************************
  * ixv_if_init - Init entry point
  *
  *   Used in two ways: It is used by the stack as an init entry
  *   point in network interface structure. It is also used
  *   by the driver as a hw/sw initialization routine to get
  *   to a consistent state.
  *
  *   return 0 on success, positive on failure
  ************************************************************************/
 static void
 ixv_if_init(if_ctx_t ctx)
 {
 	struct adapter  *adapter = iflib_get_softc(ctx);
 	struct ifnet    *ifp = iflib_get_ifp(ctx);
 	device_t        dev = iflib_get_dev(ctx);
 	struct ixgbe_hw *hw = &adapter->hw;
 	int             error = 0;
 
 	INIT_DEBUGOUT("ixv_if_init: begin");
 	hw->adapter_stopped = FALSE;
 	hw->mac.ops.stop_adapter(hw);
 
 	/* reprogram the RAR[0] in case user changed it. */
 	hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
 
 	/* Get the latest mac address, User can use a LAA */
 	bcopy(IF_LLADDR(ifp), hw->mac.addr, IXGBE_ETH_LENGTH_OF_ADDRESS);
 	hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, 1);
 
 	/* Reset VF and renegotiate mailbox API version */
 	hw->mac.ops.reset_hw(hw);
 	hw->mac.ops.start_hw(hw);
 	error = ixv_negotiate_api(adapter);
 	if (error) {
 		device_printf(dev,
 		    "Mailbox API negotiation failed in if_init!\n");
 		return;
 	}
 
 	ixv_initialize_transmit_units(ctx);
 
 	/* Setup Multicast table */
 	ixv_if_multi_set(ctx);
 
-	/*
-	 * Determine the correct mbuf pool
-	 * for doing jumbo/headersplit
-	 */
-	if (ifp->if_mtu > ETHERMTU)
-		adapter->rx_mbuf_sz = MJUMPAGESIZE;
-	else
-		adapter->rx_mbuf_sz = MCLBYTES;
+	adapter->rx_mbuf_sz = iflib_get_rx_mbuf_sz(ctx);
 
 	/* Configure RX settings */
 	ixv_initialize_receive_units(ctx);
 
 	/* Set up VLAN offload and filter */
 	ixv_setup_vlan_support(ctx);
 
 	/* Set up MSI-X routing */
 	ixv_configure_ivars(adapter);
 
 	/* Set up auto-mask */
 	IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, IXGBE_EICS_RTX_QUEUE);
 
 	/* Set moderation on the Link interrupt */
 	IXGBE_WRITE_REG(hw, IXGBE_VTEITR(adapter->vector), IXGBE_LINK_ITR);
 
 	/* Stats init */
 	ixv_init_stats(adapter);
 
 	/* Config/Enable Link */
 	hw->mac.ops.check_link(hw, &adapter->link_speed, &adapter->link_up,
 	    FALSE);
 
 	/* And now turn on interrupts */
 	ixv_if_enable_intr(ctx);
 
 	return;
 } /* ixv_if_init */
 
 /************************************************************************
  * ixv_enable_queue
  ************************************************************************/
 static inline void
 ixv_enable_queue(struct adapter *adapter, u32 vector)
 {
 	struct ixgbe_hw *hw = &adapter->hw;
 	u32             queue = 1 << vector;
 	u32             mask;
 
 	mask = (IXGBE_EIMS_RTX_QUEUE & queue);
 	IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask);
 } /* ixv_enable_queue */
 
 /************************************************************************
  * ixv_disable_queue
  ************************************************************************/
 static inline void
 ixv_disable_queue(struct adapter *adapter, u32 vector)
 {
 	struct ixgbe_hw *hw = &adapter->hw;
 	u64             queue = (u64)(1 << vector);
 	u32             mask;
 
 	mask = (IXGBE_EIMS_RTX_QUEUE & queue);
 	IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, mask);
 } /* ixv_disable_queue */
 
 
 /************************************************************************
  * ixv_msix_que - MSI-X Queue Interrupt Service routine
  ************************************************************************/
 static int
 ixv_msix_que(void *arg)
 {
 	struct ix_rx_queue *que = arg;
 	struct adapter     *adapter = que->adapter;
 
 	ixv_disable_queue(adapter, que->msix);
 	++que->irqs;
 
 	return (FILTER_SCHEDULE_THREAD);
 } /* ixv_msix_que */
 
 /************************************************************************
  * ixv_msix_mbx
  ************************************************************************/
 static int
 ixv_msix_mbx(void *arg)
 {
 	struct adapter  *adapter = arg;
 	struct ixgbe_hw *hw = &adapter->hw;
 	u32             reg;
 
 	++adapter->link_irq;
 
 	/* First get the cause */
 	reg = IXGBE_READ_REG(hw, IXGBE_VTEICS);
 	/* Clear interrupt with write */
 	IXGBE_WRITE_REG(hw, IXGBE_VTEICR, reg);
 
 	/* Link status change */
 	if (reg & IXGBE_EICR_LSC)
 		iflib_admin_intr_deferred(adapter->ctx);
 
 	IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, IXGBE_EIMS_OTHER);
 
 	return (FILTER_HANDLED);
 } /* ixv_msix_mbx */
 
 /************************************************************************
  * ixv_media_status - Media Ioctl callback
  *
  *   Called whenever the user queries the status of
  *   the interface using ifconfig.
  ************************************************************************/
 static void
 ixv_if_media_status(if_ctx_t ctx, struct ifmediareq * ifmr)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 
 	INIT_DEBUGOUT("ixv_media_status: begin");
 
 	iflib_admin_intr_deferred(ctx);
 
 	ifmr->ifm_status = IFM_AVALID;
 	ifmr->ifm_active = IFM_ETHER;
 
 	if (!adapter->link_active)
 		return;
 
 	ifmr->ifm_status |= IFM_ACTIVE;
 
 	switch (adapter->link_speed) {
 		case IXGBE_LINK_SPEED_1GB_FULL:
 			ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
 			break;
 		case IXGBE_LINK_SPEED_10GB_FULL:
 			ifmr->ifm_active |= IFM_10G_T | IFM_FDX;
 			break;
 		case IXGBE_LINK_SPEED_100_FULL:
 			ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
 			break;
 		case IXGBE_LINK_SPEED_10_FULL:
 			ifmr->ifm_active |= IFM_10_T | IFM_FDX;
 			break;
 	}
 } /* ixv_if_media_status */
 
 /************************************************************************
  * ixv_if_media_change - Media Ioctl callback
  *
  *   Called when the user changes speed/duplex using
  *   media/mediopt option with ifconfig.
  ************************************************************************/
 static int
 ixv_if_media_change(if_ctx_t ctx)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	struct ifmedia *ifm = iflib_get_media(ctx);
 
 	INIT_DEBUGOUT("ixv_media_change: begin");
 
 	if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
 		return (EINVAL);
 
 	switch (IFM_SUBTYPE(ifm->ifm_media)) {
 	case IFM_AUTO:
 		break;
 	default:
 		device_printf(adapter->dev, "Only auto media type\n");
 		return (EINVAL);
 	}
 
 	return (0);
 } /* ixv_if_media_change */
 
 
 /************************************************************************
  * ixv_negotiate_api
  *
  *   Negotiate the Mailbox API with the PF;
  *   start with the most featured API first.
  ************************************************************************/
 static int
 ixv_negotiate_api(struct adapter *adapter)
 {
 	struct ixgbe_hw *hw = &adapter->hw;
 	int             mbx_api[] = { ixgbe_mbox_api_11,
 	                              ixgbe_mbox_api_10,
 	                              ixgbe_mbox_api_unknown };
 	int             i = 0;
 
 	while (mbx_api[i] != ixgbe_mbox_api_unknown) {
 		if (ixgbevf_negotiate_api_version(hw, mbx_api[i]) == 0)
 			return (0);
 		i++;
 	}
 
 	return (EINVAL);
 } /* ixv_negotiate_api */
 
 
 /************************************************************************
  * ixv_if_multi_set - Multicast Update
  *
  *   Called whenever multicast address list is updated.
  ************************************************************************/
 static void
 ixv_if_multi_set(if_ctx_t ctx)
 {
 	u8       mta[MAX_NUM_MULTICAST_ADDRESSES * IXGBE_ETH_LENGTH_OF_ADDRESS];
 	struct adapter     *adapter = iflib_get_softc(ctx);
 	u8                 *update_ptr;
 	struct ifmultiaddr *ifma;
 	if_t               ifp = iflib_get_ifp(ctx);
 	int                mcnt = 0;
 
 	IOCTL_DEBUGOUT("ixv_if_multi_set: begin");
 
 	CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
 		if (ifma->ifma_addr->sa_family != AF_LINK)
 			continue;
 		bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
 		    &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
 		    IXGBE_ETH_LENGTH_OF_ADDRESS);
 		mcnt++;
 	}
 
 	update_ptr = mta;
 
 	adapter->hw.mac.ops.update_mc_addr_list(&adapter->hw, update_ptr, mcnt,
 	    ixv_mc_array_itr, TRUE);
 } /* ixv_if_multi_set */
 
 /************************************************************************
  * ixv_mc_array_itr
  *
  *   An iterator function needed by the multicast shared code.
  *   It feeds the shared code routine the addresses in the
  *   array of ixv_set_multi() one by one.
  ************************************************************************/
 static u8 *
 ixv_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
 {
 	u8 *addr = *update_ptr;
 	u8 *newptr;
 
 	*vmdq = 0;
 
 	newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
 	*update_ptr = newptr;
 
 	return addr;
 } /* ixv_mc_array_itr */
 
 /************************************************************************
  * ixv_if_local_timer - Timer routine
  *
  *   Checks for link status, updates statistics,
  *   and runs the watchdog check.
  ************************************************************************/
 static void
 ixv_if_local_timer(if_ctx_t ctx, uint16_t qid)
 {
 	if (qid != 0)
 		return;
 
 	/* Fire off the adminq task */
 	iflib_admin_intr_deferred(ctx);
 } /* ixv_if_local_timer */
 
 /************************************************************************
  * ixv_if_update_admin_status - Update OS on link state
  *
  * Note: Only updates the OS on the cached link state.
  *       The real check of the hardware only happens with
  *       a link interrupt.
  ************************************************************************/
 static void
 ixv_if_update_admin_status(if_ctx_t ctx)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	device_t       dev = iflib_get_dev(ctx);
 	s32            status;
 
 	adapter->hw.mac.get_link_status = TRUE;
 
 	status = ixgbe_check_link(&adapter->hw, &adapter->link_speed,
 	    &adapter->link_up, FALSE);
 
 	if (status != IXGBE_SUCCESS && adapter->hw.adapter_stopped == FALSE) {
 		/* Mailbox's Clear To Send status is lost or timeout occurred.
 		 * We need reinitialization. */
 		iflib_get_ifp(ctx)->if_init(ctx);
 	}
 
 	if (adapter->link_up) {
 		if (adapter->link_active == FALSE) {
 			if (bootverbose)
 				device_printf(dev, "Link is up %d Gbps %s \n",
 				    ((adapter->link_speed == 128) ? 10 : 1),
 				    "Full Duplex");
 			adapter->link_active = TRUE;
 			iflib_link_state_change(ctx, LINK_STATE_UP,
 			    IF_Gbps(10));
 		}
 	} else { /* Link down */
 		if (adapter->link_active == TRUE) {
 			if (bootverbose)
 				device_printf(dev, "Link is Down\n");
 			iflib_link_state_change(ctx, LINK_STATE_DOWN,  0);
 			adapter->link_active = FALSE;
 		}
 	}
 
 	/* Stats Update */
 	ixv_update_stats(adapter);
 } /* ixv_if_update_admin_status */
 
 
 /************************************************************************
  * ixv_if_stop - Stop the hardware
  *
  *   Disables all traffic on the adapter by issuing a
  *   global reset on the MAC and deallocates TX/RX buffers.
  ************************************************************************/
 static void
 ixv_if_stop(if_ctx_t ctx)
 {
 	struct adapter  *adapter = iflib_get_softc(ctx);
 	struct ixgbe_hw *hw = &adapter->hw;
 
 	INIT_DEBUGOUT("ixv_stop: begin\n");
 
 	ixv_if_disable_intr(ctx);
 
 	hw->mac.ops.reset_hw(hw);
 	adapter->hw.adapter_stopped = FALSE;
 	hw->mac.ops.stop_adapter(hw);
 
 	/* Update the stack */
 	adapter->link_up = FALSE;
 	ixv_if_update_admin_status(ctx);
 
 	/* reprogram the RAR[0] in case user changed it. */
 	hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
 } /* ixv_if_stop */
 
 
 /************************************************************************
  * ixv_identify_hardware - Determine hardware revision.
  ************************************************************************/
 static void
 ixv_identify_hardware(if_ctx_t ctx)
 {
 	struct adapter  *adapter = iflib_get_softc(ctx);
 	device_t        dev = iflib_get_dev(ctx);
 	struct ixgbe_hw *hw = &adapter->hw;
 
 	/* Save off the information about this board */
 	hw->vendor_id = pci_get_vendor(dev);
 	hw->device_id = pci_get_device(dev);
 	hw->revision_id = pci_get_revid(dev);
 	hw->subsystem_vendor_id = pci_get_subvendor(dev);
 	hw->subsystem_device_id = pci_get_subdevice(dev);
 
 	/* A subset of set_mac_type */
 	switch (hw->device_id) {
 	case IXGBE_DEV_ID_82599_VF:
 		hw->mac.type = ixgbe_mac_82599_vf;
 		break;
 	case IXGBE_DEV_ID_X540_VF:
 		hw->mac.type = ixgbe_mac_X540_vf;
 		break;
 	case IXGBE_DEV_ID_X550_VF:
 		hw->mac.type = ixgbe_mac_X550_vf;
 		break;
 	case IXGBE_DEV_ID_X550EM_X_VF:
 		hw->mac.type = ixgbe_mac_X550EM_x_vf;
 		break;
 	case IXGBE_DEV_ID_X550EM_A_VF:
 		hw->mac.type = ixgbe_mac_X550EM_a_vf;
 		break;
 	default:
 		device_printf(dev, "unknown mac type\n");
 		hw->mac.type = ixgbe_mac_unknown;
 		break;
 	}
 } /* ixv_identify_hardware */
 
 /************************************************************************
  * ixv_if_msix_intr_assign - Setup MSI-X Interrupt resources and handlers
  ************************************************************************/
 static int
 ixv_if_msix_intr_assign(if_ctx_t ctx, int msix)
 {
 	struct adapter     *adapter = iflib_get_softc(ctx);
 	device_t           dev = iflib_get_dev(ctx);
 	struct ix_rx_queue *rx_que = adapter->rx_queues;
 	struct ix_tx_queue *tx_que;
 	int                error, rid, vector = 0;
 	char               buf[16];
 
 	for (int i = 0; i < adapter->num_rx_queues; i++, vector++, rx_que++) {
 		rid = vector + 1;
 
 		snprintf(buf, sizeof(buf), "rxq%d", i);
 		error = iflib_irq_alloc_generic(ctx, &rx_que->que_irq, rid,
 		    IFLIB_INTR_RX, ixv_msix_que, rx_que, rx_que->rxr.me, buf);
 
 		if (error) {
 			device_printf(iflib_get_dev(ctx),
 			    "Failed to allocate que int %d err: %d", i, error);
 			adapter->num_rx_queues = i + 1;
 			goto fail;
 		}
 
 		rx_que->msix = vector;
 		adapter->active_queues |= (u64)(1 << rx_que->msix);
 
 	}
 
 	for (int i = 0; i < adapter->num_tx_queues; i++) {
 		snprintf(buf, sizeof(buf), "txq%d", i);
 		tx_que = &adapter->tx_queues[i];
 		tx_que->msix = i % adapter->num_rx_queues;
 		iflib_softirq_alloc_generic(ctx,
 		    &adapter->rx_queues[tx_que->msix].que_irq,
 		    IFLIB_INTR_TX, tx_que, tx_que->txr.me, buf);
 	}
 	rid = vector + 1;
 	error = iflib_irq_alloc_generic(ctx, &adapter->irq, rid,
 	    IFLIB_INTR_ADMIN, ixv_msix_mbx, adapter, 0, "aq");
 	if (error) {
 		device_printf(iflib_get_dev(ctx),
 		    "Failed to register admin handler");
 		return (error);
 	}
 
 	adapter->vector = vector;
 	/*
 	 * Due to a broken design QEMU will fail to properly
 	 * enable the guest for MSIX unless the vectors in
 	 * the table are all set up, so we must rewrite the
 	 * ENABLE in the MSIX control register again at this
 	 * point to cause it to successfully initialize us.
 	 */
 	if (adapter->hw.mac.type == ixgbe_mac_82599_vf) {
 		int msix_ctrl;
 		pci_find_cap(dev, PCIY_MSIX, &rid);
 		rid += PCIR_MSIX_CTRL;
 		msix_ctrl = pci_read_config(dev, rid, 2);
 		msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE;
 		pci_write_config(dev, rid, msix_ctrl, 2);
 	}
 
 	return (0);
 
 fail:
 	iflib_irq_free(ctx, &adapter->irq);
 	rx_que = adapter->rx_queues;
 	for (int i = 0; i < adapter->num_rx_queues; i++, rx_que++)
 		iflib_irq_free(ctx, &rx_que->que_irq);
 
 	return (error);
 } /* ixv_if_msix_intr_assign */
 
 /************************************************************************
  * ixv_allocate_pci_resources
  ************************************************************************/
 static int
 ixv_allocate_pci_resources(if_ctx_t ctx)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	device_t       dev = iflib_get_dev(ctx);
 	int            rid;
 
 	rid = PCIR_BAR(0);
 	adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 	    RF_ACTIVE);
 
 	if (!(adapter->pci_mem)) {
 		device_printf(dev, "Unable to allocate bus resource: memory\n");
 		return (ENXIO);
 	}
 
 	adapter->osdep.mem_bus_space_tag = rman_get_bustag(adapter->pci_mem);
 	adapter->osdep.mem_bus_space_handle =
 	    rman_get_bushandle(adapter->pci_mem);
 	adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle;
 
 	return (0);
 } /* ixv_allocate_pci_resources */
 
 /************************************************************************
  * ixv_free_pci_resources
  ************************************************************************/
 static void
 ixv_free_pci_resources(if_ctx_t ctx)
 {
 	struct adapter     *adapter = iflib_get_softc(ctx);
 	struct ix_rx_queue *que = adapter->rx_queues;
 	device_t           dev = iflib_get_dev(ctx);
 
 	/* Release all MSI-X queue resources */
 	if (adapter->intr_type == IFLIB_INTR_MSIX)
 		iflib_irq_free(ctx, &adapter->irq);
 
 	if (que != NULL) {
 		for (int i = 0; i < adapter->num_rx_queues; i++, que++) {
 			iflib_irq_free(ctx, &que->que_irq);
 		}
 	}
 
 	if (adapter->pci_mem != NULL)
 		bus_release_resource(dev, SYS_RES_MEMORY,
 		    rman_get_rid(adapter->pci_mem), adapter->pci_mem);
 } /* ixv_free_pci_resources */
 
 /************************************************************************
  * ixv_setup_interface
  *
  *   Setup networking device structure and register an interface.
  ************************************************************************/
 static int
 ixv_setup_interface(if_ctx_t ctx)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	if_softc_ctx_t scctx = adapter->shared;
 	struct ifnet   *ifp = iflib_get_ifp(ctx);
 
 	INIT_DEBUGOUT("ixv_setup_interface: begin");
 
 	if_setbaudrate(ifp, IF_Gbps(10));
 	ifp->if_snd.ifq_maxlen = scctx->isc_ntxd[0] - 2;
 
 
 	adapter->max_frame_size = ifp->if_mtu + IXGBE_MTU_HDR;
 	ifmedia_add(adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
 	ifmedia_set(adapter->media, IFM_ETHER | IFM_AUTO);
 
 	return 0;
 } /* ixv_setup_interface */
 
 /************************************************************************
  * ixv_if_get_counter
  ************************************************************************/
 static uint64_t
 ixv_if_get_counter(if_ctx_t ctx, ift_counter cnt)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	if_t           ifp = iflib_get_ifp(ctx);
 
 	switch (cnt) {
 	case IFCOUNTER_IPACKETS:
 		return (adapter->ipackets);
 	case IFCOUNTER_OPACKETS:
 		return (adapter->opackets);
 	case IFCOUNTER_IBYTES:
 		return (adapter->ibytes);
 	case IFCOUNTER_OBYTES:
 		return (adapter->obytes);
 	case IFCOUNTER_IMCASTS:
 		return (adapter->imcasts);
 	default:
 		return (if_get_counter_default(ifp, cnt));
 	}
 } /* ixv_if_get_counter */
 
 /************************************************************************
  * ixv_initialize_transmit_units - Enable transmit unit.
  ************************************************************************/
 static void
 ixv_initialize_transmit_units(if_ctx_t ctx)
 {
 	struct adapter     *adapter = iflib_get_softc(ctx);
 	struct ixgbe_hw    *hw = &adapter->hw;
 	if_softc_ctx_t     scctx = adapter->shared;
 	struct ix_tx_queue *que = adapter->tx_queues;
 	int                i;
 
 	for (i = 0; i < adapter->num_tx_queues; i++, que++) {
 		struct tx_ring *txr = &que->txr;
 		u64            tdba = txr->tx_paddr;
 		u32            txctrl, txdctl;
 		int            j = txr->me;
 
 		/* Set WTHRESH to 8, burst writeback */
 		txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j));
 		txdctl |= (8 << 16);
 		IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl);
 
 		/* Set the HW Tx Head and Tail indices */
 		IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDH(j), 0);
 		IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDT(j), 0);
 
 		/* Set Tx Tail register */
 		txr->tail = IXGBE_VFTDT(j);
 
 		txr->tx_rs_cidx = txr->tx_rs_pidx;
 		/* Initialize the last processed descriptor to be the end of
 		 * the ring, rather than the start, so that we avoid an
 		 * off-by-one error when calculating how many descriptors are
 		 * done in the credits_update function.
 		 */
 		txr->tx_cidx_processed = scctx->isc_ntxd[0] - 1;
 		for (int k = 0; k < scctx->isc_ntxd[0]; k++)
 			txr->tx_rsq[k] = QIDX_INVALID;
 
 		/* Set Ring parameters */
 		IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(j),
 		    (tdba & 0x00000000ffffffffULL));
 		IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(j), (tdba >> 32));
 		IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(j),
 		    scctx->isc_ntxd[0] * sizeof(struct ixgbe_legacy_tx_desc));
 		txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(j));
 		txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
 		IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(j), txctrl);
 
 		/* Now enable */
 		txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j));
 		txdctl |= IXGBE_TXDCTL_ENABLE;
 		IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl);
 	}
 
 	return;
 } /* ixv_initialize_transmit_units */
 
 /************************************************************************
  * ixv_initialize_rss_mapping
  ************************************************************************/
 static void
 ixv_initialize_rss_mapping(struct adapter *adapter)
 {
 	struct ixgbe_hw *hw = &adapter->hw;
 	u32             reta = 0, mrqc, rss_key[10];
 	int             queue_id;
 	int             i, j;
 	u32             rss_hash_config;
 
 	if (adapter->feat_en & IXGBE_FEATURE_RSS) {
 		/* Fetch the configured RSS key */
 		rss_getkey((uint8_t *)&rss_key);
 	} else {
 		/* set up random bits */
 		arc4rand(&rss_key, sizeof(rss_key), 0);
 	}
 
 	/* Now fill out hash function seeds */
 	for (i = 0; i < 10; i++)
 		IXGBE_WRITE_REG(hw, IXGBE_VFRSSRK(i), rss_key[i]);
 
 	/* Set up the redirection table */
 	for (i = 0, j = 0; i < 64; i++, j++) {
 		if (j == adapter->num_rx_queues)
 			j = 0;
 
 		if (adapter->feat_en & IXGBE_FEATURE_RSS) {
 			/*
 			 * Fetch the RSS bucket id for the given indirection
 			 * entry. Cap it at the number of configured buckets
 			 * (which is num_rx_queues.)
 			 */
 			queue_id = rss_get_indirection_to_bucket(i);
 			queue_id = queue_id % adapter->num_rx_queues;
 		} else
 			queue_id = j;
 
 		/*
 		 * The low 8 bits are for hash value (n+0);
 		 * The next 8 bits are for hash value (n+1), etc.
 		 */
 		reta >>= 8;
 		reta |= ((uint32_t)queue_id) << 24;
 		if ((i & 3) == 3) {
 			IXGBE_WRITE_REG(hw, IXGBE_VFRETA(i >> 2), reta);
 			reta = 0;
 		}
 	}
 
 	/* Perform hash on these packet types */
 	if (adapter->feat_en & IXGBE_FEATURE_RSS)
 		rss_hash_config = rss_gethashconfig();
 	else {
 		/*
 		 * Disable UDP - IP fragments aren't currently being handled
 		 * and so we end up with a mix of 2-tuple and 4-tuple
 		 * traffic.
 		 */
 		rss_hash_config = RSS_HASHTYPE_RSS_IPV4
 		                | RSS_HASHTYPE_RSS_TCP_IPV4
 		                | RSS_HASHTYPE_RSS_IPV6
 		                | RSS_HASHTYPE_RSS_TCP_IPV6;
 	}
 
 	mrqc = IXGBE_MRQC_RSSEN;
 	if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4)
 		mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4;
 	if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4)
 		mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP;
 	if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6)
 		mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6;
 	if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6)
 		mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP;
 	if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX)
 		device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_IPV6_EX defined, but not supported\n",
 		    __func__);
 	if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6_EX)
 		device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_TCP_IPV6_EX defined, but not supported\n",
 		    __func__);
 	if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4)
 		mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP;
 	if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6)
 		mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP;
 	if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6_EX)
 		device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_UDP_IPV6_EX defined, but not supported\n",
 		    __func__);
 	IXGBE_WRITE_REG(hw, IXGBE_VFMRQC, mrqc);
 } /* ixv_initialize_rss_mapping */
 
 
 /************************************************************************
  * ixv_initialize_receive_units - Setup receive registers and features.
  ************************************************************************/
 static void
 ixv_initialize_receive_units(if_ctx_t ctx)
 {
 	struct adapter     *adapter = iflib_get_softc(ctx);
 	if_softc_ctx_t     scctx;
 	struct ixgbe_hw    *hw = &adapter->hw;
 	struct ifnet       *ifp = iflib_get_ifp(ctx);
 	struct ix_rx_queue *que = adapter->rx_queues;
 	u32                bufsz, psrtype;
 
 	if (ifp->if_mtu > ETHERMTU)
 		bufsz = 4096 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
 	else
 		bufsz = 2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
 
 	psrtype = IXGBE_PSRTYPE_TCPHDR
 	        | IXGBE_PSRTYPE_UDPHDR
 	        | IXGBE_PSRTYPE_IPV4HDR
 	        | IXGBE_PSRTYPE_IPV6HDR
 	        | IXGBE_PSRTYPE_L2HDR;
 
 	if (adapter->num_rx_queues > 1)
 		psrtype |= 1 << 29;
 
 	IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype);
 
 	/* Tell PF our max_frame size */
 	if (ixgbevf_rlpml_set_vf(hw, adapter->max_frame_size) != 0) {
 		device_printf(adapter->dev, "There is a problem with the PF setup.  It is likely the receive unit for this VF will not function correctly.\n");
 	}
 	scctx = adapter->shared;
 
 	for (int i = 0; i < adapter->num_rx_queues; i++, que++) {
 		struct rx_ring *rxr = &que->rxr;
 		u64            rdba = rxr->rx_paddr;
 		u32            reg, rxdctl;
 		int            j = rxr->me;
 
 		/* Disable the queue */
 		rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j));
 		rxdctl &= ~IXGBE_RXDCTL_ENABLE;
 		IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl);
 		for (int k = 0; k < 10; k++) {
 			if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) &
 			    IXGBE_RXDCTL_ENABLE)
 				msec_delay(1);
 			else
 				break;
 		}
 		wmb();
 		/* Setup the Base and Length of the Rx Descriptor Ring */
 		IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(j),
 		    (rdba & 0x00000000ffffffffULL));
 		IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(j), (rdba >> 32));
 		IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(j),
 		    scctx->isc_nrxd[0] * sizeof(union ixgbe_adv_rx_desc));
 
 		/* Reset the ring indices */
 		IXGBE_WRITE_REG(hw, IXGBE_VFRDH(rxr->me), 0);
 		IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 0);
 
 		/* Set up the SRRCTL register */
 		reg = IXGBE_READ_REG(hw, IXGBE_VFSRRCTL(j));
 		reg &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
 		reg &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
 		reg |= bufsz;
 		reg |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
 		IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(j), reg);
 
 		/* Capture Rx Tail index */
 		rxr->tail = IXGBE_VFRDT(rxr->me);
 
 		/* Do the queue enabling last */
 		rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME;
 		IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl);
 		for (int l = 0; l < 10; l++) {
 			if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) &
 			    IXGBE_RXDCTL_ENABLE)
 				break;
 			msec_delay(1);
 		}
 		wmb();
 
 		/* Set the Tail Pointer */
 #ifdef DEV_NETMAP
 		/*
 		 * In netmap mode, we must preserve the buffers made
 		 * available to userspace before the if_init()
 		 * (this is true by default on the TX side, because
 		 * init makes all buffers available to userspace).
 		 *
 		 * netmap_reset() and the device specific routines
 		 * (e.g. ixgbe_setup_receive_rings()) map these
 		 * buffers at the end of the NIC ring, so here we
 		 * must set the RDT (tail) register to make sure
 		 * they are not overwritten.
 		 *
 		 * In this driver the NIC ring starts at RDH = 0,
 		 * RDT points to the last slot available for reception (?),
 		 * so RDT = num_rx_desc - 1 means the whole ring is available.
 		 */
 		if (ifp->if_capenable & IFCAP_NETMAP) {
 			struct netmap_adapter *na = NA(ifp);
 			struct netmap_kring *kring = na->rx_rings[j];
 			int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring);
 
 			IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), t);
 		} else
 #endif /* DEV_NETMAP */
 			IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me),
 			    scctx->isc_nrxd[0] - 1);
 	}
 
 	ixv_initialize_rss_mapping(adapter);
 } /* ixv_initialize_receive_units */
 
 /************************************************************************
  * ixv_setup_vlan_support
  ************************************************************************/
 static void
 ixv_setup_vlan_support(if_ctx_t ctx)
 {
 	struct ifnet	*ifp = iflib_get_ifp(ctx);
 	struct adapter  *adapter = iflib_get_softc(ctx);
 	struct ixgbe_hw *hw = &adapter->hw;
 	u32             ctrl, vid, vfta, retry;
 
 	/*
 	 * We get here thru if_init, meaning
 	 * a soft reset, this has already cleared
 	 * the VFTA and other state, so if there
 	 * have been no vlan's registered do nothing.
 	 */
 	if (adapter->num_vlans == 0)
 		return;
 
 	if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) {
 		/* Enable the queues */
 		for (int i = 0; i < adapter->num_rx_queues; i++) {
 			ctrl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i));
 			ctrl |= IXGBE_RXDCTL_VME;
 			IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(i), ctrl);
 			/*
 			 * Let Rx path know that it needs to store VLAN tag
 			 * as part of extra mbuf info.
 			 */
 			adapter->rx_queues[i].rxr.vtag_strip = TRUE;
 		}
 	}
 
 	/*
 	 * If filtering VLAN tags is disabled,
 	 * there is no need to fill VLAN Filter Table Array (VFTA).
 	 */
 	if ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0)
 		return;
 
 	/*
 	 * A soft reset zero's out the VFTA, so
 	 * we need to repopulate it now.
 	 */
 	for (int i = 0; i < IXGBE_VFTA_SIZE; i++) {
 		if (ixv_shadow_vfta[i] == 0)
 			continue;
 		vfta = ixv_shadow_vfta[i];
 		/*
 		 * Reconstruct the vlan id's
 		 * based on the bits set in each
 		 * of the array ints.
 		 */
 		for (int j = 0; j < 32; j++) {
 			retry = 0;
 			if ((vfta & (1 << j)) == 0)
 				continue;
 			vid = (i * 32) + j;
 			/* Call the shared code mailbox routine */
 			while (hw->mac.ops.set_vfta(hw, vid, 0, TRUE, FALSE)) {
 				if (++retry > 5)
 					break;
 			}
 		}
 	}
 } /* ixv_setup_vlan_support */
 
 /************************************************************************
  * ixv_if_register_vlan
  *
  *   Run via a vlan config EVENT, it enables us to use the
  *   HW Filter table since we can get the vlan id. This just
  *   creates the entry in the soft version of the VFTA, init
  *   will repopulate the real table.
  ************************************************************************/
 static void
 ixv_if_register_vlan(if_ctx_t ctx, u16 vtag)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	u16            index, bit;
 
 	index = (vtag >> 5) & 0x7F;
 	bit = vtag & 0x1F;
 	ixv_shadow_vfta[index] |= (1 << bit);
 	++adapter->num_vlans;
 } /* ixv_if_register_vlan */
 
 /************************************************************************
  * ixv_if_unregister_vlan
  *
  *   Run via a vlan unconfig EVENT, remove our entry
  *   in the soft vfta.
  ************************************************************************/
 static void
 ixv_if_unregister_vlan(if_ctx_t ctx, u16 vtag)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	u16            index, bit;
 
 	index = (vtag >> 5) & 0x7F;
 	bit = vtag & 0x1F;
 	ixv_shadow_vfta[index] &= ~(1 << bit);
 	--adapter->num_vlans;
 } /* ixv_if_unregister_vlan */
 
 /************************************************************************
  * ixv_if_enable_intr
  ************************************************************************/
 static void
 ixv_if_enable_intr(if_ctx_t ctx)
 {
 	struct adapter  *adapter = iflib_get_softc(ctx);
 	struct ixgbe_hw *hw = &adapter->hw;
 	struct ix_rx_queue *que = adapter->rx_queues;
 	u32             mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
 
 	IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask);
 
 	mask = IXGBE_EIMS_ENABLE_MASK;
 	mask &= ~(IXGBE_EIMS_OTHER | IXGBE_EIMS_LSC);
 	IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, mask);
 
 	for (int i = 0; i < adapter->num_rx_queues; i++, que++)
 		ixv_enable_queue(adapter, que->msix);
 
 	IXGBE_WRITE_FLUSH(hw);
 } /* ixv_if_enable_intr */
 
 /************************************************************************
  * ixv_if_disable_intr
  ************************************************************************/
 static void
 ixv_if_disable_intr(if_ctx_t ctx)
 {
 	struct adapter *adapter = iflib_get_softc(ctx);
 	IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIAC, 0);
 	IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIMC, ~0);
 	IXGBE_WRITE_FLUSH(&adapter->hw);
 } /* ixv_if_disable_intr */
 
 /************************************************************************
  * ixv_if_rx_queue_intr_enable
  ************************************************************************/
 static int
 ixv_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid)
 {
 	struct adapter	*adapter = iflib_get_softc(ctx);
 	struct ix_rx_queue *que = &adapter->rx_queues[rxqid];
 
 	ixv_enable_queue(adapter, que->rxr.me);
 
 	return (0);
 } /* ixv_if_rx_queue_intr_enable */
 
 /************************************************************************
  * ixv_set_ivar
  *
  *   Setup the correct IVAR register for a particular MSI-X interrupt
  *    - entry is the register array entry
  *    - vector is the MSI-X vector for this queue
  *    - type is RX/TX/MISC
  ************************************************************************/
 static void
 ixv_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type)
 {
 	struct ixgbe_hw *hw = &adapter->hw;
 	u32             ivar, index;
 
 	vector |= IXGBE_IVAR_ALLOC_VAL;
 
 	if (type == -1) { /* MISC IVAR */
 		ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC);
 		ivar &= ~0xFF;
 		ivar |= vector;
 		IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar);
 	} else {          /* RX/TX IVARS */
 		index = (16 * (entry & 1)) + (8 * type);
 		ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(entry >> 1));
 		ivar &= ~(0xFF << index);
 		ivar |= (vector << index);
 		IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(entry >> 1), ivar);
 	}
 } /* ixv_set_ivar */
 
 /************************************************************************
  * ixv_configure_ivars
  ************************************************************************/
 static void
 ixv_configure_ivars(struct adapter *adapter)
 {
 	struct ix_rx_queue *que = adapter->rx_queues;
 
 	MPASS(adapter->num_rx_queues == adapter->num_tx_queues);
 
 	for (int i = 0; i < adapter->num_rx_queues; i++, que++) {
 		/* First the RX queue entry */
 		ixv_set_ivar(adapter, i, que->msix, 0);
 		/* ... and the TX */
 		ixv_set_ivar(adapter, i, que->msix, 1);
 		/* Set an initial value in EITR */
 		IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEITR(que->msix),
 		    IXGBE_EITR_DEFAULT);
 	}
 
 	/* For the mailbox interrupt */
 	ixv_set_ivar(adapter, 1, adapter->vector, -1);
 } /* ixv_configure_ivars */
 
 /************************************************************************
  * ixv_save_stats
  *
  *   The VF stats registers never have a truly virgin
  *   starting point, so this routine tries to make an
  *   artificial one, marking ground zero on attach as
  *   it were.
  ************************************************************************/
 static void
 ixv_save_stats(struct adapter *adapter)
 {
 	if (adapter->stats.vf.vfgprc || adapter->stats.vf.vfgptc) {
 		adapter->stats.vf.saved_reset_vfgprc +=
 		    adapter->stats.vf.vfgprc - adapter->stats.vf.base_vfgprc;
 		adapter->stats.vf.saved_reset_vfgptc +=
 		    adapter->stats.vf.vfgptc - adapter->stats.vf.base_vfgptc;
 		adapter->stats.vf.saved_reset_vfgorc +=
 		    adapter->stats.vf.vfgorc - adapter->stats.vf.base_vfgorc;
 		adapter->stats.vf.saved_reset_vfgotc +=
 		    adapter->stats.vf.vfgotc - adapter->stats.vf.base_vfgotc;
 		adapter->stats.vf.saved_reset_vfmprc +=
 		    adapter->stats.vf.vfmprc - adapter->stats.vf.base_vfmprc;
 	}
 } /* ixv_save_stats */
 
 /************************************************************************
  * ixv_init_stats
  ************************************************************************/
 static void
 ixv_init_stats(struct adapter *adapter)
 {
 	struct ixgbe_hw *hw = &adapter->hw;
 
 	adapter->stats.vf.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC);
 	adapter->stats.vf.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB);
 	adapter->stats.vf.last_vfgorc |=
 	    (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32);
 
 	adapter->stats.vf.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC);
 	adapter->stats.vf.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB);
 	adapter->stats.vf.last_vfgotc |=
 	    (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32);
 
 	adapter->stats.vf.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC);
 
 	adapter->stats.vf.base_vfgprc = adapter->stats.vf.last_vfgprc;
 	adapter->stats.vf.base_vfgorc = adapter->stats.vf.last_vfgorc;
 	adapter->stats.vf.base_vfgptc = adapter->stats.vf.last_vfgptc;
 	adapter->stats.vf.base_vfgotc = adapter->stats.vf.last_vfgotc;
 	adapter->stats.vf.base_vfmprc = adapter->stats.vf.last_vfmprc;
 } /* ixv_init_stats */
 
 #define UPDATE_STAT_32(reg, last, count)                \
 {                                                       \
 	u32 current = IXGBE_READ_REG(hw, reg);          \
 	if (current < last)                             \
 		count += 0x100000000LL;                 \
 	last = current;                                 \
 	count &= 0xFFFFFFFF00000000LL;                  \
 	count |= current;                               \
 }
 
 #define UPDATE_STAT_36(lsb, msb, last, count)           \
 {                                                       \
 	u64 cur_lsb = IXGBE_READ_REG(hw, lsb);          \
 	u64 cur_msb = IXGBE_READ_REG(hw, msb);          \
 	u64 current = ((cur_msb << 32) | cur_lsb);      \
 	if (current < last)                             \
 		count += 0x1000000000LL;                \
 	last = current;                                 \
 	count &= 0xFFFFFFF000000000LL;                  \
 	count |= current;                               \
 }
 
 /************************************************************************
  * ixv_update_stats - Update the board statistics counters.
  ************************************************************************/
 void
 ixv_update_stats(struct adapter *adapter)
 {
 	struct ixgbe_hw *hw = &adapter->hw;
 	struct ixgbevf_hw_stats *stats = &adapter->stats.vf;
 
 	UPDATE_STAT_32(IXGBE_VFGPRC, adapter->stats.vf.last_vfgprc,
 	    adapter->stats.vf.vfgprc);
 	UPDATE_STAT_32(IXGBE_VFGPTC, adapter->stats.vf.last_vfgptc,
 	    adapter->stats.vf.vfgptc);
 	UPDATE_STAT_36(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
 	    adapter->stats.vf.last_vfgorc, adapter->stats.vf.vfgorc);
 	UPDATE_STAT_36(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
 	    adapter->stats.vf.last_vfgotc, adapter->stats.vf.vfgotc);
 	UPDATE_STAT_32(IXGBE_VFMPRC, adapter->stats.vf.last_vfmprc,
 	    adapter->stats.vf.vfmprc);
 
 	/* Fill out the OS statistics structure */
 	IXGBE_SET_IPACKETS(adapter, stats->vfgprc);
 	IXGBE_SET_OPACKETS(adapter, stats->vfgptc);
 	IXGBE_SET_IBYTES(adapter, stats->vfgorc);
 	IXGBE_SET_OBYTES(adapter, stats->vfgotc);
 	IXGBE_SET_IMCASTS(adapter, stats->vfmprc);
 } /* ixv_update_stats */
 
 /************************************************************************
  * ixv_add_stats_sysctls - Add statistic sysctls for the VF.
  ************************************************************************/
 static void
 ixv_add_stats_sysctls(struct adapter *adapter)
 {
 	device_t                dev = adapter->dev;
 	struct ix_tx_queue      *tx_que = adapter->tx_queues;
 	struct ix_rx_queue      *rx_que = adapter->rx_queues;
 	struct sysctl_ctx_list  *ctx = device_get_sysctl_ctx(dev);
 	struct sysctl_oid       *tree = device_get_sysctl_tree(dev);
 	struct sysctl_oid_list  *child = SYSCTL_CHILDREN(tree);
 	struct ixgbevf_hw_stats *stats = &adapter->stats.vf;
 	struct sysctl_oid       *stat_node, *queue_node;
 	struct sysctl_oid_list  *stat_list, *queue_list;
 
 #define QUEUE_NAME_LEN 32
 	char                    namebuf[QUEUE_NAME_LEN];
 
 	/* Driver Statistics */
 	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
 	    CTLFLAG_RD, &adapter->watchdog_events, "Watchdog timeouts");
 	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
 	    CTLFLAG_RD, &adapter->link_irq, "Link MSI-X IRQ Handled");
 
 	for (int i = 0; i < adapter->num_tx_queues; i++, tx_que++) {
 		struct tx_ring *txr = &tx_que->txr;
 		snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
 		queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
 		    CTLFLAG_RD, NULL, "Queue Name");
 		queue_list = SYSCTL_CHILDREN(queue_node);
 
 		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tso_tx",
 		    CTLFLAG_RD, &(txr->tso_tx), "TSO Packets");
 		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
 		    CTLFLAG_RD, &(txr->total_packets), "TX Packets");
 	}
 
 	for (int i = 0; i < adapter->num_rx_queues; i++, rx_que++) {
 		struct rx_ring *rxr = &rx_que->rxr;
 		snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
 		queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
 		    CTLFLAG_RD, NULL, "Queue Name");
 		queue_list = SYSCTL_CHILDREN(queue_node);
 
 		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
 		    CTLFLAG_RD, &(rx_que->irqs), "IRQs on queue");
 		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
 		    CTLFLAG_RD, &(rxr->rx_packets), "RX packets");
 		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
 		    CTLFLAG_RD, &(rxr->rx_bytes), "RX bytes");
 		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_discarded",
 		    CTLFLAG_RD, &(rxr->rx_discarded), "Discarded RX packets");
 	}
 
 	stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac",
 	    CTLFLAG_RD, NULL, "VF Statistics (read from HW registers)");
 	stat_list = SYSCTL_CHILDREN(stat_node);
 
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd",
 	    CTLFLAG_RD, &stats->vfgprc, "Good Packets Received");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd",
 	    CTLFLAG_RD, &stats->vfgorc, "Good Octets Received");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd",
 	    CTLFLAG_RD, &stats->vfmprc, "Multicast Packets Received");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
 	    CTLFLAG_RD, &stats->vfgptc, "Good Packets Transmitted");
 	SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
 	    CTLFLAG_RD, &stats->vfgotc, "Good Octets Transmitted");
 } /* ixv_add_stats_sysctls */
 
 /************************************************************************
  * ixv_print_debug_info
  *
  *   Called only when em_display_debug_stats is enabled.
  *   Provides a way to take a look at important statistics
  *   maintained by the driver and hardware.
  ************************************************************************/
 static void
 ixv_print_debug_info(struct adapter *adapter)
 {
 	device_t        dev = adapter->dev;
 	struct ixgbe_hw *hw = &adapter->hw;
 
 	device_printf(dev, "Error Byte Count = %u \n",
 	    IXGBE_READ_REG(hw, IXGBE_ERRBC));
 
 	device_printf(dev, "MBX IRQ Handled: %lu\n", (long)adapter->link_irq);
 } /* ixv_print_debug_info */
 
 /************************************************************************
  * ixv_sysctl_debug
  ************************************************************************/
 static int
 ixv_sysctl_debug(SYSCTL_HANDLER_ARGS)
 {
 	struct adapter *adapter;
 	int            error, result;
 
 	result = -1;
 	error = sysctl_handle_int(oidp, &result, 0, req);
 
 	if (error || !req->newptr)
 		return (error);
 
 	if (result == 1) {
 		adapter = (struct adapter *)arg1;
 		ixv_print_debug_info(adapter);
 	}
 
 	return error;
 } /* ixv_sysctl_debug */
 
 /************************************************************************
  * ixv_init_device_features
  ************************************************************************/
 static void
 ixv_init_device_features(struct adapter *adapter)
 {
 	adapter->feat_cap = IXGBE_FEATURE_NETMAP
 	                  | IXGBE_FEATURE_VF
 	                  | IXGBE_FEATURE_RSS
 	                  | IXGBE_FEATURE_LEGACY_TX;
 
 	/* A tad short on feature flags for VFs, atm. */
 	switch (adapter->hw.mac.type) {
 	case ixgbe_mac_82599_vf:
 		break;
 	case ixgbe_mac_X540_vf:
 		break;
 	case ixgbe_mac_X550_vf:
 	case ixgbe_mac_X550EM_x_vf:
 	case ixgbe_mac_X550EM_a_vf:
 		adapter->feat_cap |= IXGBE_FEATURE_NEEDS_CTXD;
 		break;
 	default:
 		break;
 	}
 
 	/* Enabled by default... */
 	/* Is a virtual function (VF) */
 	if (adapter->feat_cap & IXGBE_FEATURE_VF)
 		adapter->feat_en |= IXGBE_FEATURE_VF;
 	/* Netmap */
 	if (adapter->feat_cap & IXGBE_FEATURE_NETMAP)
 		adapter->feat_en |= IXGBE_FEATURE_NETMAP;
 	/* Receive-Side Scaling (RSS) */
 	if (adapter->feat_cap & IXGBE_FEATURE_RSS)
 		adapter->feat_en |= IXGBE_FEATURE_RSS;
 	/* Needs advanced context descriptor regardless of offloads req'd */
 	if (adapter->feat_cap & IXGBE_FEATURE_NEEDS_CTXD)
 		adapter->feat_en |= IXGBE_FEATURE_NEEDS_CTXD;
 } /* ixv_init_device_features */
 
Index: head/sys/dev/ixl/if_iavf.c
===================================================================
--- head/sys/dev/ixl/if_iavf.c	(revision 345304)
+++ head/sys/dev/ixl/if_iavf.c	(revision 345305)
@@ -1,2435 +1,2431 @@
 /******************************************************************************
 
   Copyright (c) 2013-2018, Intel Corporation
   All rights reserved.
   
   Redistribution and use in source and binary forms, with or without 
   modification, are permitted provided that the following conditions are met:
   
    1. Redistributions of source code must retain the above copyright notice, 
       this list of conditions and the following disclaimer.
   
    2. Redistributions in binary form must reproduce the above copyright 
       notice, this list of conditions and the following disclaimer in the 
       documentation and/or other materials provided with the distribution.
   
    3. Neither the name of the Intel Corporation nor the names of its 
       contributors may be used to endorse or promote products derived from 
       this software without specific prior written permission.
   
   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 
   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 
   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 
   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 
   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 
   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 
   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 
   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
   POSSIBILITY OF SUCH DAMAGE.
 
 ******************************************************************************/
 /*$FreeBSD$*/
 
 #include "iavf.h"
 
 /*********************************************************************
  *  Driver version
  *********************************************************************/
 #define IAVF_DRIVER_VERSION_MAJOR	2
 #define IAVF_DRIVER_VERSION_MINOR	0
 #define IAVF_DRIVER_VERSION_BUILD	0
 
 #define IAVF_DRIVER_VERSION_STRING			\
     __XSTRING(IAVF_DRIVER_VERSION_MAJOR) "."		\
     __XSTRING(IAVF_DRIVER_VERSION_MINOR) "."		\
     __XSTRING(IAVF_DRIVER_VERSION_BUILD) "-k"
 
 /*********************************************************************
  *  PCI Device ID Table
  *
  *  Used by probe to select devices to load on
  *
  *  ( Vendor ID, Device ID, Branding String )
  *********************************************************************/
 
 static pci_vendor_info_t iavf_vendor_info_array[] =
 {
 	PVID(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_VF, "Intel(R) Ethernet Virtual Function 700 Series"),
 	PVID(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_X722_VF, "Intel(R) Ethernet Virtual Function 700 Series (X722)"),
 	PVID(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_ADAPTIVE_VF, "Intel(R) Ethernet Adaptive Virtual Function"),
 	/* required last entry */
 	PVID_END
 };
 
 /*********************************************************************
  *  Function prototypes
  *********************************************************************/
 static void	 *iavf_register(device_t dev);
 static int	 iavf_if_attach_pre(if_ctx_t ctx);
 static int	 iavf_if_attach_post(if_ctx_t ctx);
 static int	 iavf_if_detach(if_ctx_t ctx);
 static int	 iavf_if_shutdown(if_ctx_t ctx);
 static int	 iavf_if_suspend(if_ctx_t ctx);
 static int	 iavf_if_resume(if_ctx_t ctx);
 static int	 iavf_if_msix_intr_assign(if_ctx_t ctx, int msix);
 static void	 iavf_if_enable_intr(if_ctx_t ctx);
 static void	 iavf_if_disable_intr(if_ctx_t ctx);
 static int	 iavf_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid);
 static int	 iavf_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid);
 static int	 iavf_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int ntxqs, int ntxqsets);
 static int	 iavf_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nqs, int nqsets);
 static void	 iavf_if_queues_free(if_ctx_t ctx);
 static void	 iavf_if_update_admin_status(if_ctx_t ctx);
 static void	 iavf_if_multi_set(if_ctx_t ctx);
 static int	 iavf_if_mtu_set(if_ctx_t ctx, uint32_t mtu);
 static void	 iavf_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr);
 static int	 iavf_if_media_change(if_ctx_t ctx);
 static int	 iavf_if_promisc_set(if_ctx_t ctx, int flags);
 static void	 iavf_if_timer(if_ctx_t ctx, uint16_t qid);
 static void	 iavf_if_vlan_register(if_ctx_t ctx, u16 vtag);
 static void	 iavf_if_vlan_unregister(if_ctx_t ctx, u16 vtag);
 static uint64_t	 iavf_if_get_counter(if_ctx_t ctx, ift_counter cnt);
 static void	 iavf_if_stop(if_ctx_t ctx);
 
 static int	iavf_allocate_pci_resources(struct iavf_sc *);
 static int	iavf_reset_complete(struct i40e_hw *);
 static int	iavf_setup_vc(struct iavf_sc *);
 static int	iavf_reset(struct iavf_sc *);
 static int	iavf_vf_config(struct iavf_sc *);
 static void	iavf_init_filters(struct iavf_sc *);
 static void	iavf_free_pci_resources(struct iavf_sc *);
 static void	iavf_free_filters(struct iavf_sc *);
 static void	iavf_setup_interface(device_t, struct iavf_sc *);
 static void	iavf_add_device_sysctls(struct iavf_sc *);
 static void	iavf_enable_adminq_irq(struct i40e_hw *);
 static void	iavf_disable_adminq_irq(struct i40e_hw *);
 static void	iavf_enable_queue_irq(struct i40e_hw *, int);
 static void	iavf_disable_queue_irq(struct i40e_hw *, int);
 static void	iavf_config_rss(struct iavf_sc *);
 static void	iavf_stop(struct iavf_sc *);
 
 static int	iavf_add_mac_filter(struct iavf_sc *, u8 *, u16);
 static int	iavf_del_mac_filter(struct iavf_sc *sc, u8 *macaddr);
 static int	iavf_msix_que(void *);
 static int	iavf_msix_adminq(void *);
 //static void	iavf_del_multi(struct iavf_sc *sc);
 static void	iavf_init_multi(struct iavf_sc *sc);
 static void	iavf_configure_itr(struct iavf_sc *sc);
 
 static int	iavf_sysctl_rx_itr(SYSCTL_HANDLER_ARGS);
 static int	iavf_sysctl_tx_itr(SYSCTL_HANDLER_ARGS);
 static int	iavf_sysctl_current_speed(SYSCTL_HANDLER_ARGS);
 static int	iavf_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS);
 static int	iavf_sysctl_queue_interrupt_table(SYSCTL_HANDLER_ARGS);
 static int	iavf_sysctl_vf_reset(SYSCTL_HANDLER_ARGS);
 static int	iavf_sysctl_vflr_reset(SYSCTL_HANDLER_ARGS);
 
 static void	iavf_save_tunables(struct iavf_sc *);
 static enum i40e_status_code
     iavf_process_adminq(struct iavf_sc *, u16 *);
 static int	iavf_send_vc_msg(struct iavf_sc *sc, u32 op);
 static int	iavf_send_vc_msg_sleep(struct iavf_sc *sc, u32 op);
 
 /*********************************************************************
  *  FreeBSD Device Interface Entry Points
  *********************************************************************/
 
 static device_method_t iavf_methods[] = {
 	/* Device interface */
 	DEVMETHOD(device_register, iavf_register),
 	DEVMETHOD(device_probe, iflib_device_probe),
 	DEVMETHOD(device_attach, iflib_device_attach),
 	DEVMETHOD(device_detach, iflib_device_detach),
 	DEVMETHOD(device_shutdown, iflib_device_shutdown),
 	DEVMETHOD_END
 };
 
 static driver_t iavf_driver = {
 	"iavf", iavf_methods, sizeof(struct iavf_sc),
 };
 
 devclass_t iavf_devclass;
 DRIVER_MODULE(iavf, pci, iavf_driver, iavf_devclass, 0, 0);
 MODULE_PNP_INFO("U32:vendor;U32:device;U32:subvendor;U32:subdevice;U32:revision",
     pci, iavf, iavf_vendor_info_array,
         nitems(iavf_vendor_info_array) - 1);
 MODULE_VERSION(iavf, 1);
 
 MODULE_DEPEND(iavf, pci, 1, 1, 1);
 MODULE_DEPEND(iavf, ether, 1, 1, 1);
 MODULE_DEPEND(iavf, iflib, 1, 1, 1);
 
 MALLOC_DEFINE(M_IAVF, "iavf", "iavf driver allocations");
 
 static device_method_t iavf_if_methods[] = {
 	DEVMETHOD(ifdi_attach_pre, iavf_if_attach_pre),
 	DEVMETHOD(ifdi_attach_post, iavf_if_attach_post),
 	DEVMETHOD(ifdi_detach, iavf_if_detach),
 	DEVMETHOD(ifdi_shutdown, iavf_if_shutdown),
 	DEVMETHOD(ifdi_suspend, iavf_if_suspend),
 	DEVMETHOD(ifdi_resume, iavf_if_resume),
 	DEVMETHOD(ifdi_init, iavf_if_init),
 	DEVMETHOD(ifdi_stop, iavf_if_stop),
 	DEVMETHOD(ifdi_msix_intr_assign, iavf_if_msix_intr_assign),
 	DEVMETHOD(ifdi_intr_enable, iavf_if_enable_intr),
 	DEVMETHOD(ifdi_intr_disable, iavf_if_disable_intr),
 	DEVMETHOD(ifdi_rx_queue_intr_enable, iavf_if_rx_queue_intr_enable),
 	DEVMETHOD(ifdi_tx_queue_intr_enable, iavf_if_tx_queue_intr_enable),
 	DEVMETHOD(ifdi_tx_queues_alloc, iavf_if_tx_queues_alloc),
 	DEVMETHOD(ifdi_rx_queues_alloc, iavf_if_rx_queues_alloc),
 	DEVMETHOD(ifdi_queues_free, iavf_if_queues_free),
 	DEVMETHOD(ifdi_update_admin_status, iavf_if_update_admin_status),
 	DEVMETHOD(ifdi_multi_set, iavf_if_multi_set),
 	DEVMETHOD(ifdi_mtu_set, iavf_if_mtu_set),
 	DEVMETHOD(ifdi_media_status, iavf_if_media_status),
 	DEVMETHOD(ifdi_media_change, iavf_if_media_change),
 	DEVMETHOD(ifdi_promisc_set, iavf_if_promisc_set),
 	DEVMETHOD(ifdi_timer, iavf_if_timer),
 	DEVMETHOD(ifdi_vlan_register, iavf_if_vlan_register),
 	DEVMETHOD(ifdi_vlan_unregister, iavf_if_vlan_unregister),
 	DEVMETHOD(ifdi_get_counter, iavf_if_get_counter),
 	DEVMETHOD_END
 };
 
 static driver_t iavf_if_driver = {
 	"iavf_if", iavf_if_methods, sizeof(struct iavf_sc)
 };
 
 /*
 ** TUNEABLE PARAMETERS:
 */
 
 static SYSCTL_NODE(_hw, OID_AUTO, iavf, CTLFLAG_RD, 0,
     "iavf driver parameters");
 
 /*
  * Different method for processing TX descriptor
  * completion.
  */
 static int iavf_enable_head_writeback = 0;
 TUNABLE_INT("hw.iavf.enable_head_writeback",
     &iavf_enable_head_writeback);
 SYSCTL_INT(_hw_iavf, OID_AUTO, enable_head_writeback, CTLFLAG_RDTUN,
     &iavf_enable_head_writeback, 0,
     "For detecting last completed TX descriptor by hardware, use value written by HW instead of checking descriptors");
 
 static int iavf_core_debug_mask = 0;
 TUNABLE_INT("hw.iavf.core_debug_mask",
     &iavf_core_debug_mask);
 SYSCTL_INT(_hw_iavf, OID_AUTO, core_debug_mask, CTLFLAG_RDTUN,
     &iavf_core_debug_mask, 0,
     "Display debug statements that are printed in non-shared code");
 
 static int iavf_shared_debug_mask = 0;
 TUNABLE_INT("hw.iavf.shared_debug_mask",
     &iavf_shared_debug_mask);
 SYSCTL_INT(_hw_iavf, OID_AUTO, shared_debug_mask, CTLFLAG_RDTUN,
     &iavf_shared_debug_mask, 0,
     "Display debug statements that are printed in shared code");
 
 int iavf_rx_itr = IXL_ITR_8K;
 TUNABLE_INT("hw.iavf.rx_itr", &iavf_rx_itr);
 SYSCTL_INT(_hw_iavf, OID_AUTO, rx_itr, CTLFLAG_RDTUN,
     &iavf_rx_itr, 0, "RX Interrupt Rate");
 
 int iavf_tx_itr = IXL_ITR_4K;
 TUNABLE_INT("hw.iavf.tx_itr", &iavf_tx_itr);
 SYSCTL_INT(_hw_iavf, OID_AUTO, tx_itr, CTLFLAG_RDTUN,
     &iavf_tx_itr, 0, "TX Interrupt Rate");
 
 extern struct if_txrx ixl_txrx_hwb;
 extern struct if_txrx ixl_txrx_dwb;
 
 static struct if_shared_ctx iavf_sctx_init = {
 	.isc_magic = IFLIB_MAGIC,
 	.isc_q_align = PAGE_SIZE,/* max(DBA_ALIGN, PAGE_SIZE) */
 	.isc_tx_maxsize = IXL_TSO_SIZE + sizeof(struct ether_vlan_header),
 	.isc_tx_maxsegsize = IXL_MAX_DMA_SEG_SIZE,
 	.isc_tso_maxsize = IXL_TSO_SIZE + sizeof(struct ether_vlan_header),
 	.isc_tso_maxsegsize = IXL_MAX_DMA_SEG_SIZE,
 	.isc_rx_maxsize = 16384,
 	.isc_rx_nsegments = IXL_MAX_RX_SEGS,
 	.isc_rx_maxsegsize = IXL_MAX_DMA_SEG_SIZE,
 	.isc_nfl = 1,
 	.isc_ntxqs = 1,
 	.isc_nrxqs = 1,
 
 	.isc_admin_intrcnt = 1,
 	.isc_vendor_info = iavf_vendor_info_array,
 	.isc_driver_version = IAVF_DRIVER_VERSION_STRING,
 	.isc_driver = &iavf_if_driver,
 	.isc_flags = IFLIB_NEED_SCRATCH | IFLIB_NEED_ZERO_CSUM | IFLIB_TSO_INIT_IP | IFLIB_IS_VF,
 
 	.isc_nrxd_min = {IXL_MIN_RING},
 	.isc_ntxd_min = {IXL_MIN_RING},
 	.isc_nrxd_max = {IXL_MAX_RING},
 	.isc_ntxd_max = {IXL_MAX_RING},
 	.isc_nrxd_default = {IXL_DEFAULT_RING},
 	.isc_ntxd_default = {IXL_DEFAULT_RING},
 };
 
 if_shared_ctx_t iavf_sctx = &iavf_sctx_init;
 
 /*** Functions ***/
 static void *
 iavf_register(device_t dev)
 {
 	return (iavf_sctx);
 }
 
 static int
 iavf_allocate_pci_resources(struct iavf_sc *sc)
 {
 	struct i40e_hw *hw = &sc->hw;
 	device_t dev = iflib_get_dev(sc->vsi.ctx);
 	int             rid;
 
 	/* Map BAR0 */
 	rid = PCIR_BAR(0);
 	sc->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
 	    &rid, RF_ACTIVE);
 
 	if (!(sc->pci_mem)) {
 		device_printf(dev, "Unable to allocate bus resource: PCI memory\n");
 		return (ENXIO);
  	}
  
 	/* Save off the PCI information */
 	hw->vendor_id = pci_get_vendor(dev);
 	hw->device_id = pci_get_device(dev);
 	hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
 	hw->subsystem_vendor_id =
 	    pci_read_config(dev, PCIR_SUBVEND_0, 2);
 	hw->subsystem_device_id =
 	    pci_read_config(dev, PCIR_SUBDEV_0, 2);
 
 	hw->bus.device = pci_get_slot(dev);
 	hw->bus.func = pci_get_function(dev);
 
 	/* Save off register access information */
 	sc->osdep.mem_bus_space_tag =
 		rman_get_bustag(sc->pci_mem);
 	sc->osdep.mem_bus_space_handle =
 		rman_get_bushandle(sc->pci_mem);
 	sc->osdep.mem_bus_space_size = rman_get_size(sc->pci_mem);
 	sc->osdep.flush_reg = I40E_VFGEN_RSTAT;
 	sc->osdep.dev = dev;
 
 	sc->hw.hw_addr = (u8 *) &sc->osdep.mem_bus_space_handle;
 	sc->hw.back = &sc->osdep;
 
  	return (0);
 }
 
 static int
 iavf_if_attach_pre(if_ctx_t ctx)
 {
 	device_t dev;
 	struct iavf_sc *sc;
 	struct i40e_hw *hw;
 	struct ixl_vsi *vsi;
 	if_softc_ctx_t scctx;
 	int error = 0;
 
 	dev = iflib_get_dev(ctx);
 	sc = iflib_get_softc(ctx);
 
 	vsi = &sc->vsi;
 	vsi->back = sc;
 	sc->dev = dev;
 	hw = &sc->hw;
 
 	vsi->dev = dev;
 	vsi->hw = &sc->hw;
 	vsi->num_vlans = 0;
 	vsi->ctx = ctx;
 	vsi->media = iflib_get_media(ctx);
 	vsi->shared = scctx = iflib_get_softc_ctx(ctx);
 
 	iavf_save_tunables(sc);
 
 	/* Do PCI setup - map BAR0, etc */
 	if (iavf_allocate_pci_resources(sc)) {
 		device_printf(dev, "%s: Allocation of PCI resources failed\n",
 		    __func__);
 		error = ENXIO;
 		goto err_early;
 	}
 
 	iavf_dbg_init(sc, "Allocated PCI resources and MSI-X vectors\n");
 
 	/*
 	 * XXX: This is called by init_shared_code in the PF driver,
 	 * but the rest of that function does not support VFs.
 	 */
 	error = i40e_set_mac_type(hw);
 	if (error) {
 		device_printf(dev, "%s: set_mac_type failed: %d\n",
 		    __func__, error);
 		goto err_pci_res;
 	}
 
 	error = iavf_reset_complete(hw);
 	if (error) {
 		device_printf(dev, "%s: Device is still being reset\n",
 		    __func__);
 		goto err_pci_res;
 	}
 
 	iavf_dbg_init(sc, "VF Device is ready for configuration\n");
 
 	/* Sets up Admin Queue */
 	error = iavf_setup_vc(sc);
 	if (error) {
 		device_printf(dev, "%s: Error setting up PF comms, %d\n",
 		    __func__, error);
 		goto err_pci_res;
 	}
 
 	iavf_dbg_init(sc, "PF API version verified\n");
 
 	/* Need API version before sending reset message */
 	error = iavf_reset(sc);
 	if (error) {
 		device_printf(dev, "VF reset failed; reload the driver\n");
 		goto err_aq;
 	}
 
 	iavf_dbg_init(sc, "VF reset complete\n");
 
 	/* Ask for VF config from PF */
 	error = iavf_vf_config(sc);
 	if (error) {
 		device_printf(dev, "Error getting configuration from PF: %d\n",
 		    error);
 		goto err_aq;
 	}
 
 	device_printf(dev,
 	    "VSIs %d, QPs %d, MSI-X %d, RSS sizes: key %d lut %d\n",
 	    sc->vf_res->num_vsis,
 	    sc->vf_res->num_queue_pairs,
 	    sc->vf_res->max_vectors,
 	    sc->vf_res->rss_key_size,
 	    sc->vf_res->rss_lut_size);
 	iavf_dbg_info(sc, "Capabilities=%b\n",
 	    sc->vf_res->vf_cap_flags, IAVF_PRINTF_VF_OFFLOAD_FLAGS);
 
 	/* got VF config message back from PF, now we can parse it */
 	for (int i = 0; i < sc->vf_res->num_vsis; i++) {
 		/* XXX: We only use the first VSI we find */
 		if (sc->vf_res->vsi_res[i].vsi_type == I40E_VSI_SRIOV)
 			sc->vsi_res = &sc->vf_res->vsi_res[i];
 	}
 	if (!sc->vsi_res) {
 		device_printf(dev, "%s: no LAN VSI found\n", __func__);
 		error = EIO;
 		goto err_res_buf;
 	}
 	vsi->id = sc->vsi_res->vsi_id;
 
 	iavf_dbg_init(sc, "Resource Acquisition complete\n");
 
 	/* If no mac address was assigned just make a random one */
 	if (!iavf_check_ether_addr(hw->mac.addr)) {
 		u8 addr[ETHER_ADDR_LEN];
 		arc4rand(&addr, sizeof(addr), 0);
 		addr[0] &= 0xFE;
 		addr[0] |= 0x02;
 		bcopy(addr, hw->mac.addr, sizeof(addr));
 	}
 	bcopy(hw->mac.addr, hw->mac.perm_addr, ETHER_ADDR_LEN);
 	iflib_set_mac(ctx, hw->mac.addr);
 
 	/* Allocate filter lists */
 	iavf_init_filters(sc);
 
 	/* Fill out more iflib parameters */
 	scctx->isc_ntxqsets_max = scctx->isc_nrxqsets_max =
 	    sc->vsi_res->num_queue_pairs;
 	if (vsi->enable_head_writeback) {
 		scctx->isc_txqsizes[0] = roundup2(scctx->isc_ntxd[0]
 		    * sizeof(struct i40e_tx_desc) + sizeof(u32), DBA_ALIGN);
 		scctx->isc_txrx = &ixl_txrx_hwb;
 	} else {
 		scctx->isc_txqsizes[0] = roundup2(scctx->isc_ntxd[0]
 		    * sizeof(struct i40e_tx_desc), DBA_ALIGN);
 		scctx->isc_txrx = &ixl_txrx_dwb;
 	}
 	scctx->isc_rxqsizes[0] = roundup2(scctx->isc_nrxd[0]
 	    * sizeof(union i40e_32byte_rx_desc), DBA_ALIGN);
 	scctx->isc_msix_bar = PCIR_BAR(IXL_MSIX_BAR);
 	scctx->isc_tx_nsegments = IXL_MAX_TX_SEGS;
 	scctx->isc_tx_tso_segments_max = IXL_MAX_TSO_SEGS;
 	scctx->isc_tx_tso_size_max = IXL_TSO_SIZE;
 	scctx->isc_tx_tso_segsize_max = IXL_MAX_DMA_SEG_SIZE;
 	scctx->isc_rss_table_size = IXL_RSS_VSI_LUT_SIZE;
 	scctx->isc_tx_csum_flags = CSUM_OFFLOAD;
 	scctx->isc_capabilities = scctx->isc_capenable = IXL_CAPS;
 
 	return (0);
 
 err_res_buf:
 	free(sc->vf_res, M_IAVF);
 err_aq:
 	i40e_shutdown_adminq(hw);
 err_pci_res:
 	iavf_free_pci_resources(sc);
 err_early:
 	return (error);
 }
 
 static int
 iavf_if_attach_post(if_ctx_t ctx)
 {
 	device_t dev;
 	struct iavf_sc	*sc;
 	struct i40e_hw	*hw;
 	struct ixl_vsi *vsi;
 	int error = 0;
 
 	INIT_DBG_DEV(dev, "begin");
 
 	dev = iflib_get_dev(ctx);
 	sc = iflib_get_softc(ctx);
 	vsi = &sc->vsi;
 	vsi->ifp = iflib_get_ifp(ctx);
 	hw = &sc->hw;
 
 	/* Save off determined number of queues for interface */
 	vsi->num_rx_queues = vsi->shared->isc_nrxqsets;
 	vsi->num_tx_queues = vsi->shared->isc_ntxqsets;
 
 	/* Setup the stack interface */
 	iavf_setup_interface(dev, sc);
 
 	INIT_DBG_DEV(dev, "Interface setup complete");
 
 	/* Initialize statistics & add sysctls */
 	bzero(&sc->vsi.eth_stats, sizeof(struct i40e_eth_stats));
 	iavf_add_device_sysctls(sc);
 
 	sc->init_state = IAVF_INIT_READY;
 	atomic_store_rel_32(&sc->queues_enabled, 0);
 
 	/* We want AQ enabled early for init */
 	iavf_enable_adminq_irq(hw);
 
 	INIT_DBG_DEV(dev, "end");
 
 	return (error);
 }
 
 /**
  * XXX: iflib always ignores the return value of detach()
  * -> This means that this isn't allowed to fail
  */
 static int
 iavf_if_detach(if_ctx_t ctx)
 {
 	struct iavf_sc *sc = iflib_get_softc(ctx);
 	struct ixl_vsi *vsi = &sc->vsi;
 	struct i40e_hw *hw = &sc->hw;
 	device_t dev = sc->dev;
 	enum i40e_status_code status;
 
 	INIT_DBG_DEV(dev, "begin");
 
 	/* Remove all the media and link information */
 	ifmedia_removeall(vsi->media);
 
 	iavf_disable_adminq_irq(hw);
 	status = i40e_shutdown_adminq(&sc->hw);
 	if (status != I40E_SUCCESS) {
 		device_printf(dev,
 		    "i40e_shutdown_adminq() failed with status %s\n",
 		    i40e_stat_str(hw, status));
 	}
 
 	free(sc->vf_res, M_IAVF);
 	iavf_free_pci_resources(sc);
 	iavf_free_filters(sc);
 
 	INIT_DBG_DEV(dev, "end");
 	return (0);
 }
 
 static int
 iavf_if_shutdown(if_ctx_t ctx)
 {
 	return (0);
 }
 
 static int
 iavf_if_suspend(if_ctx_t ctx)
 {
 	return (0);
 }
 
 static int
 iavf_if_resume(if_ctx_t ctx)
 {
 	return (0);
 }
 
 static int
 iavf_send_vc_msg_sleep(struct iavf_sc *sc, u32 op)
 {
 	int error = 0;
 	if_ctx_t ctx = sc->vsi.ctx;
 
 	error = ixl_vc_send_cmd(sc, op);
 	if (error != 0) {
 		iavf_dbg_vc(sc, "Error sending %b: %d\n", op, IAVF_FLAGS, error);
 		return (error);
 	}
 
 	/* Don't wait for a response if the device is being detached. */
 	if (!iflib_in_detach(ctx)) {
 		iavf_dbg_vc(sc, "Sleeping for op %b\n", op, IAVF_FLAGS);
 		error = sx_sleep(ixl_vc_get_op_chan(sc, op),
 		    iflib_ctx_lock_get(ctx), PRI_MAX, "iavf_vc", IAVF_AQ_TIMEOUT);
 
 		if (error == EWOULDBLOCK)
 			device_printf(sc->dev, "%b timed out\n", op, IAVF_FLAGS);
 	}
 
 	return (error);
 }
 
 static int
 iavf_send_vc_msg(struct iavf_sc *sc, u32 op)
 {
 	int error = 0;
 
 	error = ixl_vc_send_cmd(sc, op);
 	if (error != 0)
 		iavf_dbg_vc(sc, "Error sending %b: %d\n", op, IAVF_FLAGS, error);
 
 	return (error);
 }
 
 static void
 iavf_init_queues(struct ixl_vsi *vsi)
 {
-	if_softc_ctx_t scctx = vsi->shared;
 	struct ixl_tx_queue *tx_que = vsi->tx_queues;
 	struct ixl_rx_queue *rx_que = vsi->rx_queues;
 	struct rx_ring *rxr;
 
 	for (int i = 0; i < vsi->num_tx_queues; i++, tx_que++)
 		ixl_init_tx_ring(vsi, tx_que);
 
 	for (int i = 0; i < vsi->num_rx_queues; i++, rx_que++) {
 		rxr = &rx_que->rxr;
 
-		if (scctx->isc_max_frame_size <= MCLBYTES)
-			rxr->mbuf_sz = MCLBYTES;
-		else
-			rxr->mbuf_sz = MJUMPAGESIZE;
+		rxr->mbuf_sz = iflib_get_rx_mbuf_sz(vsi->ctx);
 
 		wr32(vsi->hw, rxr->tail, 0);
 	}
 }
 
 void
 iavf_if_init(if_ctx_t ctx)
 {
 	struct iavf_sc *sc = iflib_get_softc(ctx);
 	struct ixl_vsi *vsi = &sc->vsi;
 	struct i40e_hw *hw = &sc->hw;
 	struct ifnet *ifp = iflib_get_ifp(ctx);
 	u8 tmpaddr[ETHER_ADDR_LEN];
 	int error = 0;
 
 	INIT_DBG_IF(ifp, "begin");
 
 	MPASS(sx_xlocked(iflib_ctx_lock_get(ctx)));
 
 	error = iavf_reset_complete(hw);
 	if (error) {
 		device_printf(sc->dev, "%s: VF reset failed\n",
 		    __func__);
 	}
 
 	if (!i40e_check_asq_alive(hw)) {
 		iavf_dbg_info(sc, "ASQ is not alive, re-initializing AQ\n");
 		pci_enable_busmaster(sc->dev);
 		i40e_shutdown_adminq(hw);
 		i40e_init_adminq(hw);
 	}
 
 	/* Make sure queues are disabled */
 	iavf_send_vc_msg(sc, IAVF_FLAG_AQ_DISABLE_QUEUES);
 
 	bcopy(IF_LLADDR(ifp), tmpaddr, ETHER_ADDR_LEN);
 	if (!cmp_etheraddr(hw->mac.addr, tmpaddr) &&
 	    (i40e_validate_mac_addr(tmpaddr) == I40E_SUCCESS)) {
 		error = iavf_del_mac_filter(sc, hw->mac.addr);
 		if (error == 0)
 			iavf_send_vc_msg(sc, IAVF_FLAG_AQ_DEL_MAC_FILTER);
 
 		bcopy(tmpaddr, hw->mac.addr, ETH_ALEN);
 	}
 
 	error = iavf_add_mac_filter(sc, hw->mac.addr, 0);
 	if (!error || error == EEXIST)
 		iavf_send_vc_msg(sc, IAVF_FLAG_AQ_ADD_MAC_FILTER);
 	iflib_set_mac(ctx, hw->mac.addr);
 
 	/* Prepare the queues for operation */
 	iavf_init_queues(vsi);
 
 	/* Set initial ITR values */
 	iavf_configure_itr(sc);
 
 	iavf_send_vc_msg(sc, IAVF_FLAG_AQ_CONFIGURE_QUEUES);
 
 	/* Set up RSS */
 	iavf_config_rss(sc);
 
 	/* Map vectors */
 	iavf_send_vc_msg(sc, IAVF_FLAG_AQ_MAP_VECTORS);
 
 	/* Init SW TX ring indices */
 	if (vsi->enable_head_writeback)
 		ixl_init_tx_cidx(vsi);
 	else
 		ixl_init_tx_rsqs(vsi);
 
 	/* Configure promiscuous mode */
 	iavf_if_promisc_set(ctx, if_getflags(ifp));
 
 	/* Enable queues */
 	iavf_send_vc_msg_sleep(sc, IAVF_FLAG_AQ_ENABLE_QUEUES);
 
 	sc->init_state = IAVF_RUNNING;
 }
 
 /*
  * iavf_attach() helper function; initalizes the admin queue
  * and attempts to establish contact with the PF by
  * retrying the initial "API version" message several times
  * or until the PF responds.
  */
 static int
 iavf_setup_vc(struct iavf_sc *sc)
 {
 	struct i40e_hw *hw = &sc->hw;
 	device_t dev = sc->dev;
 	int error = 0, ret_error = 0, asq_retries = 0;
 	bool send_api_ver_retried = 0;
 
 	/* Need to set these AQ paramters before initializing AQ */
 	hw->aq.num_arq_entries = IXL_AQ_LEN;
 	hw->aq.num_asq_entries = IXL_AQ_LEN;
 	hw->aq.arq_buf_size = IXL_AQ_BUF_SZ;
 	hw->aq.asq_buf_size = IXL_AQ_BUF_SZ;
 
 	for (int i = 0; i < IAVF_AQ_MAX_ERR; i++) {
 		/* Initialize admin queue */
 		error = i40e_init_adminq(hw);
 		if (error) {
 			device_printf(dev, "%s: init_adminq failed: %d\n",
 			    __func__, error);
 			ret_error = 1;
 			continue;
 		}
 
 		iavf_dbg_init(sc, "Initialized Admin Queue; starting"
 		    " send_api_ver attempt %d", i+1);
 
 retry_send:
 		/* Send VF's API version */
 		error = iavf_send_api_ver(sc);
 		if (error) {
 			i40e_shutdown_adminq(hw);
 			ret_error = 2;
 			device_printf(dev, "%s: unable to send api"
 			    " version to PF on attempt %d, error %d\n",
 			    __func__, i+1, error);
 		}
 
 		asq_retries = 0;
 		while (!i40e_asq_done(hw)) {
 			if (++asq_retries > IAVF_AQ_MAX_ERR) {
 				i40e_shutdown_adminq(hw);
 				device_printf(dev, "Admin Queue timeout "
 				    "(waiting for send_api_ver), %d more tries...\n",
 				    IAVF_AQ_MAX_ERR - (i + 1));
 				ret_error = 3;
 				break;
 			} 
 			i40e_msec_pause(10);
 		}
 		if (asq_retries > IAVF_AQ_MAX_ERR)
 			continue;
 
 		iavf_dbg_init(sc, "Sent API version message to PF");
 
 		/* Verify that the VF accepts the PF's API version */
 		error = iavf_verify_api_ver(sc);
 		if (error == ETIMEDOUT) {
 			if (!send_api_ver_retried) {
 				/* Resend message, one more time */
 				send_api_ver_retried = true;
 				device_printf(dev,
 				    "%s: Timeout while verifying API version on first"
 				    " try!\n", __func__);
 				goto retry_send;
 			} else {
 				device_printf(dev,
 				    "%s: Timeout while verifying API version on second"
 				    " try!\n", __func__);
 				ret_error = 4;
 				break;
 			}
 		}
 		if (error) {
 			device_printf(dev,
 			    "%s: Unable to verify API version,"
 			    " error %s\n", __func__, i40e_stat_str(hw, error));
 			ret_error = 5;
 		}
 		break;
 	}
 
 	if (ret_error >= 4)
 		i40e_shutdown_adminq(hw);
 	return (ret_error);
 }
 
 /*
  * iavf_attach() helper function; asks the PF for this VF's
  * configuration, and saves the information if it receives it.
  */
 static int
 iavf_vf_config(struct iavf_sc *sc)
 {
 	struct i40e_hw *hw = &sc->hw;
 	device_t dev = sc->dev;
 	int bufsz, error = 0, ret_error = 0;
 	int asq_retries, retried = 0;
 
 retry_config:
 	error = iavf_send_vf_config_msg(sc);
 	if (error) {
 		device_printf(dev,
 		    "%s: Unable to send VF config request, attempt %d,"
 		    " error %d\n", __func__, retried + 1, error);
 		ret_error = 2;
 	}
 
 	asq_retries = 0;
 	while (!i40e_asq_done(hw)) {
 		if (++asq_retries > IAVF_AQ_MAX_ERR) {
 			device_printf(dev, "%s: Admin Queue timeout "
 			    "(waiting for send_vf_config_msg), attempt %d\n",
 			    __func__, retried + 1);
 			ret_error = 3;
 			goto fail;
 		}
 		i40e_msec_pause(10);
 	}
 
 	iavf_dbg_init(sc, "Sent VF config message to PF, attempt %d\n",
 	    retried + 1);
 
 	if (!sc->vf_res) {
 		bufsz = sizeof(struct virtchnl_vf_resource) +
 		    (I40E_MAX_VF_VSI * sizeof(struct virtchnl_vsi_resource));
 		sc->vf_res = malloc(bufsz, M_IAVF, M_NOWAIT);
 		if (!sc->vf_res) {
 			device_printf(dev,
 			    "%s: Unable to allocate memory for VF configuration"
 			    " message from PF on attempt %d\n", __func__, retried + 1);
 			ret_error = 1;
 			goto fail;
 		}
 	}
 
 	/* Check for VF config response */
 	error = iavf_get_vf_config(sc);
 	if (error == ETIMEDOUT) {
 		/* The 1st time we timeout, send the configuration message again */
 		if (!retried) {
 			retried++;
 			goto retry_config;
 		}
 		device_printf(dev,
 		    "%s: iavf_get_vf_config() timed out waiting for a response\n",
 		    __func__);
 	}
 	if (error) {
 		device_printf(dev,
 		    "%s: Unable to get VF configuration from PF after %d tries!\n",
 		    __func__, retried + 1);
 		ret_error = 4;
 	}
 	goto done;
 
 fail:
 	free(sc->vf_res, M_IAVF);
 done:
 	return (ret_error);
 }
 
 static int
 iavf_if_msix_intr_assign(if_ctx_t ctx, int msix)
 {
 	struct iavf_sc *sc = iflib_get_softc(ctx);
 	struct ixl_vsi *vsi = &sc->vsi;
 	struct ixl_rx_queue *rx_que = vsi->rx_queues;
 	struct ixl_tx_queue *tx_que = vsi->tx_queues;
 	int err, i, rid, vector = 0;
 	char buf[16];
 
 	MPASS(vsi->shared->isc_nrxqsets > 0);
 	MPASS(vsi->shared->isc_ntxqsets > 0);
 
 	/* Admin Que is vector 0*/
 	rid = vector + 1;
 	err = iflib_irq_alloc_generic(ctx, &vsi->irq, rid, IFLIB_INTR_ADMIN,
 	    iavf_msix_adminq, sc, 0, "aq");
 	if (err) {
 		iflib_irq_free(ctx, &vsi->irq);
 		device_printf(iflib_get_dev(ctx),
 		    "Failed to register Admin Que handler");
 		return (err);
 	}
 
 	/* Now set up the stations */
 	for (i = 0, vector = 1; i < vsi->shared->isc_nrxqsets; i++, vector++, rx_que++) {
 		rid = vector + 1;
 
 		snprintf(buf, sizeof(buf), "rxq%d", i);
 		err = iflib_irq_alloc_generic(ctx, &rx_que->que_irq, rid,
 		    IFLIB_INTR_RX, iavf_msix_que, rx_que, rx_que->rxr.me, buf);
 		/* XXX: Does the driver work as expected if there are fewer num_rx_queues than
 		 * what's expected in the iflib context? */
 		if (err) {
 			device_printf(iflib_get_dev(ctx),
 			    "Failed to allocate queue RX int vector %d, err: %d\n", i, err);
 			vsi->num_rx_queues = i + 1;
 			goto fail;
 		}
 		rx_que->msix = vector;
 	}
 
 	bzero(buf, sizeof(buf));
 
 	for (i = 0; i < vsi->shared->isc_ntxqsets; i++, tx_que++) {
 		snprintf(buf, sizeof(buf), "txq%d", i);
 		iflib_softirq_alloc_generic(ctx,
 		    &vsi->rx_queues[i % vsi->shared->isc_nrxqsets].que_irq,
 		    IFLIB_INTR_TX, tx_que, tx_que->txr.me, buf);
 
 		/* TODO: Maybe call a strategy function for this to figure out which
 		* interrupts to map Tx queues to. I don't know if there's an immediately
 		* better way than this other than a user-supplied map, though. */
 		tx_que->msix = (i % vsi->shared->isc_nrxqsets) + 1;
 	}
 
 	return (0);
 fail:
 	iflib_irq_free(ctx, &vsi->irq);
 	rx_que = vsi->rx_queues;
 	for (int i = 0; i < vsi->num_rx_queues; i++, rx_que++)
 		iflib_irq_free(ctx, &rx_que->que_irq);
 	return (err);
 }
 
 /* Enable all interrupts */
 static void
 iavf_if_enable_intr(if_ctx_t ctx)
 {
 	struct iavf_sc *sc = iflib_get_softc(ctx);
 	struct ixl_vsi *vsi = &sc->vsi;
 
 	iavf_enable_intr(vsi);
 }
 
 /* Disable all interrupts */
 static void
 iavf_if_disable_intr(if_ctx_t ctx)
 {
 	struct iavf_sc *sc = iflib_get_softc(ctx);
 	struct ixl_vsi *vsi = &sc->vsi;
 
 	iavf_disable_intr(vsi);
 }
 
 static int
 iavf_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid)
 {
 	struct iavf_sc *sc = iflib_get_softc(ctx);
 	struct ixl_vsi *vsi = &sc->vsi;
 	struct i40e_hw *hw = vsi->hw;
 	struct ixl_rx_queue *rx_que = &vsi->rx_queues[rxqid];
 
 	iavf_enable_queue_irq(hw, rx_que->msix - 1);
 	return (0);
 }
 
 static int
 iavf_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid)
 {
 	struct iavf_sc *sc = iflib_get_softc(ctx);
 	struct ixl_vsi *vsi = &sc->vsi;
 	struct i40e_hw *hw = vsi->hw;
 	struct ixl_tx_queue *tx_que = &vsi->tx_queues[txqid];
 
 	iavf_enable_queue_irq(hw, tx_que->msix - 1);
 	return (0);
 }
 
 static int
 iavf_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int ntxqs, int ntxqsets)
 {
 	struct iavf_sc *sc = iflib_get_softc(ctx);
 	struct ixl_vsi *vsi = &sc->vsi;
 	if_softc_ctx_t scctx = vsi->shared;
 	struct ixl_tx_queue *que;
 	int i, j, error = 0;
 
 	MPASS(scctx->isc_ntxqsets > 0);
 	MPASS(ntxqs == 1);
 	MPASS(scctx->isc_ntxqsets == ntxqsets);
 
 	/* Allocate queue structure memory */
 	if (!(vsi->tx_queues =
 	    (struct ixl_tx_queue *) malloc(sizeof(struct ixl_tx_queue) *ntxqsets, M_IAVF, M_NOWAIT | M_ZERO))) {
 		device_printf(iflib_get_dev(ctx), "Unable to allocate TX ring memory\n");
 		return (ENOMEM);
 	}
 
 	for (i = 0, que = vsi->tx_queues; i < ntxqsets; i++, que++) {
 		struct tx_ring *txr = &que->txr;
 
 		txr->me = i;
 		que->vsi = vsi;
 
 		if (!vsi->enable_head_writeback) {
 			/* Allocate report status array */
 			if (!(txr->tx_rsq = malloc(sizeof(qidx_t) * scctx->isc_ntxd[0], M_IAVF, M_NOWAIT))) {
 				device_printf(iflib_get_dev(ctx), "failed to allocate tx_rsq memory\n");
 				error = ENOMEM;
 				goto fail;
 			}
 			/* Init report status array */
 			for (j = 0; j < scctx->isc_ntxd[0]; j++)
 				txr->tx_rsq[j] = QIDX_INVALID;
 		}
 		/* get the virtual and physical address of the hardware queues */
 		txr->tail = I40E_QTX_TAIL1(txr->me);
 		txr->tx_base = (struct i40e_tx_desc *)vaddrs[i * ntxqs];
 		txr->tx_paddr = paddrs[i * ntxqs];
 		txr->que = que;
 	}
 
 	return (0);
 fail:
 	iavf_if_queues_free(ctx);
 	return (error);
 }
 
 static int
 iavf_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nrxqs, int nrxqsets)
 {
 	struct iavf_sc *sc = iflib_get_softc(ctx);
 	struct ixl_vsi *vsi = &sc->vsi;
 	struct ixl_rx_queue *que;
 	int i, error = 0;
 
 #ifdef INVARIANTS
 	if_softc_ctx_t scctx = vsi->shared;
 	MPASS(scctx->isc_nrxqsets > 0);
 	MPASS(nrxqs == 1);
 	MPASS(scctx->isc_nrxqsets == nrxqsets);
 #endif
 
 	/* Allocate queue structure memory */
 	if (!(vsi->rx_queues =
 	    (struct ixl_rx_queue *) malloc(sizeof(struct ixl_rx_queue) *
 	    nrxqsets, M_IAVF, M_NOWAIT | M_ZERO))) {
 		device_printf(iflib_get_dev(ctx), "Unable to allocate RX ring memory\n");
 		error = ENOMEM;
 		goto fail;
 	}
 
 	for (i = 0, que = vsi->rx_queues; i < nrxqsets; i++, que++) {
 		struct rx_ring *rxr = &que->rxr;
 
 		rxr->me = i;
 		que->vsi = vsi;
 
 		/* get the virtual and physical address of the hardware queues */
 		rxr->tail = I40E_QRX_TAIL1(rxr->me);
 		rxr->rx_base = (union i40e_rx_desc *)vaddrs[i * nrxqs];
 		rxr->rx_paddr = paddrs[i * nrxqs];
 		rxr->que = que;
 	}
 
 	return (0);
 fail:
 	iavf_if_queues_free(ctx);
 	return (error);
 }
 
 static void
 iavf_if_queues_free(if_ctx_t ctx)
 {
 	struct iavf_sc *sc = iflib_get_softc(ctx);
 	struct ixl_vsi *vsi = &sc->vsi;
 
 	if (!vsi->enable_head_writeback) {
 		struct ixl_tx_queue *que;
 		int i = 0;
 
 		for (i = 0, que = vsi->tx_queues; i < vsi->shared->isc_ntxqsets; i++, que++) {
 			struct tx_ring *txr = &que->txr;
 			if (txr->tx_rsq != NULL) {
 				free(txr->tx_rsq, M_IAVF);
 				txr->tx_rsq = NULL;
 			}
 		}
 	}
 
 	if (vsi->tx_queues != NULL) {
 		free(vsi->tx_queues, M_IAVF);
 		vsi->tx_queues = NULL;
 	}
 	if (vsi->rx_queues != NULL) {
 		free(vsi->rx_queues, M_IAVF);
 		vsi->rx_queues = NULL;
 	}
 }
 
 static int
 iavf_check_aq_errors(struct iavf_sc *sc)
 {
 	struct i40e_hw *hw = &sc->hw;
 	device_t dev = sc->dev;
 	u32 reg, oldreg;
 	u8 aq_error = false;
 
 	/* check for Admin queue errors */
 	oldreg = reg = rd32(hw, hw->aq.arq.len);
 	if (reg & I40E_VF_ARQLEN1_ARQVFE_MASK) {
 		device_printf(dev, "ARQ VF Error detected\n");
 		reg &= ~I40E_VF_ARQLEN1_ARQVFE_MASK;
 		aq_error = true;
 	}
 	if (reg & I40E_VF_ARQLEN1_ARQOVFL_MASK) {
 		device_printf(dev, "ARQ Overflow Error detected\n");
 		reg &= ~I40E_VF_ARQLEN1_ARQOVFL_MASK;
 		aq_error = true;
 	}
 	if (reg & I40E_VF_ARQLEN1_ARQCRIT_MASK) {
 		device_printf(dev, "ARQ Critical Error detected\n");
 		reg &= ~I40E_VF_ARQLEN1_ARQCRIT_MASK;
 		aq_error = true;
 	}
 	if (oldreg != reg)
 		wr32(hw, hw->aq.arq.len, reg);
 
 	oldreg = reg = rd32(hw, hw->aq.asq.len);
 	if (reg & I40E_VF_ATQLEN1_ATQVFE_MASK) {
 		device_printf(dev, "ASQ VF Error detected\n");
 		reg &= ~I40E_VF_ATQLEN1_ATQVFE_MASK;
 		aq_error = true;
 	}
 	if (reg & I40E_VF_ATQLEN1_ATQOVFL_MASK) {
 		device_printf(dev, "ASQ Overflow Error detected\n");
 		reg &= ~I40E_VF_ATQLEN1_ATQOVFL_MASK;
 		aq_error = true;
 	}
 	if (reg & I40E_VF_ATQLEN1_ATQCRIT_MASK) {
 		device_printf(dev, "ASQ Critical Error detected\n");
 		reg &= ~I40E_VF_ATQLEN1_ATQCRIT_MASK;
 		aq_error = true;
 	}
 	if (oldreg != reg)
 		wr32(hw, hw->aq.asq.len, reg);
 
 	if (aq_error) {
 		device_printf(dev, "WARNING: Stopping VF!\n");
 		/*
 		 * A VF reset might not be enough to fix a problem here;
 		 * a PF reset could be required.
 		 */
 		sc->init_state = IAVF_RESET_REQUIRED;
 		iavf_stop(sc);
 		iavf_request_reset(sc);
 	}
 
 	return (aq_error ? EIO : 0);
 }
 
 static enum i40e_status_code
 iavf_process_adminq(struct iavf_sc *sc, u16 *pending)
 {
 	enum i40e_status_code status = I40E_SUCCESS;
 	struct i40e_arq_event_info event;
 	struct i40e_hw *hw = &sc->hw;
 	struct virtchnl_msg *v_msg;
 	int error = 0, loop = 0;
 	u32 reg;
 
 	error = iavf_check_aq_errors(sc);
 	if (error)
 		return (I40E_ERR_ADMIN_QUEUE_CRITICAL_ERROR);
 
 	event.buf_len = IXL_AQ_BUF_SZ;
         event.msg_buf = sc->aq_buffer;
 	bzero(event.msg_buf, IXL_AQ_BUF_SZ);
 	v_msg = (struct virtchnl_msg *)&event.desc;
 
 	/* clean and process any events */
 	do {
 		status = i40e_clean_arq_element(hw, &event, pending);
 		/*
 		 * Also covers normal case when i40e_clean_arq_element()
 		 * returns "I40E_ERR_ADMIN_QUEUE_NO_WORK"
 		 */
 		if (status)
 			break;
 		iavf_vc_completion(sc, v_msg->v_opcode,
 		    v_msg->v_retval, event.msg_buf, event.msg_len);
 		bzero(event.msg_buf, IXL_AQ_BUF_SZ);
 	} while (*pending && (loop++ < IXL_ADM_LIMIT));
 
 	/* Re-enable admin queue interrupt cause */
 	reg = rd32(hw, I40E_VFINT_ICR0_ENA1);
 	reg |= I40E_VFINT_ICR0_ENA1_ADMINQ_MASK;
 	wr32(hw, I40E_VFINT_ICR0_ENA1, reg);
 
 	return (status);
 }
 
 static void
 iavf_if_update_admin_status(if_ctx_t ctx)
 {
 	struct iavf_sc *sc = iflib_get_softc(ctx);
 	struct i40e_hw *hw = &sc->hw;
 	u16 pending;
 
 	iavf_process_adminq(sc, &pending);
 	iavf_update_link_status(sc);
 	
 	/*
 	 * If there are still messages to process, reschedule.
 	 * Otherwise, re-enable the Admin Queue interrupt.
 	 */
 	if (pending > 0)
 		iflib_admin_intr_deferred(ctx);
 	else
 		iavf_enable_adminq_irq(hw);
 }
 
 static int
 iavf_mc_filter_apply(void *arg, struct ifmultiaddr *ifma, int count __unused)
 {
 	struct iavf_sc *sc = arg;
 	int error = 0;
 
 	if (ifma->ifma_addr->sa_family != AF_LINK)
 		return (0);
 	error = iavf_add_mac_filter(sc,
 	    (u8*)LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
 	    IXL_FILTER_MC);
 
 	return (!error);
 }
 
 static void
 iavf_if_multi_set(if_ctx_t ctx)
 {
 	struct iavf_sc *sc = iflib_get_softc(ctx);
 	int mcnt = 0;
 
 	IOCTL_DEBUGOUT("iavf_if_multi_set: begin");
 
 	mcnt = if_multiaddr_count(iflib_get_ifp(ctx), MAX_MULTICAST_ADDR);
 	if (__predict_false(mcnt == MAX_MULTICAST_ADDR)) {
 		/* Delete MC filters and enable mulitcast promisc instead */
 		iavf_init_multi(sc);
 		sc->promisc_flags |= FLAG_VF_MULTICAST_PROMISC;
 		iavf_send_vc_msg(sc, IAVF_FLAG_AQ_CONFIGURE_PROMISC);
 		return;
 	}
 
 	/* If there aren't too many filters, delete existing MC filters */
 	iavf_init_multi(sc);
 
 	/* And (re-)install filters for all mcast addresses */
 	mcnt = if_multi_apply(iflib_get_ifp(ctx), iavf_mc_filter_apply, sc);
 
 	if (mcnt > 0)
 		iavf_send_vc_msg(sc, IAVF_FLAG_AQ_ADD_MAC_FILTER);
 }
 
 static int
 iavf_if_mtu_set(if_ctx_t ctx, uint32_t mtu)
 {
 	struct iavf_sc *sc = iflib_get_softc(ctx);
 	struct ixl_vsi *vsi = &sc->vsi;
 
 	IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
 	if (mtu > IXL_MAX_FRAME - ETHER_HDR_LEN - ETHER_CRC_LEN -
 		ETHER_VLAN_ENCAP_LEN)
 		return (EINVAL);
 
 	vsi->shared->isc_max_frame_size = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN +
 		ETHER_VLAN_ENCAP_LEN;
 
 	return (0);
 }
 
 static void
 iavf_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr)
 {
 #ifdef IXL_DEBUG
 	struct ifnet *ifp = iflib_get_ifp(ctx);
 #endif
 	struct iavf_sc *sc = iflib_get_softc(ctx);
 
 	INIT_DBG_IF(ifp, "begin");
 
 	iavf_update_link_status(sc);
 
 	ifmr->ifm_status = IFM_AVALID;
 	ifmr->ifm_active = IFM_ETHER;
 
 	if (!sc->link_up)
 		return;
 
 	ifmr->ifm_status |= IFM_ACTIVE;
 	/* Hardware is always full-duplex */
 	ifmr->ifm_active |= IFM_FDX;
 
 	/* Based on the link speed reported by the PF over the AdminQ, choose a
 	 * PHY type to report. This isn't 100% correct since we don't really
 	 * know the underlying PHY type of the PF, but at least we can report
 	 * a valid link speed...
 	 */
 	switch (sc->link_speed) {
 	case VIRTCHNL_LINK_SPEED_100MB:
 		ifmr->ifm_active |= IFM_100_TX;
 		break;
 	case VIRTCHNL_LINK_SPEED_1GB:
 		ifmr->ifm_active |= IFM_1000_T;
 		break;
 	case VIRTCHNL_LINK_SPEED_10GB:
 		ifmr->ifm_active |= IFM_10G_SR;
 		break;
 	case VIRTCHNL_LINK_SPEED_20GB:
 	case VIRTCHNL_LINK_SPEED_25GB:
 		ifmr->ifm_active |= IFM_25G_SR;
 		break;
 	case VIRTCHNL_LINK_SPEED_40GB:
 		ifmr->ifm_active |= IFM_40G_SR4;
 		break;
 	default:
 		ifmr->ifm_active |= IFM_UNKNOWN;
 		break;
 	}
 
 	INIT_DBG_IF(ifp, "end");
 }
 
 static int
 iavf_if_media_change(if_ctx_t ctx)
 {
 	struct ifmedia *ifm = iflib_get_media(ctx);
 
 	INIT_DEBUGOUT("ixl_media_change: begin");
 
 	if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
 		return (EINVAL);
 
 	if_printf(iflib_get_ifp(ctx), "Media change is not supported.\n");
 	return (ENODEV);
 }
 
 static int
 iavf_if_promisc_set(if_ctx_t ctx, int flags)
 {
 	struct iavf_sc *sc = iflib_get_softc(ctx);
 	struct ifnet	*ifp = iflib_get_ifp(ctx);
 
 	sc->promisc_flags = 0;
 
 	if (flags & IFF_ALLMULTI ||
 		if_multiaddr_count(ifp, MAX_MULTICAST_ADDR) == MAX_MULTICAST_ADDR)
 		sc->promisc_flags |= FLAG_VF_MULTICAST_PROMISC;
 	if (flags & IFF_PROMISC)
 		sc->promisc_flags |= FLAG_VF_UNICAST_PROMISC;
 
 	iavf_send_vc_msg(sc, IAVF_FLAG_AQ_CONFIGURE_PROMISC);
 
 	return (0);
 }
 
 static void
 iavf_if_timer(if_ctx_t ctx, uint16_t qid)
 {
 	struct iavf_sc *sc = iflib_get_softc(ctx);
 	struct i40e_hw *hw = &sc->hw;
 	u32 val;
 
 	if (qid != 0)
 		return;
 
 	/* Check for when PF triggers a VF reset */
 	val = rd32(hw, I40E_VFGEN_RSTAT) &
 	    I40E_VFGEN_RSTAT_VFR_STATE_MASK;
 	if (val != VIRTCHNL_VFR_VFACTIVE
 	    && val != VIRTCHNL_VFR_COMPLETED) {
 		iavf_dbg_info(sc, "reset in progress! (%d)\n", val);
 		return;
 	}
 
 	/* Fire off the adminq task */
 	iflib_admin_intr_deferred(ctx);
 
 	/* Update stats */
 	iavf_request_stats(sc);
 }
 
 static void
 iavf_if_vlan_register(if_ctx_t ctx, u16 vtag)
 {
 	struct iavf_sc *sc = iflib_get_softc(ctx);
 	struct ixl_vsi *vsi = &sc->vsi;
 	struct iavf_vlan_filter	*v;
 
 	if ((vtag == 0) || (vtag > 4095))	/* Invalid */
 		return;
 
 	++vsi->num_vlans;
 	v = malloc(sizeof(struct iavf_vlan_filter), M_IAVF, M_WAITOK | M_ZERO);
 	SLIST_INSERT_HEAD(sc->vlan_filters, v, next);
 	v->vlan = vtag;
 	v->flags = IXL_FILTER_ADD;
 
 	iavf_send_vc_msg(sc, IAVF_FLAG_AQ_ADD_VLAN_FILTER);
 }
 
 static void
 iavf_if_vlan_unregister(if_ctx_t ctx, u16 vtag)
 {
 	struct iavf_sc *sc = iflib_get_softc(ctx);
 	struct ixl_vsi *vsi = &sc->vsi;
 	struct iavf_vlan_filter	*v;
 	int			i = 0;
 
 	if ((vtag == 0) || (vtag > 4095))	/* Invalid */
 		return;
 
 	SLIST_FOREACH(v, sc->vlan_filters, next) {
 		if (v->vlan == vtag) {
 			v->flags = IXL_FILTER_DEL;
 			++i;
 			--vsi->num_vlans;
 		}
 	}
 	if (i)
 		iavf_send_vc_msg(sc, IAVF_FLAG_AQ_DEL_VLAN_FILTER);
 }
 
 static uint64_t
 iavf_if_get_counter(if_ctx_t ctx, ift_counter cnt)
 {
 	struct iavf_sc *sc = iflib_get_softc(ctx);
 	struct ixl_vsi *vsi = &sc->vsi;
 	if_t ifp = iflib_get_ifp(ctx);
 
 	switch (cnt) {
 	case IFCOUNTER_IPACKETS:
 		return (vsi->ipackets);
 	case IFCOUNTER_IERRORS:
 		return (vsi->ierrors);
 	case IFCOUNTER_OPACKETS:
 		return (vsi->opackets);
 	case IFCOUNTER_OERRORS:
 		return (vsi->oerrors);
 	case IFCOUNTER_COLLISIONS:
 		/* Collisions are by standard impossible in 40G/10G Ethernet */
 		return (0);
 	case IFCOUNTER_IBYTES:
 		return (vsi->ibytes);
 	case IFCOUNTER_OBYTES:
 		return (vsi->obytes);
 	case IFCOUNTER_IMCASTS:
 		return (vsi->imcasts);
 	case IFCOUNTER_OMCASTS:
 		return (vsi->omcasts);
 	case IFCOUNTER_IQDROPS:
 		return (vsi->iqdrops);
 	case IFCOUNTER_OQDROPS:
 		return (vsi->oqdrops);
 	case IFCOUNTER_NOPROTO:
 		return (vsi->noproto);
 	default:
 		return (if_get_counter_default(ifp, cnt));
 	}
 }
 
  
 static void
 iavf_free_pci_resources(struct iavf_sc *sc)
 {
 	struct ixl_vsi		*vsi = &sc->vsi;
 	struct ixl_rx_queue	*rx_que = vsi->rx_queues;
 	device_t                dev = sc->dev;
 
 	/* We may get here before stations are set up */
 	if (rx_que == NULL)
 		goto early;
 
 	/* Release all interrupts */
 	iflib_irq_free(vsi->ctx, &vsi->irq);
 
 	for (int i = 0; i < vsi->num_rx_queues; i++, rx_que++)
 		iflib_irq_free(vsi->ctx, &rx_que->que_irq);
 
 early:
 	if (sc->pci_mem != NULL)
 		bus_release_resource(dev, SYS_RES_MEMORY,
 		    rman_get_rid(sc->pci_mem), sc->pci_mem);
 }
 
 
 /*
 ** Requests a VF reset from the PF.
 **
 ** Requires the VF's Admin Queue to be initialized.
 */
 static int
 iavf_reset(struct iavf_sc *sc)
 {
 	struct i40e_hw	*hw = &sc->hw;
 	device_t	dev = sc->dev;
 	int		error = 0;
 
 	/* Ask the PF to reset us if we are initiating */
 	if (sc->init_state != IAVF_RESET_PENDING)
 		iavf_request_reset(sc);
 
 	i40e_msec_pause(100);
 	error = iavf_reset_complete(hw);
 	if (error) {
 		device_printf(dev, "%s: VF reset failed\n",
 		    __func__);
 		return (error);
 	}
 	pci_enable_busmaster(dev);
 
 	error = i40e_shutdown_adminq(hw);
 	if (error) {
 		device_printf(dev, "%s: shutdown_adminq failed: %d\n",
 		    __func__, error);
 		return (error);
 	}
 
 	error = i40e_init_adminq(hw);
 	if (error) {
 		device_printf(dev, "%s: init_adminq failed: %d\n",
 		    __func__, error);
 		return (error);
 	}
 
 	iavf_enable_adminq_irq(hw);
 	return (0);
 }
 
 static int
 iavf_reset_complete(struct i40e_hw *hw)
 {
 	u32 reg;
 
 	/* Wait up to ~10 seconds */
 	for (int i = 0; i < 100; i++) {
 		reg = rd32(hw, I40E_VFGEN_RSTAT) &
 		    I40E_VFGEN_RSTAT_VFR_STATE_MASK;
 
                 if ((reg == VIRTCHNL_VFR_VFACTIVE) ||
 		    (reg == VIRTCHNL_VFR_COMPLETED))
 			return (0);
 		i40e_msec_pause(100);
 	}
 
 	return (EBUSY);
 }
 
 static void
 iavf_setup_interface(device_t dev, struct iavf_sc *sc)
 {
 	struct ixl_vsi *vsi = &sc->vsi;
 	if_ctx_t ctx = vsi->ctx;
 	struct ifnet *ifp = iflib_get_ifp(ctx);
 
 	INIT_DBG_DEV(dev, "begin");
 
 	vsi->shared->isc_max_frame_size =
 	    ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN
 	    + ETHER_VLAN_ENCAP_LEN;
 #if __FreeBSD_version >= 1100000
 	if_setbaudrate(ifp, IF_Gbps(40));
 #else
 	if_initbaudrate(ifp, IF_Gbps(40));
 #endif
 
 	ifmedia_add(vsi->media, IFM_ETHER | IFM_AUTO, 0, NULL);
 	ifmedia_set(vsi->media, IFM_ETHER | IFM_AUTO);
 }
 
 /*
 ** Get a new filter and add it to the mac filter list.
 */
 static struct iavf_mac_filter *
 iavf_get_mac_filter(struct iavf_sc *sc)
 {
 	struct iavf_mac_filter	*f;
 
 	f = malloc(sizeof(struct iavf_mac_filter),
 	    M_IAVF, M_NOWAIT | M_ZERO);
 	if (f)
 		SLIST_INSERT_HEAD(sc->mac_filters, f, next);
 
 	return (f);
 }
 
 /*
 ** Find the filter with matching MAC address
 */
 static struct iavf_mac_filter *
 iavf_find_mac_filter(struct iavf_sc *sc, u8 *macaddr)
 {
 	struct iavf_mac_filter	*f;
 	bool match = FALSE;
 
 	SLIST_FOREACH(f, sc->mac_filters, next) {
 		if (cmp_etheraddr(f->macaddr, macaddr)) {
 			match = TRUE;
 			break;
 		}
 	}	
 
 	if (!match)
 		f = NULL;
 	return (f);
 }
 
 /*
 ** Admin Queue interrupt handler
 */
 static int
 iavf_msix_adminq(void *arg)
 {
 	struct iavf_sc	*sc = arg;
 	struct i40e_hw	*hw = &sc->hw;
 	u32		reg, mask;
 	bool		do_task = FALSE;
 
 	++sc->admin_irq;
 
         reg = rd32(hw, I40E_VFINT_ICR01);
 	/*
 	 * For masking off interrupt causes that need to be handled before
 	 * they can be re-enabled
 	 */
         mask = rd32(hw, I40E_VFINT_ICR0_ENA1);
 
 	/* Check on the cause */
 	if (reg & I40E_VFINT_ICR0_ADMINQ_MASK) {
 		mask &= ~I40E_VFINT_ICR0_ENA_ADMINQ_MASK;
 		do_task = TRUE;
 	}
 
 	wr32(hw, I40E_VFINT_ICR0_ENA1, mask);
 	iavf_enable_adminq_irq(hw);
 
 	if (do_task)
 		return (FILTER_SCHEDULE_THREAD);
 	else
 		return (FILTER_HANDLED);
 }
 
 void
 iavf_enable_intr(struct ixl_vsi *vsi)
 {
 	struct i40e_hw *hw = vsi->hw;
 	struct ixl_rx_queue *que = vsi->rx_queues;
 
 	iavf_enable_adminq_irq(hw);
 	for (int i = 0; i < vsi->num_rx_queues; i++, que++)
 		iavf_enable_queue_irq(hw, que->rxr.me);
 }
 
 void
 iavf_disable_intr(struct ixl_vsi *vsi)
 {
         struct i40e_hw *hw = vsi->hw;
         struct ixl_rx_queue *que = vsi->rx_queues;
 
 	for (int i = 0; i < vsi->num_rx_queues; i++, que++)
 		iavf_disable_queue_irq(hw, que->rxr.me);
 }
 
 static void
 iavf_disable_adminq_irq(struct i40e_hw *hw)
 {
 	wr32(hw, I40E_VFINT_DYN_CTL01, 0);
 	wr32(hw, I40E_VFINT_ICR0_ENA1, 0);
 	/* flush */
 	rd32(hw, I40E_VFGEN_RSTAT);
 }
 
 static void
 iavf_enable_adminq_irq(struct i40e_hw *hw)
 {
 	wr32(hw, I40E_VFINT_DYN_CTL01,
 	    I40E_VFINT_DYN_CTL01_INTENA_MASK |
 	    I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
 	wr32(hw, I40E_VFINT_ICR0_ENA1, I40E_VFINT_ICR0_ENA1_ADMINQ_MASK);
 	/* flush */
 	rd32(hw, I40E_VFGEN_RSTAT);
 }
 
 static void
 iavf_enable_queue_irq(struct i40e_hw *hw, int id)
 {
 	u32		reg;
 
 	reg = I40E_VFINT_DYN_CTLN1_INTENA_MASK |
 	    I40E_VFINT_DYN_CTLN1_CLEARPBA_MASK |
 	    I40E_VFINT_DYN_CTLN1_ITR_INDX_MASK;
 	wr32(hw, I40E_VFINT_DYN_CTLN1(id), reg);
 }
 
 static void
 iavf_disable_queue_irq(struct i40e_hw *hw, int id)
 {
 	wr32(hw, I40E_VFINT_DYN_CTLN1(id),
 	    I40E_VFINT_DYN_CTLN1_ITR_INDX_MASK);
 	rd32(hw, I40E_VFGEN_RSTAT);
 }
 
 static void
 iavf_configure_tx_itr(struct iavf_sc *sc)
 {
 	struct i40e_hw		*hw = &sc->hw;
 	struct ixl_vsi		*vsi = &sc->vsi;
 	struct ixl_tx_queue	*que = vsi->tx_queues;
 
 	vsi->tx_itr_setting = sc->tx_itr;
 
 	for (int i = 0; i < vsi->num_tx_queues; i++, que++) {
 		struct tx_ring	*txr = &que->txr;
 
 		wr32(hw, I40E_VFINT_ITRN1(IXL_TX_ITR, i),
 		    vsi->tx_itr_setting);
 		txr->itr = vsi->tx_itr_setting;
 		txr->latency = IXL_AVE_LATENCY;
 	}
 }
 
 static void
 iavf_configure_rx_itr(struct iavf_sc *sc)
 {
 	struct i40e_hw		*hw = &sc->hw;
 	struct ixl_vsi		*vsi = &sc->vsi;
 	struct ixl_rx_queue	*que = vsi->rx_queues;
 
 	vsi->rx_itr_setting = sc->rx_itr;
 
 	for (int i = 0; i < vsi->num_rx_queues; i++, que++) {
 		struct rx_ring 	*rxr = &que->rxr;
 
 		wr32(hw, I40E_VFINT_ITRN1(IXL_RX_ITR, i),
 		    vsi->rx_itr_setting);
 		rxr->itr = vsi->rx_itr_setting;
 		rxr->latency = IXL_AVE_LATENCY;
 	}
 }
 
 /*
  * Get initial ITR values from tunable values.
  */
 static void
 iavf_configure_itr(struct iavf_sc *sc)
 {
 	iavf_configure_tx_itr(sc);
 	iavf_configure_rx_itr(sc);
 }
 
 /*
 ** Provide a update to the queue RX
 ** interrupt moderation value.
 */
 static void
 iavf_set_queue_rx_itr(struct ixl_rx_queue *que)
 {
 	struct ixl_vsi	*vsi = que->vsi;
 	struct i40e_hw	*hw = vsi->hw;
 	struct rx_ring	*rxr = &que->rxr;
 
 	/* Idle, do nothing */
 	if (rxr->bytes == 0)
 		return;
 
 	/* Update the hardware if needed */
 	if (rxr->itr != vsi->rx_itr_setting) {
 		rxr->itr = vsi->rx_itr_setting;
 		wr32(hw, I40E_VFINT_ITRN1(IXL_RX_ITR,
 		    que->rxr.me), rxr->itr);
 	}
 }
 
 static int
 iavf_msix_que(void *arg)
 {
 	struct ixl_rx_queue *rx_que = arg;
 
 	++rx_que->irqs;
 
 	iavf_set_queue_rx_itr(rx_que);
 	// iavf_set_queue_tx_itr(que);
 
 	return (FILTER_SCHEDULE_THREAD);
 }
 
 /*********************************************************************
  *  Multicast Initialization
  *
  *  This routine is called by init to reset a fresh state.
  *
  **********************************************************************/
 static void
 iavf_init_multi(struct iavf_sc *sc)
 {
 	struct iavf_mac_filter *f;
 	int mcnt = 0;
 
 	/* First clear any multicast filters */
 	SLIST_FOREACH(f, sc->mac_filters, next) {
 		if ((f->flags & IXL_FILTER_USED)
 		    && (f->flags & IXL_FILTER_MC)) {
 			f->flags |= IXL_FILTER_DEL;
 			mcnt++;
 		}
 	}
 	if (mcnt > 0)
 		iavf_send_vc_msg(sc, IAVF_FLAG_AQ_DEL_MAC_FILTER);
 }
 
 /*
 ** Note: this routine updates the OS on the link state
 **	the real check of the hardware only happens with
 **	a link interrupt.
 */
 void
 iavf_update_link_status(struct iavf_sc *sc)
 {
 	struct ixl_vsi *vsi = &sc->vsi;
 	u64 baudrate;
 
 	if (sc->link_up){ 
 		if (vsi->link_active == FALSE) {
 			vsi->link_active = TRUE;
 			baudrate = ixl_max_vc_speed_to_value(sc->link_speed);
 			iavf_dbg_info(sc, "baudrate: %lu\n", baudrate);
 			iflib_link_state_change(vsi->ctx, LINK_STATE_UP, baudrate);
 		}
 	} else { /* Link down */
 		if (vsi->link_active == TRUE) {
 			vsi->link_active = FALSE;
 			iflib_link_state_change(vsi->ctx, LINK_STATE_DOWN, 0);
 		}
 	}
 }
 
 /*********************************************************************
  *
  *  This routine disables all traffic on the adapter by issuing a
  *  global reset on the MAC and deallocates TX/RX buffers.
  *
  **********************************************************************/
 
 static void
 iavf_stop(struct iavf_sc *sc)
 {
 	struct ifnet *ifp;
 
 	ifp = sc->vsi.ifp;
 
 	iavf_disable_intr(&sc->vsi);
 
 	if (atomic_load_acq_32(&sc->queues_enabled))
 		iavf_send_vc_msg_sleep(sc, IAVF_FLAG_AQ_DISABLE_QUEUES);
 }
 
 static void
 iavf_if_stop(if_ctx_t ctx)
 {
 	struct iavf_sc *sc = iflib_get_softc(ctx);
 
 	iavf_stop(sc);
 }
 
 static void
 iavf_config_rss_reg(struct iavf_sc *sc)
 {
 	struct i40e_hw	*hw = &sc->hw;
 	struct ixl_vsi	*vsi = &sc->vsi;
 	u32		lut = 0;
 	u64		set_hena = 0, hena;
 	int		i, j, que_id;
 	u32		rss_seed[IXL_RSS_KEY_SIZE_REG];
 #ifdef RSS
 	u32		rss_hash_config;
 #endif
         
 	/* Don't set up RSS if using a single queue */
 	if (vsi->num_rx_queues == 1) {
 		wr32(hw, I40E_VFQF_HENA(0), 0);
 		wr32(hw, I40E_VFQF_HENA(1), 0);
 		ixl_flush(hw);
 		return;
 	}
 
 #ifdef RSS
 	/* Fetch the configured RSS key */
 	rss_getkey((uint8_t *) &rss_seed);
 #else
 	ixl_get_default_rss_key(rss_seed);
 #endif
 
 	/* Fill out hash function seed */
 	for (i = 0; i < IXL_RSS_KEY_SIZE_REG; i++)
                 wr32(hw, I40E_VFQF_HKEY(i), rss_seed[i]);
 
 	/* Enable PCTYPES for RSS: */
 #ifdef RSS
 	rss_hash_config = rss_gethashconfig();
 	if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4)
                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER);
 	if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4)
                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP);
 	if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4)
                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP);
 	if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6)
                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER);
 	if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX)
 		set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6);
 	if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6)
                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP);
         if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6)
                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP);
 #else
 	set_hena = IXL_DEFAULT_RSS_HENA_XL710;
 #endif
 	hena = (u64)rd32(hw, I40E_VFQF_HENA(0)) |
 	    ((u64)rd32(hw, I40E_VFQF_HENA(1)) << 32);
 	hena |= set_hena;
 	wr32(hw, I40E_VFQF_HENA(0), (u32)hena);
 	wr32(hw, I40E_VFQF_HENA(1), (u32)(hena >> 32));
 
 	/* Populate the LUT with max no. of queues in round robin fashion */
 	for (i = 0, j = 0; i < IXL_RSS_VSI_LUT_SIZE; i++, j++) {
                 if (j == vsi->num_rx_queues)
                         j = 0;
 #ifdef RSS
 		/*
 		 * Fetch the RSS bucket id for the given indirection entry.
 		 * Cap it at the number of configured buckets (which is
 		 * num_rx_queues.)
 		 */
 		que_id = rss_get_indirection_to_bucket(i);
 		que_id = que_id % vsi->num_rx_queues;
 #else
 		que_id = j;
 #endif
                 /* lut = 4-byte sliding window of 4 lut entries */
                 lut = (lut << 8) | (que_id & IXL_RSS_VF_LUT_ENTRY_MASK);
                 /* On i = 3, we have 4 entries in lut; write to the register */
                 if ((i & 3) == 3) {
                         wr32(hw, I40E_VFQF_HLUT(i >> 2), lut);
 			DDPRINTF(sc->dev, "HLUT(%2d): %#010x", i, lut);
 		}
         }
 	ixl_flush(hw);
 }
 
 static void
 iavf_config_rss_pf(struct iavf_sc *sc)
 {
 	iavf_send_vc_msg(sc, IAVF_FLAG_AQ_CONFIG_RSS_KEY);
 
 	iavf_send_vc_msg(sc, IAVF_FLAG_AQ_SET_RSS_HENA);
 
 	iavf_send_vc_msg(sc, IAVF_FLAG_AQ_CONFIG_RSS_LUT);
 }
 
 /*
 ** iavf_config_rss - setup RSS 
 **
 ** RSS keys and table are cleared on VF reset.
 */
 static void
 iavf_config_rss(struct iavf_sc *sc)
 {
 	if (sc->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_REG) {
 		iavf_dbg_info(sc, "Setting up RSS using VF registers...");
 		iavf_config_rss_reg(sc);
 	} else if (sc->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) {
 		iavf_dbg_info(sc, "Setting up RSS using messages to PF...");
 		iavf_config_rss_pf(sc);
 	} else
 		device_printf(sc->dev, "VF does not support RSS capability sent by PF.\n");
 }
 
 /*
 ** This routine adds new MAC filters to the sc's list;
 ** these are later added in hardware by sending a virtual
 ** channel message.
 */
 static int
 iavf_add_mac_filter(struct iavf_sc *sc, u8 *macaddr, u16 flags)
 {
 	struct iavf_mac_filter	*f;
 
 	/* Does one already exist? */
 	f = iavf_find_mac_filter(sc, macaddr);
 	if (f != NULL) {
 		iavf_dbg_filter(sc, "exists: " MAC_FORMAT "\n",
 		    MAC_FORMAT_ARGS(macaddr));
 		return (EEXIST);
 	}
 
 	/* If not, get a new empty filter */
 	f = iavf_get_mac_filter(sc);
 	if (f == NULL) {
 		device_printf(sc->dev, "%s: no filters available!!\n",
 		    __func__);
 		return (ENOMEM);
 	}
 
 	iavf_dbg_filter(sc, "marked: " MAC_FORMAT "\n",
 	    MAC_FORMAT_ARGS(macaddr));
 
 	bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN);
 	f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED);
 	f->flags |= flags;
 	return (0);
 }
 
 /*
 ** Marks a MAC filter for deletion.
 */
 static int
 iavf_del_mac_filter(struct iavf_sc *sc, u8 *macaddr)
 {
 	struct iavf_mac_filter	*f;
 
 	f = iavf_find_mac_filter(sc, macaddr);
 	if (f == NULL)
 		return (ENOENT);
 
 	f->flags |= IXL_FILTER_DEL;
 	return (0);
 }
 
 /*
  * Re-uses the name from the PF driver.
  */
 static void
 iavf_add_device_sysctls(struct iavf_sc *sc)
 {
 	struct ixl_vsi *vsi = &sc->vsi;
 	device_t dev = sc->dev;
 
 	struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
 	struct sysctl_oid_list *ctx_list =
 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev));
 	struct sysctl_oid *debug_node;
 	struct sysctl_oid_list *debug_list;
 
 	SYSCTL_ADD_PROC(ctx, ctx_list,
 	    OID_AUTO, "current_speed", CTLTYPE_STRING | CTLFLAG_RD,
 	    sc, 0, iavf_sysctl_current_speed, "A", "Current Port Speed");
 
 	SYSCTL_ADD_PROC(ctx, ctx_list,
 	    OID_AUTO, "tx_itr", CTLTYPE_INT | CTLFLAG_RW,
 	    sc, 0, iavf_sysctl_tx_itr, "I",
 	    "Immediately set TX ITR value for all queues");
 
 	SYSCTL_ADD_PROC(ctx, ctx_list,
 	    OID_AUTO, "rx_itr", CTLTYPE_INT | CTLFLAG_RW,
 	    sc, 0, iavf_sysctl_rx_itr, "I",
 	    "Immediately set RX ITR value for all queues");
 
 	/* Add sysctls meant to print debug information, but don't list them
 	 * in "sysctl -a" output. */
 	debug_node = SYSCTL_ADD_NODE(ctx, ctx_list,
 	    OID_AUTO, "debug", CTLFLAG_RD | CTLFLAG_SKIP, NULL, "Debug Sysctls");
 	debug_list = SYSCTL_CHILDREN(debug_node);
 
 	SYSCTL_ADD_UINT(ctx, debug_list,
 	    OID_AUTO, "shared_debug_mask", CTLFLAG_RW,
 	    &sc->hw.debug_mask, 0, "Shared code debug message level");
 
 	SYSCTL_ADD_UINT(ctx, debug_list,
 	    OID_AUTO, "core_debug_mask", CTLFLAG_RW,
 	    &sc->dbg_mask, 0, "Non-shared code debug message level");
 
 	SYSCTL_ADD_PROC(ctx, debug_list,
 	    OID_AUTO, "filter_list", CTLTYPE_STRING | CTLFLAG_RD,
 	    sc, 0, iavf_sysctl_sw_filter_list, "A", "SW Filter List");
 
 	SYSCTL_ADD_PROC(ctx, debug_list,
 	    OID_AUTO, "queue_interrupt_table", CTLTYPE_STRING | CTLFLAG_RD,
 	    sc, 0, iavf_sysctl_queue_interrupt_table, "A", "View MSI-X indices for TX/RX queues");
 
 	SYSCTL_ADD_PROC(ctx, debug_list,
 	    OID_AUTO, "do_vf_reset", CTLTYPE_INT | CTLFLAG_WR,
 	    sc, 0, iavf_sysctl_vf_reset, "A", "Request a VF reset from PF");
 
 	SYSCTL_ADD_PROC(ctx, debug_list,
 	    OID_AUTO, "do_vflr_reset", CTLTYPE_INT | CTLFLAG_WR,
 	    sc, 0, iavf_sysctl_vflr_reset, "A", "Request a VFLR reset from HW");
 
 	/* Add stats sysctls */
 	ixl_add_vsi_sysctls(dev, vsi, ctx, "vsi");
 	ixl_add_queues_sysctls(dev, vsi);
 
 }
 
 static void
 iavf_init_filters(struct iavf_sc *sc)
 {
 	sc->mac_filters = malloc(sizeof(struct mac_list),
 	    M_IAVF, M_WAITOK | M_ZERO);
 	SLIST_INIT(sc->mac_filters);
 	sc->vlan_filters = malloc(sizeof(struct vlan_list),
 	    M_IAVF, M_WAITOK | M_ZERO);
 	SLIST_INIT(sc->vlan_filters);
 }
 
 static void
 iavf_free_filters(struct iavf_sc *sc)
 {
 	struct iavf_mac_filter *f;
 	struct iavf_vlan_filter *v;
 
 	while (!SLIST_EMPTY(sc->mac_filters)) {
 		f = SLIST_FIRST(sc->mac_filters);
 		SLIST_REMOVE_HEAD(sc->mac_filters, next);
 		free(f, M_IAVF);
 	}
 	free(sc->mac_filters, M_IAVF);
 	while (!SLIST_EMPTY(sc->vlan_filters)) {
 		v = SLIST_FIRST(sc->vlan_filters);
 		SLIST_REMOVE_HEAD(sc->vlan_filters, next);
 		free(v, M_IAVF);
 	}
 	free(sc->vlan_filters, M_IAVF);
 }
 
 char *
 iavf_vc_speed_to_string(enum virtchnl_link_speed link_speed)
 {
 	int index;
 
 	char *speeds[] = {
 		"Unknown",
 		"100 Mbps",
 		"1 Gbps",
 		"10 Gbps",
 		"40 Gbps",
 		"20 Gbps",
 		"25 Gbps",
 	};
 
 	switch (link_speed) {
 	case VIRTCHNL_LINK_SPEED_100MB:
 		index = 1;
 		break;
 	case VIRTCHNL_LINK_SPEED_1GB:
 		index = 2;
 		break;
 	case VIRTCHNL_LINK_SPEED_10GB:
 		index = 3;
 		break;
 	case VIRTCHNL_LINK_SPEED_40GB:
 		index = 4;
 		break;
 	case VIRTCHNL_LINK_SPEED_20GB:
 		index = 5;
 		break;
 	case VIRTCHNL_LINK_SPEED_25GB:
 		index = 6;
 		break;
 	case VIRTCHNL_LINK_SPEED_UNKNOWN:
 	default:
 		index = 0;
 		break;
 	}
 
 	return speeds[index];
 }
 
 static int
 iavf_sysctl_current_speed(SYSCTL_HANDLER_ARGS)
 {
 	struct iavf_sc *sc = (struct iavf_sc *)arg1;
 	int error = 0;
 
 	error = sysctl_handle_string(oidp,
 	  iavf_vc_speed_to_string(sc->link_speed),
 	  8, req);
 	return (error);
 }
 
 /*
  * Sanity check and save off tunable values.
  */
 static void
 iavf_save_tunables(struct iavf_sc *sc)
 {
 	device_t dev = sc->dev;
 
 	/* Save tunable information */
 	sc->dbg_mask = iavf_core_debug_mask;
 	sc->hw.debug_mask = iavf_shared_debug_mask;
 	sc->vsi.enable_head_writeback = !!(iavf_enable_head_writeback);
 
 	if (iavf_tx_itr < 0 || iavf_tx_itr > IXL_MAX_ITR) {
 		device_printf(dev, "Invalid tx_itr value of %d set!\n",
 		    iavf_tx_itr);
 		device_printf(dev, "tx_itr must be between %d and %d, "
 		    "inclusive\n",
 		    0, IXL_MAX_ITR);
 		device_printf(dev, "Using default value of %d instead\n",
 		    IXL_ITR_4K);
 		sc->tx_itr = IXL_ITR_4K;
 	} else
 		sc->tx_itr = iavf_tx_itr;
 
 	if (iavf_rx_itr < 0 || iavf_rx_itr > IXL_MAX_ITR) {
 		device_printf(dev, "Invalid rx_itr value of %d set!\n",
 		    iavf_rx_itr);
 		device_printf(dev, "rx_itr must be between %d and %d, "
 		    "inclusive\n",
 		    0, IXL_MAX_ITR);
 		device_printf(dev, "Using default value of %d instead\n",
 		    IXL_ITR_8K);
 		sc->rx_itr = IXL_ITR_8K;
 	} else
 		sc->rx_itr = iavf_rx_itr;
 }
 
 /*
  * Used to set the Tx ITR value for all of the VF's queues.
  * Writes to the ITR registers immediately.
  */
 static int
 iavf_sysctl_tx_itr(SYSCTL_HANDLER_ARGS)
 {
 	struct iavf_sc *sc = (struct iavf_sc *)arg1;
 	device_t dev = sc->dev;
 	int requested_tx_itr;
 	int error = 0;
 
 	requested_tx_itr = sc->tx_itr;
 	error = sysctl_handle_int(oidp, &requested_tx_itr, 0, req);
 	if ((error) || (req->newptr == NULL))
 		return (error);
 	if (requested_tx_itr < 0 || requested_tx_itr > IXL_MAX_ITR) {
 		device_printf(dev,
 		    "Invalid TX itr value; value must be between 0 and %d\n",
 		        IXL_MAX_ITR);
 		return (EINVAL);
 	}
 
 	sc->tx_itr = requested_tx_itr;
 	iavf_configure_tx_itr(sc);
 
 	return (error);
 }
 
 /*
  * Used to set the Rx ITR value for all of the VF's queues.
  * Writes to the ITR registers immediately.
  */
 static int
 iavf_sysctl_rx_itr(SYSCTL_HANDLER_ARGS)
 {
 	struct iavf_sc *sc = (struct iavf_sc *)arg1;
 	device_t dev = sc->dev;
 	int requested_rx_itr;
 	int error = 0;
 
 	requested_rx_itr = sc->rx_itr;
 	error = sysctl_handle_int(oidp, &requested_rx_itr, 0, req);
 	if ((error) || (req->newptr == NULL))
 		return (error);
 	if (requested_rx_itr < 0 || requested_rx_itr > IXL_MAX_ITR) {
 		device_printf(dev,
 		    "Invalid RX itr value; value must be between 0 and %d\n",
 		        IXL_MAX_ITR);
 		return (EINVAL);
 	}
 
 	sc->rx_itr = requested_rx_itr;
 	iavf_configure_rx_itr(sc);
 
 	return (error);
 }
 
 static int
 iavf_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS)
 {
 	struct iavf_sc *sc = (struct iavf_sc *)arg1;
 	struct iavf_mac_filter *f;
 	struct iavf_vlan_filter *v;
 	device_t dev = sc->dev;
 	int ftl_len, ftl_counter = 0, error = 0;
 	struct sbuf *buf;
 
 	buf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
 	if (!buf) {
 		device_printf(dev, "Could not allocate sbuf for output.\n");
 		return (ENOMEM);
 	}
 
 	sbuf_printf(buf, "\n");
 
 	/* Print MAC filters */
 	sbuf_printf(buf, "MAC Filters:\n");
 	ftl_len = 0;
 	SLIST_FOREACH(f, sc->mac_filters, next)
 		ftl_len++;
 	if (ftl_len < 1)
 		sbuf_printf(buf, "(none)\n");
 	else {
 		SLIST_FOREACH(f, sc->mac_filters, next) {
 			sbuf_printf(buf,
 			    MAC_FORMAT ", flags %#06x\n",
 			    MAC_FORMAT_ARGS(f->macaddr), f->flags);
 		}
 	}
 
 	/* Print VLAN filters */
 	sbuf_printf(buf, "VLAN Filters:\n");
 	ftl_len = 0;
 	SLIST_FOREACH(v, sc->vlan_filters, next)
 		ftl_len++;
 	if (ftl_len < 1)
 		sbuf_printf(buf, "(none)");
 	else {
 		SLIST_FOREACH(v, sc->vlan_filters, next) {
 			sbuf_printf(buf,
 			    "%d, flags %#06x",
 			    v->vlan, v->flags);
 			/* don't print '\n' for last entry */
 			if (++ftl_counter != ftl_len)
 				sbuf_printf(buf, "\n");
 		}
 	}
 
 	error = sbuf_finish(buf);
 	if (error)
 		device_printf(dev, "Error finishing sbuf: %d\n", error);
 
 	sbuf_delete(buf);
 	return (error);
 }
 
 /*
  * Print out mapping of TX queue indexes and Rx queue indexes
  * to MSI-X vectors.
  */
 static int
 iavf_sysctl_queue_interrupt_table(SYSCTL_HANDLER_ARGS)
 {
 	struct iavf_sc *sc = (struct iavf_sc *)arg1;
 	struct ixl_vsi *vsi = &sc->vsi;
 	device_t dev = sc->dev;
 	struct sbuf *buf;
 	int error = 0;
 
 	struct ixl_rx_queue *rx_que = vsi->rx_queues;
 	struct ixl_tx_queue *tx_que = vsi->tx_queues;
 
 	buf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
 	if (!buf) {
 		device_printf(dev, "Could not allocate sbuf for output.\n");
 		return (ENOMEM);
 	}
 
 	sbuf_cat(buf, "\n");
 	for (int i = 0; i < vsi->num_rx_queues; i++) {
 		rx_que = &vsi->rx_queues[i];
 		sbuf_printf(buf, "(rxq %3d): %d\n", i, rx_que->msix);
 	}
 	for (int i = 0; i < vsi->num_tx_queues; i++) {
 		tx_que = &vsi->tx_queues[i];
 		sbuf_printf(buf, "(txq %3d): %d\n", i, tx_que->msix);
 	}
 
 	error = sbuf_finish(buf);
 	if (error)
 		device_printf(dev, "Error finishing sbuf: %d\n", error);
 	sbuf_delete(buf);
 
 	return (error);
 }
 
 #define CTX_ACTIVE(ctx) ((if_getdrvflags(iflib_get_ifp(ctx)) & IFF_DRV_RUNNING))
 static int
 iavf_sysctl_vf_reset(SYSCTL_HANDLER_ARGS)
 {
 	struct iavf_sc *sc = (struct iavf_sc *)arg1;
 	int do_reset = 0, error = 0;
 
 	error = sysctl_handle_int(oidp, &do_reset, 0, req);
 	if ((error) || (req->newptr == NULL))
 		return (error);
 
 	if (do_reset == 1) {
 		iavf_reset(sc);
 		if (CTX_ACTIVE(sc->vsi.ctx))
 			iflib_request_reset(sc->vsi.ctx);
 	}
 
 	return (error);
 }
 
 static int
 iavf_sysctl_vflr_reset(SYSCTL_HANDLER_ARGS)
 {
 	struct iavf_sc *sc = (struct iavf_sc *)arg1;
 	device_t dev = sc->dev;
 	int do_reset = 0, error = 0;
 
 	error = sysctl_handle_int(oidp, &do_reset, 0, req);
 	if ((error) || (req->newptr == NULL))
 		return (error);
 
 	if (do_reset == 1) {
 		if (!pcie_flr(dev, max(pcie_get_max_completion_timeout(dev) / 1000, 10), true)) {
 			device_printf(dev, "PCIE FLR failed\n");
 			error = EIO;
 		}
 		else if (CTX_ACTIVE(sc->vsi.ctx))
 			iflib_request_reset(sc->vsi.ctx);
 	}
 
 	return (error);
 }
 #undef CTX_ACTIVE
Index: head/sys/dev/ixl/ixl_pf_main.c
===================================================================
--- head/sys/dev/ixl/ixl_pf_main.c	(revision 345304)
+++ head/sys/dev/ixl/ixl_pf_main.c	(revision 345305)
@@ -1,5073 +1,5070 @@
 /******************************************************************************
 
   Copyright (c) 2013-2018, Intel Corporation
   All rights reserved.
   
   Redistribution and use in source and binary forms, with or without 
   modification, are permitted provided that the following conditions are met:
   
    1. Redistributions of source code must retain the above copyright notice, 
       this list of conditions and the following disclaimer.
   
    2. Redistributions in binary form must reproduce the above copyright 
       notice, this list of conditions and the following disclaimer in the 
       documentation and/or other materials provided with the distribution.
   
    3. Neither the name of the Intel Corporation nor the names of its 
       contributors may be used to endorse or promote products derived from 
       this software without specific prior written permission.
   
   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 
   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 
   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 
   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 
   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 
   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 
   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 
   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
   POSSIBILITY OF SUCH DAMAGE.
 
 ******************************************************************************/
 /*$FreeBSD$*/
 
 
 #include "ixl_pf.h"
 
 #ifdef PCI_IOV
 #include "ixl_pf_iov.h"
 #endif
 
 #ifdef IXL_IW
 #include "ixl_iw.h"
 #include "ixl_iw_int.h"
 #endif
 
 static u8	ixl_convert_sysctl_aq_link_speed(u8, bool);
 static void	ixl_sbuf_print_bytes(struct sbuf *, u8 *, int, int, bool);
 
 /* Sysctls */
 static int	ixl_sysctl_set_flowcntl(SYSCTL_HANDLER_ARGS);
 static int	ixl_sysctl_set_advertise(SYSCTL_HANDLER_ARGS);
 static int	ixl_sysctl_supported_speeds(SYSCTL_HANDLER_ARGS);
 static int	ixl_sysctl_current_speed(SYSCTL_HANDLER_ARGS);
 static int	ixl_sysctl_show_fw(SYSCTL_HANDLER_ARGS);
 static int	ixl_sysctl_unallocated_queues(SYSCTL_HANDLER_ARGS);
 static int	ixl_sysctl_pf_tx_itr(SYSCTL_HANDLER_ARGS);
 static int	ixl_sysctl_pf_rx_itr(SYSCTL_HANDLER_ARGS);
 
 /* Debug Sysctls */
 static int 	ixl_sysctl_link_status(SYSCTL_HANDLER_ARGS);
 static int	ixl_sysctl_phy_abilities(SYSCTL_HANDLER_ARGS);
 static int	ixl_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS);
 static int	ixl_sysctl_hw_res_alloc(SYSCTL_HANDLER_ARGS);
 static int	ixl_sysctl_switch_config(SYSCTL_HANDLER_ARGS);
 static int	ixl_sysctl_hkey(SYSCTL_HANDLER_ARGS);
 static int	ixl_sysctl_hena(SYSCTL_HANDLER_ARGS);
 static int	ixl_sysctl_hlut(SYSCTL_HANDLER_ARGS);
 static int	ixl_sysctl_fw_link_management(SYSCTL_HANDLER_ARGS);
 static int	ixl_sysctl_read_i2c_byte(SYSCTL_HANDLER_ARGS);
 static int	ixl_sysctl_write_i2c_byte(SYSCTL_HANDLER_ARGS);
 static int	ixl_sysctl_fec_fc_ability(SYSCTL_HANDLER_ARGS);
 static int	ixl_sysctl_fec_rs_ability(SYSCTL_HANDLER_ARGS);
 static int	ixl_sysctl_fec_fc_request(SYSCTL_HANDLER_ARGS);
 static int	ixl_sysctl_fec_rs_request(SYSCTL_HANDLER_ARGS);
 static int	ixl_sysctl_fec_auto_enable(SYSCTL_HANDLER_ARGS);
 static int	ixl_sysctl_dump_debug_data(SYSCTL_HANDLER_ARGS);
 static int	ixl_sysctl_fw_lldp(SYSCTL_HANDLER_ARGS);
 static int	ixl_sysctl_do_pf_reset(SYSCTL_HANDLER_ARGS);
 static int	ixl_sysctl_do_core_reset(SYSCTL_HANDLER_ARGS);
 static int	ixl_sysctl_do_global_reset(SYSCTL_HANDLER_ARGS);
 static int	ixl_sysctl_do_emp_reset(SYSCTL_HANDLER_ARGS);
 static int	ixl_sysctl_queue_interrupt_table(SYSCTL_HANDLER_ARGS);
 static int	ixl_sysctl_read_i2c_diag_data(SYSCTL_HANDLER_ARGS);
 #ifdef IXL_DEBUG
 static int	ixl_sysctl_qtx_tail_handler(SYSCTL_HANDLER_ARGS);
 static int	ixl_sysctl_qrx_tail_handler(SYSCTL_HANDLER_ARGS);
 #endif
 
 #ifdef IXL_IW
 extern int ixl_enable_iwarp;
 extern int ixl_limit_iwarp_msix;
 #endif
 
 const uint8_t ixl_bcast_addr[ETHER_ADDR_LEN] =
     {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
 
 const char * const ixl_fc_string[6] = {
 	"None",
 	"Rx",
 	"Tx",
 	"Full",
 	"Priority",
 	"Default"
 };
 
 static char *ixl_fec_string[3] = {
        "CL108 RS-FEC",
        "CL74 FC-FEC/BASE-R",
        "None"
 };
 
 MALLOC_DEFINE(M_IXL, "ixl", "ixl driver allocations");
 
 /*
 ** Put the FW, API, NVM, EEtrackID, and OEM version information into a string
 */
 void
 ixl_nvm_version_str(struct i40e_hw *hw, struct sbuf *buf)
 {
 	u8 oem_ver = (u8)(hw->nvm.oem_ver >> 24);
 	u16 oem_build = (u16)((hw->nvm.oem_ver >> 16) & 0xFFFF);
 	u8 oem_patch = (u8)(hw->nvm.oem_ver & 0xFF);
 
 	sbuf_printf(buf,
 	    "fw %d.%d.%05d api %d.%d nvm %x.%02x etid %08x oem %d.%d.%d",
 	    hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build,
 	    hw->aq.api_maj_ver, hw->aq.api_min_ver,
 	    (hw->nvm.version & IXL_NVM_VERSION_HI_MASK) >>
 	    IXL_NVM_VERSION_HI_SHIFT,
 	    (hw->nvm.version & IXL_NVM_VERSION_LO_MASK) >>
 	    IXL_NVM_VERSION_LO_SHIFT,
 	    hw->nvm.eetrack,
 	    oem_ver, oem_build, oem_patch);
 }
 
 void
 ixl_print_nvm_version(struct ixl_pf *pf)
 {
 	struct i40e_hw *hw = &pf->hw;
 	device_t dev = pf->dev;
 	struct sbuf *sbuf;
 
 	sbuf = sbuf_new_auto();
 	ixl_nvm_version_str(hw, sbuf);
 	sbuf_finish(sbuf);
 	device_printf(dev, "%s\n", sbuf_data(sbuf));
 	sbuf_delete(sbuf);
 }
 
 static void
 ixl_configure_tx_itr(struct ixl_pf *pf)
 {
 	struct i40e_hw		*hw = &pf->hw;
 	struct ixl_vsi		*vsi = &pf->vsi;
 	struct ixl_tx_queue	*que = vsi->tx_queues;
 
 	vsi->tx_itr_setting = pf->tx_itr;
 
 	for (int i = 0; i < vsi->num_tx_queues; i++, que++) {
 		struct tx_ring	*txr = &que->txr;
 
 		wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, i),
 		    vsi->tx_itr_setting);
 		txr->itr = vsi->tx_itr_setting;
 		txr->latency = IXL_AVE_LATENCY;
 	}
 }
 
 static void
 ixl_configure_rx_itr(struct ixl_pf *pf)
 {
 	struct i40e_hw		*hw = &pf->hw;
 	struct ixl_vsi		*vsi = &pf->vsi;
 	struct ixl_rx_queue	*que = vsi->rx_queues;
 
 	vsi->rx_itr_setting = pf->rx_itr;
 
 	for (int i = 0; i < vsi->num_rx_queues; i++, que++) {
 		struct rx_ring 	*rxr = &que->rxr;
 
 		wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, i),
 		    vsi->rx_itr_setting);
 		rxr->itr = vsi->rx_itr_setting;
 		rxr->latency = IXL_AVE_LATENCY;
 	}
 }
 
 /*
  * Write PF ITR values to queue ITR registers.
  */
 void
 ixl_configure_itr(struct ixl_pf *pf)
 {
 	ixl_configure_tx_itr(pf);
 	ixl_configure_rx_itr(pf);
 }
 
 /*********************************************************************
  *
  *  Get the hardware capabilities
  *
  **********************************************************************/
 
 int
 ixl_get_hw_capabilities(struct ixl_pf *pf)
 {
 	struct i40e_aqc_list_capabilities_element_resp *buf;
 	struct i40e_hw	*hw = &pf->hw;
 	device_t 	dev = pf->dev;
 	enum i40e_status_code status;
 	int len, i2c_intfc_num;
 	bool again = TRUE;
 	u16 needed;
 
 	len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);
 retry:
 	if (!(buf = (struct i40e_aqc_list_capabilities_element_resp *)
 	    malloc(len, M_DEVBUF, M_NOWAIT | M_ZERO))) {
 		device_printf(dev, "Unable to allocate cap memory\n");
                 return (ENOMEM);
 	}
 
 	/* This populates the hw struct */
         status = i40e_aq_discover_capabilities(hw, buf, len,
 	    &needed, i40e_aqc_opc_list_func_capabilities, NULL);
 	free(buf, M_DEVBUF);
 	if ((pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) &&
 	    (again == TRUE)) {
 		/* retry once with a larger buffer */
 		again = FALSE;
 		len = needed;
 		goto retry;
 	} else if (status != I40E_SUCCESS) {
 		device_printf(dev, "capability discovery failed; status %s, error %s\n",
 		    i40e_stat_str(hw, status), i40e_aq_str(hw, hw->aq.asq_last_status));
 		return (ENODEV);
 	}
 
 	/*
 	 * Some devices have both MDIO and I2C; since this isn't reported
 	 * by the FW, check registers to see if an I2C interface exists.
 	 */
 	i2c_intfc_num = ixl_find_i2c_interface(pf);
 	if (i2c_intfc_num != -1)
 		pf->has_i2c = true;
 
 	/* Determine functions to use for driver I2C accesses */
 	switch (pf->i2c_access_method) {
 	case 0: {
 		if (hw->mac.type == I40E_MAC_XL710 &&
 		    hw->aq.api_maj_ver == 1 &&
 		    hw->aq.api_min_ver >= 7) {
 			pf->read_i2c_byte = ixl_read_i2c_byte_aq;
 			pf->write_i2c_byte = ixl_write_i2c_byte_aq;
 		} else {
 			pf->read_i2c_byte = ixl_read_i2c_byte_reg;
 			pf->write_i2c_byte = ixl_write_i2c_byte_reg;
 		}
 		break;
 	}
 	case 3:
 		pf->read_i2c_byte = ixl_read_i2c_byte_aq;
 		pf->write_i2c_byte = ixl_write_i2c_byte_aq;
 		break;
 	case 2:
 		pf->read_i2c_byte = ixl_read_i2c_byte_reg;
 		pf->write_i2c_byte = ixl_write_i2c_byte_reg;
 		break;
 	case 1:
 		pf->read_i2c_byte = ixl_read_i2c_byte_bb;
 		pf->write_i2c_byte = ixl_write_i2c_byte_bb;
 		break;
 	default:
 		/* Should not happen */
 		device_printf(dev, "Error setting I2C access functions\n");
 		break;
 	}
 
 	/* Print a subset of the capability information. */
 	device_printf(dev,
 	    "PF-ID[%d]: VFs %d, MSI-X %d, VF MSI-X %d, QPs %d, %s\n",
 	    hw->pf_id, hw->func_caps.num_vfs, hw->func_caps.num_msix_vectors,
 	    hw->func_caps.num_msix_vectors_vf, hw->func_caps.num_tx_qp,
 	    (hw->func_caps.mdio_port_mode == 2) ? "I2C" :
 	    (hw->func_caps.mdio_port_mode == 1 && pf->has_i2c) ? "MDIO & I2C" :
 	    (hw->func_caps.mdio_port_mode == 1) ? "MDIO dedicated" :
 	    "MDIO shared");
 
 	return (0);
 }
 
 /* For the set_advertise sysctl */
 void
 ixl_set_initial_advertised_speeds(struct ixl_pf *pf)
 {
 	device_t dev = pf->dev;
 	int err;
 
 	/* Make sure to initialize the device to the complete list of
 	 * supported speeds on driver load, to ensure unloading and
 	 * reloading the driver will restore this value.
 	 */
 	err = ixl_set_advertised_speeds(pf, pf->supported_speeds, true);
 	if (err) {
 		/* Non-fatal error */
 		device_printf(dev, "%s: ixl_set_advertised_speeds() error %d\n",
 			      __func__, err);
 		return;
 	}
 
 	pf->advertised_speed =
 	    ixl_convert_sysctl_aq_link_speed(pf->supported_speeds, false);
 }
 
 int
 ixl_teardown_hw_structs(struct ixl_pf *pf)
 {
 	enum i40e_status_code status = 0;
 	struct i40e_hw *hw = &pf->hw;
 	device_t dev = pf->dev;
 
 	/* Shutdown LAN HMC */
 	if (hw->hmc.hmc_obj) {
 		status = i40e_shutdown_lan_hmc(hw);
 		if (status) {
 			device_printf(dev,
 			    "init: LAN HMC shutdown failure; status %s\n",
 			    i40e_stat_str(hw, status));
 			goto err_out;
 		}
 	}
 
 	/* Shutdown admin queue */
 	ixl_disable_intr0(hw);
 	status = i40e_shutdown_adminq(hw);
 	if (status)
 		device_printf(dev,
 		    "init: Admin Queue shutdown failure; status %s\n",
 		    i40e_stat_str(hw, status));
 
 	ixl_pf_qmgr_release(&pf->qmgr, &pf->qtag);
 err_out:
 	return (status);
 }
 
 int
 ixl_reset(struct ixl_pf *pf)
 {
 	struct i40e_hw *hw = &pf->hw;
 	device_t dev = pf->dev;
 	u32 reg;
 	int error = 0;
 
 	// XXX: clear_hw() actually writes to hw registers -- maybe this isn't necessary
 	i40e_clear_hw(hw);
 	error = i40e_pf_reset(hw);
 	if (error) {
 		device_printf(dev, "init: PF reset failure\n");
 		error = EIO;
 		goto err_out;
 	}
 
 	error = i40e_init_adminq(hw);
 	if (error) {
 		device_printf(dev, "init: Admin queue init failure;"
 		    " status code %d\n", error);
 		error = EIO;
 		goto err_out;
 	}
 
 	i40e_clear_pxe_mode(hw);
 
 #if 0
 	error = ixl_get_hw_capabilities(pf);
 	if (error) {
 		device_printf(dev, "init: Error retrieving HW capabilities;"
 		    " status code %d\n", error);
 		goto err_out;
 	}
 
 	error = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
 	    hw->func_caps.num_rx_qp, 0, 0);
 	if (error) {
 		device_printf(dev, "init: LAN HMC init failed; status code %d\n",
 		    error);
 		error = EIO;
 		goto err_out;
 	}
 
 	error = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
 	if (error) {
 		device_printf(dev, "init: LAN HMC config failed; status code %d\n",
 		    error);
 		error = EIO;
 		goto err_out;
 	}
 
 	// XXX: possible fix for panic, but our failure recovery is still broken
 	error = ixl_switch_config(pf);
 	if (error) {
 		device_printf(dev, "init: ixl_switch_config() failed: %d\n",
 		     error);
 		goto err_out;
 	}
 
 	error = i40e_aq_set_phy_int_mask(hw, IXL_DEFAULT_PHY_INT_MASK,
 	    NULL);
         if (error) {
 		device_printf(dev, "init: i40e_aq_set_phy_mask() failed: err %d,"
 		    " aq_err %d\n", error, hw->aq.asq_last_status);
 		error = EIO;
 		goto err_out;
 	}
 
 	error = i40e_set_fc(hw, &set_fc_err_mask, true);
 	if (error) {
 		device_printf(dev, "init: setting link flow control failed; retcode %d,"
 		    " fc_err_mask 0x%02x\n", error, set_fc_err_mask);
 		goto err_out;
 	}
 
 	// XXX: (Rebuild VSIs?)
 
 	/* Firmware delay workaround */
 	if (((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver < 33)) ||
 	    (hw->aq.fw_maj_ver < 4)) {
 		i40e_msec_delay(75);
 		error = i40e_aq_set_link_restart_an(hw, TRUE, NULL);
 		if (error) {
 			device_printf(dev, "init: link restart failed, aq_err %d\n",
 			    hw->aq.asq_last_status);
 			goto err_out;
 		}
 	}
 
 
 	/* Re-enable admin queue interrupt */
 	if (pf->msix > 1) {
 		ixl_configure_intr0_msix(pf);
 		ixl_enable_intr0(hw);
 	}
 
 err_out:
 	return (error);
 #endif
 	ixl_rebuild_hw_structs_after_reset(pf);
 
 	/* The PF reset should have cleared any critical errors */
 	atomic_clear_32(&pf->state, IXL_PF_STATE_PF_CRIT_ERR);
 	atomic_clear_32(&pf->state, IXL_PF_STATE_PF_RESET_REQ);
  
 	reg = rd32(hw, I40E_PFINT_ICR0_ENA);
 	reg |= IXL_ICR0_CRIT_ERR_MASK;
 	wr32(hw, I40E_PFINT_ICR0_ENA, reg);
 
  err_out:
  	return (error);
 }
 
 /*
  * TODO: Make sure this properly handles admin queue / single rx queue intr
  */
 int
 ixl_intr(void *arg)
 {
 	struct ixl_pf		*pf = arg;
 	struct i40e_hw		*hw =  &pf->hw;
 	struct ixl_vsi		*vsi = &pf->vsi;
 	struct ixl_rx_queue	*que = vsi->rx_queues;
         u32			icr0;
 
 	// pf->admin_irq++
 	++que->irqs;
 
 // TODO: Check against proper field
 #if 0
 	/* Clear PBA at start of ISR if using legacy interrupts */
 	if (pf->msix == 0)
 		wr32(hw, I40E_PFINT_DYN_CTL0,
 		    I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
 		    (IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT));
 #endif
 
 	icr0 = rd32(hw, I40E_PFINT_ICR0);
 
 
 #ifdef PCI_IOV
 	if (icr0 & I40E_PFINT_ICR0_VFLR_MASK)
 		iflib_iov_intr_deferred(vsi->ctx);
 #endif
 
 	// TODO!: Do the stuff that's done in ixl_msix_adminq here, too!
 	if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK)
 		iflib_admin_intr_deferred(vsi->ctx);
  
 	// TODO: Is intr0 enabled somewhere else?
 	ixl_enable_intr0(hw);
 
 	if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK)
 		return (FILTER_SCHEDULE_THREAD);
 	else
 		return (FILTER_HANDLED);
 }
 
 
 /*********************************************************************
  *
  *  MSI-X VSI Interrupt Service routine
  *
  **********************************************************************/
 int
 ixl_msix_que(void *arg)
 {
 	struct ixl_rx_queue *rx_que = arg;
 
 	++rx_que->irqs;
 
 	ixl_set_queue_rx_itr(rx_que);
 	// ixl_set_queue_tx_itr(que);
 
 	return (FILTER_SCHEDULE_THREAD);
 }
 
 
 /*********************************************************************
  *
  *  MSI-X Admin Queue Interrupt Service routine
  *
  **********************************************************************/
 int
 ixl_msix_adminq(void *arg)
 {
 	struct ixl_pf	*pf = arg;
 	struct i40e_hw	*hw = &pf->hw;
 	device_t	dev = pf->dev;
 	u32		reg, mask, rstat_reg;
 	bool		do_task = FALSE;
 
 	DDPRINTF(dev, "begin");
 
 	++pf->admin_irq;
 
 	reg = rd32(hw, I40E_PFINT_ICR0);
 	/*
 	 * For masking off interrupt causes that need to be handled before
 	 * they can be re-enabled
 	 */
 	mask = rd32(hw, I40E_PFINT_ICR0_ENA);
 
 	/* Check on the cause */
 	if (reg & I40E_PFINT_ICR0_ADMINQ_MASK) {
 		mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
 		do_task = TRUE;
 	}
 
 	if (reg & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
 		mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
 		atomic_set_32(&pf->state, IXL_PF_STATE_MDD_PENDING);
 		do_task = TRUE;
 	}
 
 	if (reg & I40E_PFINT_ICR0_GRST_MASK) {
 		mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK;
 		device_printf(dev, "Reset Requested!\n");
 		rstat_reg = rd32(hw, I40E_GLGEN_RSTAT);
 		rstat_reg = (rstat_reg & I40E_GLGEN_RSTAT_RESET_TYPE_MASK)
 		    >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT;
 		device_printf(dev, "Reset type: ");
 		switch (rstat_reg) {
 		/* These others might be handled similarly to an EMPR reset */
 		case I40E_RESET_CORER:
 			printf("CORER\n");
 			break;
 		case I40E_RESET_GLOBR:
 			printf("GLOBR\n");
 			break;
 		case I40E_RESET_EMPR:
 			printf("EMPR\n");
 			break;
 		default:
 			printf("POR\n");
 			break;
 		}
 		/* overload admin queue task to check reset progress */
 		atomic_set_int(&pf->state, IXL_PF_STATE_ADAPTER_RESETTING);
 		do_task = TRUE;
 	}
 
 	/*
 	 * PE / PCI / ECC exceptions are all handled in the same way:
 	 * mask out these three causes, then request a PF reset
 	 *
 	 * TODO: I think at least ECC error requires a GLOBR, not PFR
 	 */
 	if (reg & I40E_PFINT_ICR0_ECC_ERR_MASK)
  		device_printf(dev, "ECC Error detected!\n");
 	if (reg & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK)
 		device_printf(dev, "PCI Exception detected!\n");
 	if (reg & I40E_PFINT_ICR0_PE_CRITERR_MASK)
 		device_printf(dev, "Critical Protocol Engine Error detected!\n");
 	/* Checks against the conditions above */
 	if (reg & IXL_ICR0_CRIT_ERR_MASK) {
 		mask &= ~IXL_ICR0_CRIT_ERR_MASK;
 		atomic_set_32(&pf->state,
 		    IXL_PF_STATE_PF_RESET_REQ | IXL_PF_STATE_PF_CRIT_ERR);
 		do_task = TRUE;
 	}
 
 	// TODO: Linux driver never re-enables this interrupt once it has been detected
 	// Then what is supposed to happen? A PF reset? Should it never happen?
 	// TODO: Parse out this error into something human readable
 	if (reg & I40E_PFINT_ICR0_HMC_ERR_MASK) {
 		reg = rd32(hw, I40E_PFHMC_ERRORINFO);
 		if (reg & I40E_PFHMC_ERRORINFO_ERROR_DETECTED_MASK) {
 			device_printf(dev, "HMC Error detected!\n");
 			device_printf(dev, "INFO 0x%08x\n", reg);
 			reg = rd32(hw, I40E_PFHMC_ERRORDATA);
 			device_printf(dev, "DATA 0x%08x\n", reg);
 			wr32(hw, I40E_PFHMC_ERRORINFO, 0);
 		}
 	}
 
 #ifdef PCI_IOV
 	if (reg & I40E_PFINT_ICR0_VFLR_MASK) {
 		mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
 		iflib_iov_intr_deferred(pf->vsi.ctx);
 	}
 #endif
 
 	wr32(hw, I40E_PFINT_ICR0_ENA, mask);
 	ixl_enable_intr0(hw);
 
 	if (do_task)
 		return (FILTER_SCHEDULE_THREAD);
 	else
 		return (FILTER_HANDLED);
 }
 
 /*********************************************************************
  * 	Filter Routines
  *
  *	Routines for multicast and vlan filter management.
  *
  *********************************************************************/
 void
 ixl_add_multi(struct ixl_vsi *vsi)
 {
 	struct	ifmultiaddr	*ifma;
 	struct ifnet		*ifp = vsi->ifp;
 	struct i40e_hw		*hw = vsi->hw;
 	int			mcnt = 0, flags;
 
 	IOCTL_DEBUGOUT("ixl_add_multi: begin");
 
 	if_maddr_rlock(ifp);
 	/*
 	** First just get a count, to decide if we
 	** we simply use multicast promiscuous.
 	*/
 	CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
 		if (ifma->ifma_addr->sa_family != AF_LINK)
 			continue;
 		mcnt++;
 	}
 	if_maddr_runlock(ifp);
 
 	if (__predict_false(mcnt >= MAX_MULTICAST_ADDR)) {
 		/* delete existing MC filters */
 		ixl_del_hw_filters(vsi, mcnt);
 		i40e_aq_set_vsi_multicast_promiscuous(hw,
 		    vsi->seid, TRUE, NULL);
 		return;
 	}
 
 	mcnt = 0;
 	if_maddr_rlock(ifp);
 	CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
 		if (ifma->ifma_addr->sa_family != AF_LINK)
 			continue;
 		ixl_add_mc_filter(vsi,
 		    (u8*)LLADDR((struct sockaddr_dl *) ifma->ifma_addr));
 		mcnt++;
 	}
 	if_maddr_runlock(ifp);
 	if (mcnt > 0) {
 		flags = (IXL_FILTER_ADD | IXL_FILTER_USED | IXL_FILTER_MC);
 		ixl_add_hw_filters(vsi, flags, mcnt);
 	}
 
 	IOCTL_DEBUGOUT("ixl_add_multi: end");
 }
 
 int
 ixl_del_multi(struct ixl_vsi *vsi)
 {
 	struct ifnet		*ifp = vsi->ifp;
 	struct ifmultiaddr	*ifma;
 	struct ixl_mac_filter	*f;
 	int			mcnt = 0;
 	bool		match = FALSE;
 
 	IOCTL_DEBUGOUT("ixl_del_multi: begin");
 
 	/* Search for removed multicast addresses */
 	if_maddr_rlock(ifp);
 	SLIST_FOREACH(f, &vsi->ftl, next) {
 		if ((f->flags & IXL_FILTER_USED) && (f->flags & IXL_FILTER_MC)) {
 			match = FALSE;
 			CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
 				if (ifma->ifma_addr->sa_family != AF_LINK)
 					continue;
 				u8 *mc_addr = (u8 *)LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
 				if (cmp_etheraddr(f->macaddr, mc_addr)) {
 					match = TRUE;
 					break;
 				}
 			}
 			if (match == FALSE) {
 				f->flags |= IXL_FILTER_DEL;
 				mcnt++;
 			}
 		}
 	}
 	if_maddr_runlock(ifp);
 
 	if (mcnt > 0)
 		ixl_del_hw_filters(vsi, mcnt);
 	
 	return (mcnt);
 }
 
 void
 ixl_link_up_msg(struct ixl_pf *pf)
 {
 	struct i40e_hw *hw = &pf->hw;
 	struct ifnet *ifp = pf->vsi.ifp;
 	char *req_fec_string, *neg_fec_string;
 	u8 fec_abilities;
 
 	fec_abilities = hw->phy.link_info.req_fec_info;
 	/* If both RS and KR are requested, only show RS */
 	if (fec_abilities & I40E_AQ_REQUEST_FEC_RS)
 		req_fec_string = ixl_fec_string[0];
 	else if (fec_abilities & I40E_AQ_REQUEST_FEC_KR)
 		req_fec_string = ixl_fec_string[1];
 	else
 		req_fec_string = ixl_fec_string[2];
 
 	if (hw->phy.link_info.fec_info & I40E_AQ_CONFIG_FEC_RS_ENA)
 		neg_fec_string = ixl_fec_string[0];
 	else if (hw->phy.link_info.fec_info & I40E_AQ_CONFIG_FEC_KR_ENA)
 		neg_fec_string = ixl_fec_string[1];
 	else
 		neg_fec_string = ixl_fec_string[2];
 
 	log(LOG_NOTICE, "%s: Link is up, %s Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg: %s, Flow Control: %s\n",
 	    ifp->if_xname,
 	    ixl_aq_speed_to_str(hw->phy.link_info.link_speed),
 	    req_fec_string, neg_fec_string,
 	    (hw->phy.link_info.an_info & I40E_AQ_AN_COMPLETED) ? "True" : "False",
 	    (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_TX &&
 	        hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_RX) ?
 		ixl_fc_string[3] : (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_TX) ?
 		ixl_fc_string[2] : (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_RX) ?
 		ixl_fc_string[1] : ixl_fc_string[0]);
 }
 
 /*
  * Configure admin queue/misc interrupt cause registers in hardware.
  */
 void
 ixl_configure_intr0_msix(struct ixl_pf *pf)
 {
 	struct i40e_hw *hw = &pf->hw;
 	u32 reg;
 
 	/* First set up the adminq - vector 0 */
 	wr32(hw, I40E_PFINT_ICR0_ENA, 0);  /* disable all */
 	rd32(hw, I40E_PFINT_ICR0);         /* read to clear */
 
 	reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK |
 	    I40E_PFINT_ICR0_ENA_GRST_MASK |
 	    I40E_PFINT_ICR0_ENA_HMC_ERR_MASK |
 	    I40E_PFINT_ICR0_ENA_ADMINQ_MASK |
 	    I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK |
 	    I40E_PFINT_ICR0_ENA_VFLR_MASK |
 	    I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK |
 	    I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK;
 	wr32(hw, I40E_PFINT_ICR0_ENA, reg);
 
 	/*
 	 * 0x7FF is the end of the queue list.
 	 * This means we won't use MSI-X vector 0 for a queue interrupt
 	 * in MSI-X mode.
 	 */
 	wr32(hw, I40E_PFINT_LNKLST0, 0x7FF);
 	/* Value is in 2 usec units, so 0x3E is 62*2 = 124 usecs. */
 	wr32(hw, I40E_PFINT_ITR0(IXL_RX_ITR), 0x3E);
 
 	wr32(hw, I40E_PFINT_DYN_CTL0,
 	    I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |
 	    I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);
 
 	wr32(hw, I40E_PFINT_STAT_CTL0, 0);
 }
 
 /*
  * Configure queue interrupt cause registers in hardware.
  *
  * Linked list for each vector LNKLSTN(i) -> RQCTL(i) -> TQCTL(i) -> EOL
  */
 void
 ixl_configure_queue_intr_msix(struct ixl_pf *pf)
 {
 	struct i40e_hw *hw = &pf->hw;
 	struct ixl_vsi *vsi = &pf->vsi;
 	u32		reg;
 	u16		vector = 1;
 
 	// TODO: See if max is really necessary
 	for (int i = 0; i < max(vsi->num_rx_queues, vsi->num_tx_queues); i++, vector++) {
 		/* Make sure interrupt is disabled */
 		wr32(hw, I40E_PFINT_DYN_CTLN(i), 0);
 		/* Set linked list head to point to corresponding RX queue
 		 * e.g. vector 1 (LNKLSTN register 0) points to queue pair 0's RX queue */
 		reg = ((i << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT)
 		        & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK) |
 		    ((I40E_QUEUE_TYPE_RX << I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT)
 		        & I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_MASK);
 		wr32(hw, I40E_PFINT_LNKLSTN(i), reg);
 
 		reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
 		(IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
 		(vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
 		(i << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
 		(I40E_QUEUE_TYPE_TX << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
 		wr32(hw, I40E_QINT_RQCTL(i), reg);
 
 		reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
 		(IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
 		(vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
 		(IXL_QUEUE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
 		(I40E_QUEUE_TYPE_RX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
 		wr32(hw, I40E_QINT_TQCTL(i), reg);
 	}
 }
 
 /*
  * Configure for single interrupt vector operation 
  */
 void
 ixl_configure_legacy(struct ixl_pf *pf)
 {
 	struct i40e_hw	*hw = &pf->hw;
 	struct ixl_vsi	*vsi = &pf->vsi;
 	u32 reg;
 
 // TODO: Fix
 #if 0
 	/* Configure ITR */
 	vsi->tx_itr_setting = pf->tx_itr;
 	wr32(hw, I40E_PFINT_ITR0(IXL_TX_ITR),
 	    vsi->tx_itr_setting);
 	txr->itr = vsi->tx_itr_setting;
 
 	vsi->rx_itr_setting = pf->rx_itr;
 	wr32(hw, I40E_PFINT_ITR0(IXL_RX_ITR),
 	    vsi->rx_itr_setting);
 	rxr->itr = vsi->rx_itr_setting;
 	/* XXX: Assuming only 1 queue in single interrupt mode */
 #endif
 	vsi->rx_queues[0].rxr.itr = vsi->rx_itr_setting;
 
 	/* Setup "other" causes */
 	reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK
 	    | I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK
 	    | I40E_PFINT_ICR0_ENA_GRST_MASK
 	    | I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK
 	    | I40E_PFINT_ICR0_ENA_HMC_ERR_MASK
 	    | I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK
 	    | I40E_PFINT_ICR0_ENA_VFLR_MASK
 	    | I40E_PFINT_ICR0_ENA_ADMINQ_MASK
 	    ;
 	wr32(hw, I40E_PFINT_ICR0_ENA, reg);
 
 	/* No ITR for non-queue interrupts */
 	wr32(hw, I40E_PFINT_STAT_CTL0,
 	    IXL_ITR_NONE << I40E_PFINT_STAT_CTL0_OTHER_ITR_INDX_SHIFT);
 
 	/* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
 	wr32(hw, I40E_PFINT_LNKLST0, 0);
 
 	/* Associate the queue pair to the vector and enable the q int */
 	reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK
 	    | (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT)
 	    | (I40E_QUEUE_TYPE_TX << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
 	wr32(hw, I40E_QINT_RQCTL(0), reg);
 
 	reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK
 	    | (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)
 	    | (IXL_QUEUE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
 	wr32(hw, I40E_QINT_TQCTL(0), reg);
 }
 
 void
 ixl_free_pci_resources(struct ixl_pf *pf)
 {
 	struct ixl_vsi		*vsi = &pf->vsi;
 	device_t		dev = iflib_get_dev(vsi->ctx);
 	struct ixl_rx_queue	*rx_que = vsi->rx_queues;
 
 	/* We may get here before stations are set up */
 	if (rx_que == NULL)
 		goto early;
 
 	/*
 	**  Release all MSI-X VSI resources:
 	*/
 	iflib_irq_free(vsi->ctx, &vsi->irq);
 
 	for (int i = 0; i < vsi->num_rx_queues; i++, rx_que++)
 		iflib_irq_free(vsi->ctx, &rx_que->que_irq);
 early:
 	if (pf->pci_mem != NULL)
 		bus_release_resource(dev, SYS_RES_MEMORY,
 		    rman_get_rid(pf->pci_mem), pf->pci_mem);
 }
 
 void
 ixl_add_ifmedia(struct ixl_vsi *vsi, u64 phy_types)
 {
 	/* Display supported media types */
 	if (phy_types & (I40E_CAP_PHY_TYPE_100BASE_TX))
 		ifmedia_add(vsi->media, IFM_ETHER | IFM_100_TX, 0, NULL);
 
 	if (phy_types & (I40E_CAP_PHY_TYPE_1000BASE_T))
 		ifmedia_add(vsi->media, IFM_ETHER | IFM_1000_T, 0, NULL);
 	if (phy_types & (I40E_CAP_PHY_TYPE_1000BASE_SX))
 		ifmedia_add(vsi->media, IFM_ETHER | IFM_1000_SX, 0, NULL);
 	if (phy_types & (I40E_CAP_PHY_TYPE_1000BASE_LX))
 		ifmedia_add(vsi->media, IFM_ETHER | IFM_1000_LX, 0, NULL);
 
 	if (phy_types & (I40E_CAP_PHY_TYPE_XAUI) ||
 	    phy_types & (I40E_CAP_PHY_TYPE_XFI) ||
 	    phy_types & (I40E_CAP_PHY_TYPE_10GBASE_SFPP_CU))
 		ifmedia_add(vsi->media, IFM_ETHER | IFM_10G_TWINAX, 0, NULL);
 
 	if (phy_types & (I40E_CAP_PHY_TYPE_10GBASE_SR))
 		ifmedia_add(vsi->media, IFM_ETHER | IFM_10G_SR, 0, NULL);
 	if (phy_types & (I40E_CAP_PHY_TYPE_10GBASE_LR))
 		ifmedia_add(vsi->media, IFM_ETHER | IFM_10G_LR, 0, NULL);
 	if (phy_types & (I40E_CAP_PHY_TYPE_10GBASE_T))
 		ifmedia_add(vsi->media, IFM_ETHER | IFM_10G_T, 0, NULL);
 
 	if (phy_types & (I40E_CAP_PHY_TYPE_40GBASE_CR4) ||
 	    phy_types & (I40E_CAP_PHY_TYPE_40GBASE_CR4_CU) ||
 	    phy_types & (I40E_CAP_PHY_TYPE_40GBASE_AOC) ||
 	    phy_types & (I40E_CAP_PHY_TYPE_XLAUI) ||
 	    phy_types & (I40E_CAP_PHY_TYPE_40GBASE_KR4))
 		ifmedia_add(vsi->media, IFM_ETHER | IFM_40G_CR4, 0, NULL);
 	if (phy_types & (I40E_CAP_PHY_TYPE_40GBASE_SR4))
 		ifmedia_add(vsi->media, IFM_ETHER | IFM_40G_SR4, 0, NULL);
 	if (phy_types & (I40E_CAP_PHY_TYPE_40GBASE_LR4))
 		ifmedia_add(vsi->media, IFM_ETHER | IFM_40G_LR4, 0, NULL);
 
 	if (phy_types & (I40E_CAP_PHY_TYPE_1000BASE_KX))
 		ifmedia_add(vsi->media, IFM_ETHER | IFM_1000_KX, 0, NULL);
 
 	if (phy_types & (I40E_CAP_PHY_TYPE_10GBASE_CR1_CU)
 	    || phy_types & (I40E_CAP_PHY_TYPE_10GBASE_CR1))
 		ifmedia_add(vsi->media, IFM_ETHER | IFM_10G_CR1, 0, NULL);
 	if (phy_types & (I40E_CAP_PHY_TYPE_10GBASE_AOC))
 		ifmedia_add(vsi->media, IFM_ETHER | IFM_10G_AOC, 0, NULL);
 	if (phy_types & (I40E_CAP_PHY_TYPE_SFI))
 		ifmedia_add(vsi->media, IFM_ETHER | IFM_10G_SFI, 0, NULL);
 	if (phy_types & (I40E_CAP_PHY_TYPE_10GBASE_KX4))
 		ifmedia_add(vsi->media, IFM_ETHER | IFM_10G_KX4, 0, NULL);
 	if (phy_types & (I40E_CAP_PHY_TYPE_10GBASE_KR))
 		ifmedia_add(vsi->media, IFM_ETHER | IFM_10G_KR, 0, NULL);
 
 	if (phy_types & (I40E_CAP_PHY_TYPE_20GBASE_KR2))
 		ifmedia_add(vsi->media, IFM_ETHER | IFM_20G_KR2, 0, NULL);
 
 	if (phy_types & (I40E_CAP_PHY_TYPE_40GBASE_KR4))
 		ifmedia_add(vsi->media, IFM_ETHER | IFM_40G_KR4, 0, NULL);
 	if (phy_types & (I40E_CAP_PHY_TYPE_XLPPI))
 		ifmedia_add(vsi->media, IFM_ETHER | IFM_40G_XLPPI, 0, NULL);
 
 	if (phy_types & (I40E_CAP_PHY_TYPE_25GBASE_KR))
 		ifmedia_add(vsi->media, IFM_ETHER | IFM_25G_KR, 0, NULL);
 	if (phy_types & (I40E_CAP_PHY_TYPE_25GBASE_CR))
 		ifmedia_add(vsi->media, IFM_ETHER | IFM_25G_CR, 0, NULL);
 	if (phy_types & (I40E_CAP_PHY_TYPE_25GBASE_SR))
 		ifmedia_add(vsi->media, IFM_ETHER | IFM_25G_SR, 0, NULL);
 	if (phy_types & (I40E_CAP_PHY_TYPE_25GBASE_LR))
 		ifmedia_add(vsi->media, IFM_ETHER | IFM_25G_LR, 0, NULL);
 	if (phy_types & (I40E_CAP_PHY_TYPE_25GBASE_AOC))
 		ifmedia_add(vsi->media, IFM_ETHER | IFM_25G_AOC, 0, NULL);
 	if (phy_types & (I40E_CAP_PHY_TYPE_25GBASE_ACC))
 		ifmedia_add(vsi->media, IFM_ETHER | IFM_25G_ACC, 0, NULL);
 }
 
 /*********************************************************************
  *
  *  Setup networking device structure and register an interface.
  *
  **********************************************************************/
 int
 ixl_setup_interface(device_t dev, struct ixl_pf *pf)
 {
 	struct ixl_vsi *vsi = &pf->vsi;
 	if_ctx_t ctx = vsi->ctx;
 	struct i40e_hw *hw = &pf->hw;
 	struct ifnet *ifp = iflib_get_ifp(ctx);
 	struct i40e_aq_get_phy_abilities_resp abilities;
 	enum i40e_status_code aq_error = 0;
 
 	INIT_DBG_DEV(dev, "begin");
 
 	vsi->shared->isc_max_frame_size =
 	    ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN
 	    + ETHER_VLAN_ENCAP_LEN;
 
 	aq_error = i40e_aq_get_phy_capabilities(hw,
 	    FALSE, TRUE, &abilities, NULL);
 	/* May need delay to detect fiber correctly */
 	if (aq_error == I40E_ERR_UNKNOWN_PHY) {
 		/* TODO: Maybe just retry this in a task... */
 		i40e_msec_delay(200);
 		aq_error = i40e_aq_get_phy_capabilities(hw, FALSE,
 		    TRUE, &abilities, NULL);
 	}
 	if (aq_error) {
 		if (aq_error == I40E_ERR_UNKNOWN_PHY)
 			device_printf(dev, "Unknown PHY type detected!\n");
 		else
 			device_printf(dev,
 			    "Error getting supported media types, err %d,"
 			    " AQ error %d\n", aq_error, hw->aq.asq_last_status);
 	} else {
 		pf->supported_speeds = abilities.link_speed;
 #if __FreeBSD_version >= 1100000
 		if_setbaudrate(ifp, ixl_max_aq_speed_to_value(pf->supported_speeds));
 #else
 		if_initbaudrate(ifp, ixl_max_aq_speed_to_value(pf->supported_speeds));
 #endif
 
 		ixl_add_ifmedia(vsi, hw->phy.phy_types);
 	}
 
 	/* Use autoselect media by default */
 	ifmedia_add(vsi->media, IFM_ETHER | IFM_AUTO, 0, NULL);
 	ifmedia_set(vsi->media, IFM_ETHER | IFM_AUTO);
 
 	return (0);
 }
 
 /*
  * Input: bitmap of enum i40e_aq_link_speed
  */
 u64
 ixl_max_aq_speed_to_value(u8 link_speeds)
 {
 	if (link_speeds & I40E_LINK_SPEED_40GB)
 		return IF_Gbps(40);
 	if (link_speeds & I40E_LINK_SPEED_25GB)
 		return IF_Gbps(25);
 	if (link_speeds & I40E_LINK_SPEED_20GB)
 		return IF_Gbps(20);
 	if (link_speeds & I40E_LINK_SPEED_10GB)
 		return IF_Gbps(10);
 	if (link_speeds & I40E_LINK_SPEED_1GB)
 		return IF_Gbps(1);
 	if (link_speeds & I40E_LINK_SPEED_100MB)
 		return IF_Mbps(100);
 	else
 		/* Minimum supported link speed */
 		return IF_Mbps(100);
 }
 
 /*
 ** Run when the Admin Queue gets a link state change interrupt.
 */
 void
 ixl_link_event(struct ixl_pf *pf, struct i40e_arq_event_info *e)
 {
 	struct i40e_hw *hw = &pf->hw; 
 	device_t dev = iflib_get_dev(pf->vsi.ctx);
 	struct i40e_aqc_get_link_status *status =
 	    (struct i40e_aqc_get_link_status *)&e->desc.params.raw;
 
 	/* Request link status from adapter */
 	hw->phy.get_link_info = TRUE;
 	i40e_get_link_status(hw, &pf->link_up);
 
 	/* Print out message if an unqualified module is found */
 	if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) &&
 	    (pf->advertised_speed) &&
 	    (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) &&
 	    (!(status->link_info & I40E_AQ_LINK_UP)))
 		device_printf(dev, "Link failed because "
 		    "an unqualified module was detected!\n");
 
 	/* OS link info is updated elsewhere */
 }
 
 /*********************************************************************
  *
  *  Get Firmware Switch configuration
  *	- this will need to be more robust when more complex
  *	  switch configurations are enabled.
  *
  **********************************************************************/
 int
 ixl_switch_config(struct ixl_pf *pf)
 {
 	struct i40e_hw	*hw = &pf->hw; 
 	struct ixl_vsi	*vsi = &pf->vsi;
 	device_t 	dev = iflib_get_dev(vsi->ctx);
 	struct i40e_aqc_get_switch_config_resp *sw_config;
 	u8	aq_buf[I40E_AQ_LARGE_BUF];
 	int	ret;
 	u16	next = 0;
 
 	memset(&aq_buf, 0, sizeof(aq_buf));
 	sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
 	ret = i40e_aq_get_switch_config(hw, sw_config,
 	    sizeof(aq_buf), &next, NULL);
 	if (ret) {
 		device_printf(dev, "aq_get_switch_config() failed, error %d,"
 		    " aq_error %d\n", ret, pf->hw.aq.asq_last_status);
 		return (ret);
 	}
 	if (pf->dbg_mask & IXL_DBG_SWITCH_INFO) {
 		device_printf(dev,
 		    "Switch config: header reported: %d in structure, %d total\n",
 		    sw_config->header.num_reported, sw_config->header.num_total);
 		for (int i = 0; i < sw_config->header.num_reported; i++) {
 			device_printf(dev,
 			    "-> %d: type=%d seid=%d uplink=%d downlink=%d\n", i,
 			    sw_config->element[i].element_type,
 			    sw_config->element[i].seid,
 			    sw_config->element[i].uplink_seid,
 			    sw_config->element[i].downlink_seid);
 		}
 	}
 	/* Simplified due to a single VSI */
 	vsi->uplink_seid = sw_config->element[0].uplink_seid;
 	vsi->downlink_seid = sw_config->element[0].downlink_seid;
 	vsi->seid = sw_config->element[0].seid;
 	return (ret);
 }
 
 /*********************************************************************
  *
  *  Initialize the VSI:  this handles contexts, which means things
  *  			 like the number of descriptors, buffer size,
  *			 plus we init the rings thru this function.
  *
  **********************************************************************/
 int
 ixl_initialize_vsi(struct ixl_vsi *vsi)
 {
 	struct ixl_pf *pf = vsi->back;
 	if_softc_ctx_t		scctx = iflib_get_softc_ctx(vsi->ctx);
 	struct ixl_tx_queue	*tx_que = vsi->tx_queues;
 	struct ixl_rx_queue	*rx_que = vsi->rx_queues;
 	device_t		dev = iflib_get_dev(vsi->ctx);
 	struct i40e_hw		*hw = vsi->hw;
 	struct i40e_vsi_context	ctxt;
 	int 			tc_queues;
 	int			err = 0;
 
 	memset(&ctxt, 0, sizeof(ctxt));
 	ctxt.seid = vsi->seid;
 	if (pf->veb_seid != 0)
 		ctxt.uplink_seid = pf->veb_seid;
 	ctxt.pf_num = hw->pf_id;
 	err = i40e_aq_get_vsi_params(hw, &ctxt, NULL);
 	if (err) {
 		device_printf(dev, "i40e_aq_get_vsi_params() failed, error %d"
 		    " aq_error %d\n", err, hw->aq.asq_last_status);
 		return (err);
 	}
 	ixl_dbg(pf, IXL_DBG_SWITCH_INFO,
 	    "get_vsi_params: seid: %d, uplinkseid: %d, vsi_number: %d, "
 	    "vsis_allocated: %d, vsis_unallocated: %d, flags: 0x%x, "
 	    "pfnum: %d, vfnum: %d, stat idx: %d, enabled: %d\n", ctxt.seid,
 	    ctxt.uplink_seid, ctxt.vsi_number,
 	    ctxt.vsis_allocated, ctxt.vsis_unallocated,
 	    ctxt.flags, ctxt.pf_num, ctxt.vf_num,
 	    ctxt.info.stat_counter_idx, ctxt.info.up_enable_bits);
 	/*
 	** Set the queue and traffic class bits
 	**  - when multiple traffic classes are supported
 	**    this will need to be more robust.
 	*/
 	ctxt.info.valid_sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
 	ctxt.info.mapping_flags |= I40E_AQ_VSI_QUE_MAP_CONTIG;
 	/* In contig mode, que_mapping[0] is first queue index used by this VSI */
 	ctxt.info.queue_mapping[0] = 0;
 	/*
 	 * This VSI will only use traffic class 0; start traffic class 0's
 	 * queue allocation at queue 0, and assign it 2^tc_queues queues (though
 	 * the driver may not use all of them).
 	 */
 	tc_queues = fls(pf->qtag.num_allocated) - 1;
 	ctxt.info.tc_mapping[0] = ((pf->qtag.first_qidx << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT)
 	    & I40E_AQ_VSI_TC_QUE_OFFSET_MASK) |
 	    ((tc_queues << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT)
 	    & I40E_AQ_VSI_TC_QUE_NUMBER_MASK);
 
 	/* Set VLAN receive stripping mode */
 	ctxt.info.valid_sections |= I40E_AQ_VSI_PROP_VLAN_VALID;
 	ctxt.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL;
 	if (if_getcapenable(vsi->ifp) & IFCAP_VLAN_HWTAGGING)
 		ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
 	else
 		ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
 
 #ifdef IXL_IW
 	/* Set TCP Enable for iWARP capable VSI */
 	if (ixl_enable_iwarp && pf->iw_enabled) {
 		ctxt.info.valid_sections |=
 		    htole16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
 		ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA;
 	}
 #endif
 	/* Save VSI number and info for use later */
 	vsi->vsi_num = ctxt.vsi_number;
 	bcopy(&ctxt.info, &vsi->info, sizeof(vsi->info));
 
 	/* Reset VSI statistics */
 	ixl_vsi_reset_stats(vsi);
 	vsi->hw_filters_add = 0;
 	vsi->hw_filters_del = 0;
 
 	ctxt.flags = htole16(I40E_AQ_VSI_TYPE_PF);
 
 	err = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
 	if (err) {
 		device_printf(dev, "i40e_aq_update_vsi_params() failed, error %d,"
 		    " aq_error %d\n", err, hw->aq.asq_last_status);
 		return (err);
 	}
 
 	for (int i = 0; i < vsi->num_tx_queues; i++, tx_que++) {
 		struct tx_ring		*txr = &tx_que->txr;
 		struct i40e_hmc_obj_txq tctx;
 		u32			txctl;
 
 		/* Setup the HMC TX Context  */
 		bzero(&tctx, sizeof(tctx));
 		tctx.new_context = 1;
 		tctx.base = (txr->tx_paddr/IXL_TX_CTX_BASE_UNITS);
 		tctx.qlen = scctx->isc_ntxd[0];
 		tctx.fc_ena = 0;	/* Disable FCoE */
 		/*
 		 * This value needs to pulled from the VSI that this queue
 		 * is assigned to. Index into array is traffic class.
 		 */
 		tctx.rdylist = vsi->info.qs_handle[0];
 		/*
 		 * Set these to enable Head Writeback
 		 * - Address is last entry in TX ring (reserved for HWB index)
 		 * Leave these as 0 for Descriptor Writeback
 		 */
 		if (vsi->enable_head_writeback) {
 			tctx.head_wb_ena = 1;
 			tctx.head_wb_addr = txr->tx_paddr +
 			    (scctx->isc_ntxd[0] * sizeof(struct i40e_tx_desc));
 		} else {
 			tctx.head_wb_ena = 0;
 			tctx.head_wb_addr = 0; 
 		}
 		tctx.rdylist_act = 0;
 		err = i40e_clear_lan_tx_queue_context(hw, i);
 		if (err) {
 			device_printf(dev, "Unable to clear TX context\n");
 			break;
 		}
 		err = i40e_set_lan_tx_queue_context(hw, i, &tctx);
 		if (err) {
 			device_printf(dev, "Unable to set TX context\n");
 			break;
 		}
 		/* Associate the ring with this PF */
 		txctl = I40E_QTX_CTL_PF_QUEUE;
 		txctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) &
 		    I40E_QTX_CTL_PF_INDX_MASK);
 		wr32(hw, I40E_QTX_CTL(i), txctl);
 		ixl_flush(hw);
 
 		/* Do ring (re)init */
 		ixl_init_tx_ring(vsi, tx_que);
 	}
 	for (int i = 0; i < vsi->num_rx_queues; i++, rx_que++) {
 		struct rx_ring 		*rxr = &rx_que->rxr;
 		struct i40e_hmc_obj_rxq rctx;
 
 		/* Next setup the HMC RX Context  */
-		if (scctx->isc_max_frame_size <= MCLBYTES)
-			rxr->mbuf_sz = MCLBYTES;
-		else
-			rxr->mbuf_sz = MJUMPAGESIZE;
+		rxr->mbuf_sz = iflib_get_rx_mbuf_sz(vsi->ctx);
 
 		u16 max_rxmax = rxr->mbuf_sz * hw->func_caps.rx_buf_chain_len;
 
 		/* Set up an RX context for the HMC */
 		memset(&rctx, 0, sizeof(struct i40e_hmc_obj_rxq));
 		rctx.dbuff = rxr->mbuf_sz >> I40E_RXQ_CTX_DBUFF_SHIFT;
 		/* ignore header split for now */
 		rctx.hbuff = 0 >> I40E_RXQ_CTX_HBUFF_SHIFT;
 		rctx.rxmax = (scctx->isc_max_frame_size < max_rxmax) ?
 		    scctx->isc_max_frame_size : max_rxmax;
 		rctx.dtype = 0;
 		rctx.dsize = 1;		/* do 32byte descriptors */
 		rctx.hsplit_0 = 0;	/* no header split */
 		rctx.base = (rxr->rx_paddr/IXL_RX_CTX_BASE_UNITS);
 		rctx.qlen = scctx->isc_nrxd[0];
 		rctx.tphrdesc_ena = 1;
 		rctx.tphwdesc_ena = 1;
 		rctx.tphdata_ena = 0;	/* Header Split related */
 		rctx.tphhead_ena = 0;	/* Header Split related */
 		rctx.lrxqthresh = 1;	/* Interrupt at <64 desc avail */
 		rctx.crcstrip = 1;
 		rctx.l2tsel = 1;
 		rctx.showiv = 1;	/* Strip inner VLAN header */
 		rctx.fc_ena = 0;	/* Disable FCoE */
 		rctx.prefena = 1;	/* Prefetch descriptors */
 
 		err = i40e_clear_lan_rx_queue_context(hw, i);
 		if (err) {
 			device_printf(dev,
 			    "Unable to clear RX context %d\n", i);
 			break;
 		}
 		err = i40e_set_lan_rx_queue_context(hw, i, &rctx);
 		if (err) {
 			device_printf(dev, "Unable to set RX context %d\n", i);
 			break;
 		}
 		wr32(vsi->hw, I40E_QRX_TAIL(i), 0);
 	}
 	return (err);
 }
 
 void
 ixl_free_mac_filters(struct ixl_vsi *vsi)
 {
 	struct ixl_mac_filter *f;
 
 	while (!SLIST_EMPTY(&vsi->ftl)) {
 		f = SLIST_FIRST(&vsi->ftl);
 		SLIST_REMOVE_HEAD(&vsi->ftl, next);
 		free(f, M_DEVBUF);
 	}
 }
 
 /*
 ** Provide a update to the queue RX
 ** interrupt moderation value.
 */
 void
 ixl_set_queue_rx_itr(struct ixl_rx_queue *que)
 {
 	struct ixl_vsi	*vsi = que->vsi;
 	struct ixl_pf	*pf = (struct ixl_pf *)vsi->back;
 	struct i40e_hw	*hw = vsi->hw;
 	struct rx_ring	*rxr = &que->rxr;
 	u16		rx_itr;
 	u16		rx_latency = 0;
 	int		rx_bytes;
 
 	/* Idle, do nothing */
 	if (rxr->bytes == 0)
 		return;
 
 	if (pf->dynamic_rx_itr) {
 		rx_bytes = rxr->bytes/rxr->itr;
 		rx_itr = rxr->itr;
 
 		/* Adjust latency range */
 		switch (rxr->latency) {
 		case IXL_LOW_LATENCY:
 			if (rx_bytes > 10) {
 				rx_latency = IXL_AVE_LATENCY;
 				rx_itr = IXL_ITR_20K;
 			}
 			break;
 		case IXL_AVE_LATENCY:
 			if (rx_bytes > 20) {
 				rx_latency = IXL_BULK_LATENCY;
 				rx_itr = IXL_ITR_8K;
 			} else if (rx_bytes <= 10) {
 				rx_latency = IXL_LOW_LATENCY;
 				rx_itr = IXL_ITR_100K;
 			}
 			break;
 		case IXL_BULK_LATENCY:
 			if (rx_bytes <= 20) {
 				rx_latency = IXL_AVE_LATENCY;
 				rx_itr = IXL_ITR_20K;
 			}
 			break;
        		 }
 
 		rxr->latency = rx_latency;
 
 		if (rx_itr != rxr->itr) {
 			/* do an exponential smoothing */
 			rx_itr = (10 * rx_itr * rxr->itr) /
 			    ((9 * rx_itr) + rxr->itr);
 			rxr->itr = min(rx_itr, IXL_MAX_ITR);
 			wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR,
 			    rxr->me), rxr->itr);
 		}
 	} else { /* We may have have toggled to non-dynamic */
 		if (vsi->rx_itr_setting & IXL_ITR_DYNAMIC)
 			vsi->rx_itr_setting = pf->rx_itr;
 		/* Update the hardware if needed */
 		if (rxr->itr != vsi->rx_itr_setting) {
 			rxr->itr = vsi->rx_itr_setting;
 			wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR,
 			    rxr->me), rxr->itr);
 		}
 	}
 	rxr->bytes = 0;
 	rxr->packets = 0;
 }
 
 
 /*
 ** Provide a update to the queue TX
 ** interrupt moderation value.
 */
 void
 ixl_set_queue_tx_itr(struct ixl_tx_queue *que)
 {
 	struct ixl_vsi	*vsi = que->vsi;
 	struct ixl_pf	*pf = (struct ixl_pf *)vsi->back;
 	struct i40e_hw	*hw = vsi->hw;
 	struct tx_ring	*txr = &que->txr;
 	u16		tx_itr;
 	u16		tx_latency = 0;
 	int		tx_bytes;
 
 
 	/* Idle, do nothing */
 	if (txr->bytes == 0)
 		return;
 
 	if (pf->dynamic_tx_itr) {
 		tx_bytes = txr->bytes/txr->itr;
 		tx_itr = txr->itr;
 
 		switch (txr->latency) {
 		case IXL_LOW_LATENCY:
 			if (tx_bytes > 10) {
 				tx_latency = IXL_AVE_LATENCY;
 				tx_itr = IXL_ITR_20K;
 			}
 			break;
 		case IXL_AVE_LATENCY:
 			if (tx_bytes > 20) {
 				tx_latency = IXL_BULK_LATENCY;
 				tx_itr = IXL_ITR_8K;
 			} else if (tx_bytes <= 10) {
 				tx_latency = IXL_LOW_LATENCY;
 				tx_itr = IXL_ITR_100K;
 			}
 			break;
 		case IXL_BULK_LATENCY:
 			if (tx_bytes <= 20) {
 				tx_latency = IXL_AVE_LATENCY;
 				tx_itr = IXL_ITR_20K;
 			}
 			break;
 		}
 
 		txr->latency = tx_latency;
 
 		if (tx_itr != txr->itr) {
        	         /* do an exponential smoothing */
 			tx_itr = (10 * tx_itr * txr->itr) /
 			    ((9 * tx_itr) + txr->itr);
 			txr->itr = min(tx_itr, IXL_MAX_ITR);
 			wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR,
 			    txr->me), txr->itr);
 		}
 
 	} else { /* We may have have toggled to non-dynamic */
 		if (vsi->tx_itr_setting & IXL_ITR_DYNAMIC)
 			vsi->tx_itr_setting = pf->tx_itr;
 		/* Update the hardware if needed */
 		if (txr->itr != vsi->tx_itr_setting) {
 			txr->itr = vsi->tx_itr_setting;
 			wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR,
 			    txr->me), txr->itr);
 		}
 	}
 	txr->bytes = 0;
 	txr->packets = 0;
 	return;
 }
 
 #ifdef IXL_DEBUG
 /**
  * ixl_sysctl_qtx_tail_handler
  * Retrieves I40E_QTX_TAIL value from hardware
  * for a sysctl.
  */
 int
 ixl_sysctl_qtx_tail_handler(SYSCTL_HANDLER_ARGS)
 {
 	struct ixl_tx_queue *tx_que;
 	int error;
 	u32 val;
 
 	tx_que = ((struct ixl_tx_queue *)oidp->oid_arg1);
 	if (!tx_que) return 0;
 
 	val = rd32(tx_que->vsi->hw, tx_que->txr.tail);
 	error = sysctl_handle_int(oidp, &val, 0, req);
 	if (error || !req->newptr)
 		return error;
 	return (0);
 }
 
 /**
  * ixl_sysctl_qrx_tail_handler
  * Retrieves I40E_QRX_TAIL value from hardware
  * for a sysctl.
  */
 int
 ixl_sysctl_qrx_tail_handler(SYSCTL_HANDLER_ARGS)
 {
 	struct ixl_rx_queue *rx_que;
 	int error;
 	u32 val;
 
 	rx_que = ((struct ixl_rx_queue *)oidp->oid_arg1);
 	if (!rx_que) return 0;
 
 	val = rd32(rx_que->vsi->hw, rx_que->rxr.tail);
 	error = sysctl_handle_int(oidp, &val, 0, req);
 	if (error || !req->newptr)
 		return error;
 	return (0);
 }
 #endif
 
 /*
  * Used to set the Tx ITR value for all of the PF LAN VSI's queues.
  * Writes to the ITR registers immediately.
  */
 static int
 ixl_sysctl_pf_tx_itr(SYSCTL_HANDLER_ARGS)
 {
 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
 	device_t dev = pf->dev;
 	int error = 0;
 	int requested_tx_itr;
 
 	requested_tx_itr = pf->tx_itr;
 	error = sysctl_handle_int(oidp, &requested_tx_itr, 0, req);
 	if ((error) || (req->newptr == NULL))
 		return (error);
 	if (pf->dynamic_tx_itr) {
 		device_printf(dev,
 		    "Cannot set TX itr value while dynamic TX itr is enabled\n");
 		    return (EINVAL);
 	}
 	if (requested_tx_itr < 0 || requested_tx_itr > IXL_MAX_ITR) {
 		device_printf(dev,
 		    "Invalid TX itr value; value must be between 0 and %d\n",
 		        IXL_MAX_ITR);
 		return (EINVAL);
 	}
 
 	pf->tx_itr = requested_tx_itr;
 	ixl_configure_tx_itr(pf);
 
 	return (error);
 }
 
 /*
  * Used to set the Rx ITR value for all of the PF LAN VSI's queues.
  * Writes to the ITR registers immediately.
  */
 static int
 ixl_sysctl_pf_rx_itr(SYSCTL_HANDLER_ARGS)
 {
 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
 	device_t dev = pf->dev;
 	int error = 0;
 	int requested_rx_itr;
 
 	requested_rx_itr = pf->rx_itr;
 	error = sysctl_handle_int(oidp, &requested_rx_itr, 0, req);
 	if ((error) || (req->newptr == NULL))
 		return (error);
 	if (pf->dynamic_rx_itr) {
 		device_printf(dev,
 		    "Cannot set RX itr value while dynamic RX itr is enabled\n");
 		    return (EINVAL);
 	}
 	if (requested_rx_itr < 0 || requested_rx_itr > IXL_MAX_ITR) {
 		device_printf(dev,
 		    "Invalid RX itr value; value must be between 0 and %d\n",
 		        IXL_MAX_ITR);
 		return (EINVAL);
 	}
 
 	pf->rx_itr = requested_rx_itr;
 	ixl_configure_rx_itr(pf);
 
 	return (error);
 }
 
 void
 ixl_add_hw_stats(struct ixl_pf *pf)
 {
 	struct ixl_vsi *vsi = &pf->vsi;
 	device_t dev = iflib_get_dev(vsi->ctx);
 	struct i40e_hw_port_stats *pf_stats = &pf->stats;
 
 	struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
 	struct sysctl_oid *tree = device_get_sysctl_tree(dev);
 	struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
 
 	/* Driver statistics */
 	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "admin_irq",
 			CTLFLAG_RD, &pf->admin_irq,
 			"Admin Queue IRQs received");
 
 	ixl_add_vsi_sysctls(dev, vsi, ctx, "pf");
 
 	ixl_add_queues_sysctls(dev, vsi);
 
 	ixl_add_sysctls_mac_stats(ctx, child, pf_stats);
 }
 
 void
 ixl_add_sysctls_mac_stats(struct sysctl_ctx_list *ctx,
 	struct sysctl_oid_list *child,
 	struct i40e_hw_port_stats *stats)
 {
 	struct sysctl_oid *stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac",
 				    CTLFLAG_RD, NULL, "Mac Statistics");
 	struct sysctl_oid_list *stat_list = SYSCTL_CHILDREN(stat_node);
 
 	struct i40e_eth_stats *eth_stats = &stats->eth;
 	ixl_add_sysctls_eth_stats(ctx, stat_list, eth_stats);
 
 	struct ixl_sysctl_info ctls[] = 
 	{
 		{&stats->crc_errors, "crc_errors", "CRC Errors"},
 		{&stats->illegal_bytes, "illegal_bytes", "Illegal Byte Errors"},
 		{&stats->mac_local_faults, "local_faults", "MAC Local Faults"},
 		{&stats->mac_remote_faults, "remote_faults", "MAC Remote Faults"},
 		{&stats->rx_length_errors, "rx_length_errors", "Receive Length Errors"},
 		/* Packet Reception Stats */
 		{&stats->rx_size_64, "rx_frames_64", "64 byte frames received"},
 		{&stats->rx_size_127, "rx_frames_65_127", "65-127 byte frames received"},
 		{&stats->rx_size_255, "rx_frames_128_255", "128-255 byte frames received"},
 		{&stats->rx_size_511, "rx_frames_256_511", "256-511 byte frames received"},
 		{&stats->rx_size_1023, "rx_frames_512_1023", "512-1023 byte frames received"},
 		{&stats->rx_size_1522, "rx_frames_1024_1522", "1024-1522 byte frames received"},
 		{&stats->rx_size_big, "rx_frames_big", "1523-9522 byte frames received"},
 		{&stats->rx_undersize, "rx_undersize", "Undersized packets received"},
 		{&stats->rx_fragments, "rx_fragmented", "Fragmented packets received"},
 		{&stats->rx_oversize, "rx_oversized", "Oversized packets received"},
 		{&stats->rx_jabber, "rx_jabber", "Received Jabber"},
 		{&stats->checksum_error, "checksum_errors", "Checksum Errors"},
 		/* Packet Transmission Stats */
 		{&stats->tx_size_64, "tx_frames_64", "64 byte frames transmitted"},
 		{&stats->tx_size_127, "tx_frames_65_127", "65-127 byte frames transmitted"},
 		{&stats->tx_size_255, "tx_frames_128_255", "128-255 byte frames transmitted"},
 		{&stats->tx_size_511, "tx_frames_256_511", "256-511 byte frames transmitted"},
 		{&stats->tx_size_1023, "tx_frames_512_1023", "512-1023 byte frames transmitted"},
 		{&stats->tx_size_1522, "tx_frames_1024_1522", "1024-1522 byte frames transmitted"},
 		{&stats->tx_size_big, "tx_frames_big", "1523-9522 byte frames transmitted"},
 		/* Flow control */
 		{&stats->link_xon_tx, "xon_txd", "Link XON transmitted"},
 		{&stats->link_xon_rx, "xon_recvd", "Link XON received"},
 		{&stats->link_xoff_tx, "xoff_txd", "Link XOFF transmitted"},
 		{&stats->link_xoff_rx, "xoff_recvd", "Link XOFF received"},
 		/* End */
 		{0,0,0}
 	};
 
 	struct ixl_sysctl_info *entry = ctls;
 	while (entry->stat != 0)
 	{
 		SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, entry->name,
 				CTLFLAG_RD, entry->stat,
 				entry->description);
 		entry++;
 	}
 }
 
 void
 ixl_set_rss_key(struct ixl_pf *pf)
 {
 	struct i40e_hw *hw = &pf->hw;
 	struct ixl_vsi *vsi = &pf->vsi;
 	device_t	dev = pf->dev;
 	u32 rss_seed[IXL_RSS_KEY_SIZE_REG];
 	enum i40e_status_code status;
 
 #ifdef RSS
         /* Fetch the configured RSS key */
         rss_getkey((uint8_t *) &rss_seed);
 #else
 	ixl_get_default_rss_key(rss_seed);
 #endif
 	/* Fill out hash function seed */
 	if (hw->mac.type == I40E_MAC_X722) {
 		struct i40e_aqc_get_set_rss_key_data key_data;
 		bcopy(rss_seed, &key_data, 52);
 		status = i40e_aq_set_rss_key(hw, vsi->vsi_num, &key_data);
 		if (status)
 			device_printf(dev,
 			    "i40e_aq_set_rss_key status %s, error %s\n",
 			    i40e_stat_str(hw, status),
 			    i40e_aq_str(hw, hw->aq.asq_last_status));
 	} else {
 		for (int i = 0; i < IXL_RSS_KEY_SIZE_REG; i++)
 			i40e_write_rx_ctl(hw, I40E_PFQF_HKEY(i), rss_seed[i]);
 	}
 }
 
 /*
  * Configure enabled PCTYPES for RSS.
  */
 void
 ixl_set_rss_pctypes(struct ixl_pf *pf)
 {
 	struct i40e_hw *hw = &pf->hw;
 	u64		set_hena = 0, hena;
 
 #ifdef RSS
 	u32		rss_hash_config;
 
 	rss_hash_config = rss_gethashconfig();
 	if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4)
                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER);
 	if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4)
                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP);
 	if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4)
                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP);
 	if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6)
                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER);
 	if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX)
 		set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6);
 	if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6)
                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP);
         if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6)
                 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP);
 #else
 	if (hw->mac.type == I40E_MAC_X722)
 		set_hena = IXL_DEFAULT_RSS_HENA_X722;
 	else
 		set_hena = IXL_DEFAULT_RSS_HENA_XL710;
 #endif
 	hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) |
 	    ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32);
 	hena |= set_hena;
 	i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena);
 	i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));
 
 }
 
 void
 ixl_set_rss_hlut(struct ixl_pf *pf)
 {
 	struct i40e_hw	*hw = &pf->hw;
 	struct ixl_vsi *vsi = &pf->vsi;
 	device_t	dev = iflib_get_dev(vsi->ctx);
 	int		i, que_id;
 	int		lut_entry_width;
 	u32		lut = 0;
 	enum i40e_status_code status;
 
 	lut_entry_width = pf->hw.func_caps.rss_table_entry_width;
 
 	/* Populate the LUT with max no. of queues in round robin fashion */
 	u8 hlut_buf[512];
 	for (i = 0; i < pf->hw.func_caps.rss_table_size; i++) {
 #ifdef RSS
 		/*
 		 * Fetch the RSS bucket id for the given indirection entry.
 		 * Cap it at the number of configured buckets (which is
 		 * num_queues.)
 		 */
 		que_id = rss_get_indirection_to_bucket(i);
 		que_id = que_id % vsi->num_rx_queues;
 #else
 		que_id = i % vsi->num_rx_queues;
 #endif
 		lut = (que_id & ((0x1 << lut_entry_width) - 1));
 		hlut_buf[i] = lut;
 	}
 
 	if (hw->mac.type == I40E_MAC_X722) {
 		status = i40e_aq_set_rss_lut(hw, vsi->vsi_num, TRUE, hlut_buf, sizeof(hlut_buf));
 		if (status)
 			device_printf(dev, "i40e_aq_set_rss_lut status %s, error %s\n",
 			    i40e_stat_str(hw, status), i40e_aq_str(hw, hw->aq.asq_last_status));
 	} else {
 		for (i = 0; i < pf->hw.func_caps.rss_table_size >> 2; i++)
 			wr32(hw, I40E_PFQF_HLUT(i), ((u32 *)hlut_buf)[i]);
 		ixl_flush(hw);
 	}
 }
 
 /*
 ** Setup the PF's RSS parameters.
 */
 void
 ixl_config_rss(struct ixl_pf *pf)
 {
 	ixl_set_rss_key(pf);
 	ixl_set_rss_pctypes(pf);
 	ixl_set_rss_hlut(pf);
 }
 
 /*
 ** This routine updates vlan filters, called by init
 ** it scans the filter table and then updates the hw
 ** after a soft reset.
 */
 void
 ixl_setup_vlan_filters(struct ixl_vsi *vsi)
 {
 	struct ixl_mac_filter	*f;
 	int			cnt = 0, flags;
 
 	if (vsi->num_vlans == 0)
 		return;
 	/*
 	** Scan the filter list for vlan entries,
 	** mark them for addition and then call
 	** for the AQ update.
 	*/
 	SLIST_FOREACH(f, &vsi->ftl, next) {
 		if (f->flags & IXL_FILTER_VLAN) {
 			f->flags |=
 			    (IXL_FILTER_ADD |
 			    IXL_FILTER_USED);
 			cnt++;
 		}
 	}
 	if (cnt == 0) {
 		printf("setup vlan: no filters found!\n");
 		return;
 	}
 	flags = IXL_FILTER_VLAN;
 	flags |= (IXL_FILTER_ADD | IXL_FILTER_USED);
 	ixl_add_hw_filters(vsi, flags, cnt);
 }
 
 /*
  * In some firmware versions there is default MAC/VLAN filter
  * configured which interferes with filters managed by driver.
  * Make sure it's removed.
  */
 void
 ixl_del_default_hw_filters(struct ixl_vsi *vsi)
 {
 	struct i40e_aqc_remove_macvlan_element_data e;
 
 	bzero(&e, sizeof(e));
 	bcopy(vsi->hw->mac.perm_addr, e.mac_addr, ETHER_ADDR_LEN);
 	e.vlan_tag = 0;
 	e.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
 	i40e_aq_remove_macvlan(vsi->hw, vsi->seid, &e, 1, NULL);
 
 	bzero(&e, sizeof(e));
 	bcopy(vsi->hw->mac.perm_addr, e.mac_addr, ETHER_ADDR_LEN);
 	e.vlan_tag = 0;
 	e.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |
 		I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
 	i40e_aq_remove_macvlan(vsi->hw, vsi->seid, &e, 1, NULL);
 }
 
 /*
 ** Initialize filter list and add filters that the hardware
 ** needs to know about.
 **
 ** Requires VSI's filter list & seid to be set before calling.
 */
 void
 ixl_init_filters(struct ixl_vsi *vsi)
 {
 	struct ixl_pf *pf = (struct ixl_pf *)vsi->back;
 
 	/* Initialize mac filter list for VSI */
 	SLIST_INIT(&vsi->ftl);
 
 	/* Receive broadcast Ethernet frames */
 	i40e_aq_set_vsi_broadcast(&pf->hw, vsi->seid, TRUE, NULL);
 
 	ixl_del_default_hw_filters(vsi);
 
 	ixl_add_filter(vsi, vsi->hw->mac.addr, IXL_VLAN_ANY);
 	/*
 	 * Prevent Tx flow control frames from being sent out by
 	 * non-firmware transmitters.
 	 * This affects every VSI in the PF.
 	 */
 	if (pf->enable_tx_fc_filter)
 		i40e_add_filter_to_drop_tx_flow_control_frames(vsi->hw, vsi->seid);
 }
 
 /*
 ** This routine adds mulicast filters
 */
 void
 ixl_add_mc_filter(struct ixl_vsi *vsi, u8 *macaddr)
 {
 	struct ixl_mac_filter *f;
 
 	/* Does one already exist */
 	f = ixl_find_filter(vsi, macaddr, IXL_VLAN_ANY);
 	if (f != NULL)
 		return;
 
 	f = ixl_new_filter(vsi, macaddr, IXL_VLAN_ANY);
 	if (f != NULL)
 		f->flags |= IXL_FILTER_MC;
 	else
 		printf("WARNING: no filter available!!\n");
 }
 
 void
 ixl_reconfigure_filters(struct ixl_vsi *vsi)
 {
 	ixl_add_hw_filters(vsi, IXL_FILTER_USED, vsi->num_macs);
 }
 
 /*
  * This routine adds a MAC/VLAN filter to the software filter
  * list, then adds that new filter to the HW if it doesn't already
  * exist in the SW filter list.
  */
 void
 ixl_add_filter(struct ixl_vsi *vsi, const u8 *macaddr, s16 vlan)
 {
 	struct ixl_mac_filter	*f, *tmp;
 	struct ixl_pf		*pf;
 	device_t		dev;
 
 	DEBUGOUT("ixl_add_filter: begin");
 
 	pf = vsi->back;
 	dev = pf->dev;
 
 	/* Does one already exist */
 	f = ixl_find_filter(vsi, macaddr, vlan);
 	if (f != NULL)
 		return;
 	/*
 	** Is this the first vlan being registered, if so we
 	** need to remove the ANY filter that indicates we are
 	** not in a vlan, and replace that with a 0 filter.
 	*/
 	if ((vlan != IXL_VLAN_ANY) && (vsi->num_vlans == 1)) {
 		tmp = ixl_find_filter(vsi, macaddr, IXL_VLAN_ANY);
 		if (tmp != NULL) {
 			ixl_del_filter(vsi, macaddr, IXL_VLAN_ANY);
 			ixl_add_filter(vsi, macaddr, 0);
 		}
 	}
 
 	f = ixl_new_filter(vsi, macaddr, vlan);
 	if (f == NULL) {
 		device_printf(dev, "WARNING: no filter available!!\n");
 		return;
 	}
 	if (f->vlan != IXL_VLAN_ANY)
 		f->flags |= IXL_FILTER_VLAN;
 	else
 		vsi->num_macs++;
 
 	f->flags |= IXL_FILTER_USED;
 	ixl_add_hw_filters(vsi, f->flags, 1);
 }
 
 void
 ixl_del_filter(struct ixl_vsi *vsi, const u8 *macaddr, s16 vlan)
 {
 	struct ixl_mac_filter *f;
 
 	f = ixl_find_filter(vsi, macaddr, vlan);
 	if (f == NULL)
 		return;
 
 	f->flags |= IXL_FILTER_DEL;
 	ixl_del_hw_filters(vsi, 1);
 	if (f->vlan == IXL_VLAN_ANY && (f->flags & IXL_FILTER_VLAN) != 0)
 		vsi->num_macs--;
 
 	/* Check if this is the last vlan removal */
 	if (vlan != IXL_VLAN_ANY && vsi->num_vlans == 0) {
 		/* Switch back to a non-vlan filter */
 		ixl_del_filter(vsi, macaddr, 0);
 		ixl_add_filter(vsi, macaddr, IXL_VLAN_ANY);
 	}
 	return;
 }
 
 /*
 ** Find the filter with both matching mac addr and vlan id
 */
 struct ixl_mac_filter *
 ixl_find_filter(struct ixl_vsi *vsi, const u8 *macaddr, s16 vlan)
 {
 	struct ixl_mac_filter	*f;
 
 	SLIST_FOREACH(f, &vsi->ftl, next) {
 		if ((cmp_etheraddr(f->macaddr, macaddr) != 0)
 		    && (f->vlan == vlan)) {
 			return (f);
 		}
 	}	
 
 	return (NULL);
 }
 
 /*
 ** This routine takes additions to the vsi filter
 ** table and creates an Admin Queue call to create
 ** the filters in the hardware.
 */
 void
 ixl_add_hw_filters(struct ixl_vsi *vsi, int flags, int cnt)
 {
 	struct i40e_aqc_add_macvlan_element_data *a, *b;
 	struct ixl_mac_filter	*f;
 	struct ixl_pf		*pf;
 	struct i40e_hw		*hw;
 	device_t		dev;
 	enum i40e_status_code	status;
 	int			j = 0;
 
 	pf = vsi->back;
 	dev = vsi->dev;
 	hw = &pf->hw;
 
 	if (cnt < 1) {
 		ixl_dbg_info(pf, "ixl_add_hw_filters: cnt == 0\n");
 		return;
 	}
 
 	a = malloc(sizeof(struct i40e_aqc_add_macvlan_element_data) * cnt,
 	    M_DEVBUF, M_NOWAIT | M_ZERO);
 	if (a == NULL) {
 		device_printf(dev, "add_hw_filters failed to get memory\n");
 		return;
 	}
 
 	/*
 	** Scan the filter list, each time we find one
 	** we add it to the admin queue array and turn off
 	** the add bit.
 	*/
 	SLIST_FOREACH(f, &vsi->ftl, next) {
 		if ((f->flags & flags) == flags) {
 			b = &a[j]; // a pox on fvl long names :)
 			bcopy(f->macaddr, b->mac_addr, ETHER_ADDR_LEN);
 			if (f->vlan == IXL_VLAN_ANY) {
 				b->vlan_tag = 0;
 				b->flags = I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
 			} else {
 				b->vlan_tag = f->vlan;
 				b->flags = 0;
 			}
 			b->flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
 			f->flags &= ~IXL_FILTER_ADD;
 			j++;
 
 			ixl_dbg_filter(pf, "ADD: " MAC_FORMAT "\n",
 			    MAC_FORMAT_ARGS(f->macaddr));
 		}
 		if (j == cnt)
 			break;
 	}
 	if (j > 0) {
 		status = i40e_aq_add_macvlan(hw, vsi->seid, a, j, NULL);
 		if (status)
 			device_printf(dev, "i40e_aq_add_macvlan status %s, "
 			    "error %s\n", i40e_stat_str(hw, status),
 			    i40e_aq_str(hw, hw->aq.asq_last_status));
 		else
 			vsi->hw_filters_add += j;
 	}
 	free(a, M_DEVBUF);
 	return;
 }
 
 /*
 ** This routine takes removals in the vsi filter
 ** table and creates an Admin Queue call to delete
 ** the filters in the hardware.
 */
 void
 ixl_del_hw_filters(struct ixl_vsi *vsi, int cnt)
 {
 	struct i40e_aqc_remove_macvlan_element_data *d, *e;
 	struct ixl_pf		*pf;
 	struct i40e_hw		*hw;
 	device_t		dev;
 	struct ixl_mac_filter	*f, *f_temp;
 	enum i40e_status_code	status;
 	int			j = 0;
 
 	pf = vsi->back;
 	hw = &pf->hw;
 	dev = vsi->dev;
 
 	d = malloc(sizeof(struct i40e_aqc_remove_macvlan_element_data) * cnt,
 	    M_DEVBUF, M_NOWAIT | M_ZERO);
 	if (d == NULL) {
 		device_printf(dev, "%s: failed to get memory\n", __func__);
 		return;
 	}
 
 	SLIST_FOREACH_SAFE(f, &vsi->ftl, next, f_temp) {
 		if (f->flags & IXL_FILTER_DEL) {
 			e = &d[j]; // a pox on fvl long names :)
 			bcopy(f->macaddr, e->mac_addr, ETHER_ADDR_LEN);
 			e->flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
 			if (f->vlan == IXL_VLAN_ANY) {
 				e->vlan_tag = 0;
 				e->flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
 			} else {
 				e->vlan_tag = f->vlan;
 			}
 
 			ixl_dbg_filter(pf, "DEL: " MAC_FORMAT "\n",
 			    MAC_FORMAT_ARGS(f->macaddr));
 
 			/* delete entry from vsi list */
 			SLIST_REMOVE(&vsi->ftl, f, ixl_mac_filter, next);
 			free(f, M_DEVBUF);
 			j++;
 		}
 		if (j == cnt)
 			break;
 	}
 	if (j > 0) {
 		status = i40e_aq_remove_macvlan(hw, vsi->seid, d, j, NULL);
 		if (status) {
 			int sc = 0;
 			for (int i = 0; i < j; i++)
 				sc += (!d[i].error_code);
 			vsi->hw_filters_del += sc;
 			device_printf(dev,
 			    "Failed to remove %d/%d filters, error %s\n",
 			    j - sc, j, i40e_aq_str(hw, hw->aq.asq_last_status));
 		} else
 			vsi->hw_filters_del += j;
 	}
 	free(d, M_DEVBUF);
 	return;
 }
 
 int
 ixl_enable_tx_ring(struct ixl_pf *pf, struct ixl_pf_qtag *qtag, u16 vsi_qidx)
 {
 	struct i40e_hw	*hw = &pf->hw;
 	int		error = 0;
 	u32		reg;
 	u16		pf_qidx;
 
 	pf_qidx = ixl_pf_qidx_from_vsi_qidx(qtag, vsi_qidx);
 
 	ixl_dbg(pf, IXL_DBG_EN_DIS,
 	    "Enabling PF TX ring %4d / VSI TX ring %4d...\n",
 	    pf_qidx, vsi_qidx);
 
 	i40e_pre_tx_queue_cfg(hw, pf_qidx, TRUE);
 
 	reg = rd32(hw, I40E_QTX_ENA(pf_qidx));
 	reg |= I40E_QTX_ENA_QENA_REQ_MASK |
 	    I40E_QTX_ENA_QENA_STAT_MASK;
 	wr32(hw, I40E_QTX_ENA(pf_qidx), reg);
 	/* Verify the enable took */
 	for (int j = 0; j < 10; j++) {
 		reg = rd32(hw, I40E_QTX_ENA(pf_qidx));
 		if (reg & I40E_QTX_ENA_QENA_STAT_MASK)
 			break;
 		i40e_usec_delay(10);
 	}
 	if ((reg & I40E_QTX_ENA_QENA_STAT_MASK) == 0) {
 		device_printf(pf->dev, "TX queue %d still disabled!\n",
 		    pf_qidx);
 		error = ETIMEDOUT;
 	}
 
 	return (error);
 }
 
 int
 ixl_enable_rx_ring(struct ixl_pf *pf, struct ixl_pf_qtag *qtag, u16 vsi_qidx)
 {
 	struct i40e_hw	*hw = &pf->hw;
 	int		error = 0;
 	u32		reg;
 	u16		pf_qidx;
 
 	pf_qidx = ixl_pf_qidx_from_vsi_qidx(qtag, vsi_qidx);
 
 	ixl_dbg(pf, IXL_DBG_EN_DIS,
 	    "Enabling PF RX ring %4d / VSI RX ring %4d...\n",
 	    pf_qidx, vsi_qidx);
 
 	reg = rd32(hw, I40E_QRX_ENA(pf_qidx));
 	reg |= I40E_QRX_ENA_QENA_REQ_MASK |
 	    I40E_QRX_ENA_QENA_STAT_MASK;
 	wr32(hw, I40E_QRX_ENA(pf_qidx), reg);
 	/* Verify the enable took */
 	for (int j = 0; j < 10; j++) {
 		reg = rd32(hw, I40E_QRX_ENA(pf_qidx));
 		if (reg & I40E_QRX_ENA_QENA_STAT_MASK)
 			break;
 		i40e_usec_delay(10);
 	}
 	if ((reg & I40E_QRX_ENA_QENA_STAT_MASK) == 0) {
 		device_printf(pf->dev, "RX queue %d still disabled!\n",
 		    pf_qidx);
 		error = ETIMEDOUT;
 	}
 
 	return (error);
 }
 
 int
 ixl_enable_ring(struct ixl_pf *pf, struct ixl_pf_qtag *qtag, u16 vsi_qidx)
 {
 	int error = 0;
 
 	error = ixl_enable_tx_ring(pf, qtag, vsi_qidx);
 	/* Called function already prints error message */
 	if (error)
 		return (error);
 	error = ixl_enable_rx_ring(pf, qtag, vsi_qidx);
 	return (error);
 }
 
 /* For PF VSI only */
 int
 ixl_enable_rings(struct ixl_vsi *vsi)
 {
 	struct ixl_pf	*pf = vsi->back;
 	int		error = 0;
 
 	for (int i = 0; i < vsi->num_tx_queues; i++)
 		error = ixl_enable_tx_ring(pf, &pf->qtag, i);
 
 	for (int i = 0; i < vsi->num_rx_queues; i++)
 		error = ixl_enable_rx_ring(pf, &pf->qtag, i);
 
 	return (error);
 }
 
 /*
  * Returns error on first ring that is detected hung.
  */
 int
 ixl_disable_tx_ring(struct ixl_pf *pf, struct ixl_pf_qtag *qtag, u16 vsi_qidx)
 {
 	struct i40e_hw	*hw = &pf->hw;
 	int		error = 0;
 	u32		reg;
 	u16		pf_qidx;
 
 	pf_qidx = ixl_pf_qidx_from_vsi_qidx(qtag, vsi_qidx);
 
 	i40e_pre_tx_queue_cfg(hw, pf_qidx, FALSE);
 	i40e_usec_delay(500);
 
 	reg = rd32(hw, I40E_QTX_ENA(pf_qidx));
 	reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
 	wr32(hw, I40E_QTX_ENA(pf_qidx), reg);
 	/* Verify the disable took */
 	for (int j = 0; j < 10; j++) {
 		reg = rd32(hw, I40E_QTX_ENA(pf_qidx));
 		if (!(reg & I40E_QTX_ENA_QENA_STAT_MASK))
 			break;
 		i40e_msec_delay(10);
 	}
 	if (reg & I40E_QTX_ENA_QENA_STAT_MASK) {
 		device_printf(pf->dev, "TX queue %d still enabled!\n",
 		    pf_qidx);
 		error = ETIMEDOUT;
 	}
 
 	return (error);
 }
 
 /*
  * Returns error on first ring that is detected hung.
  */
 int
 ixl_disable_rx_ring(struct ixl_pf *pf, struct ixl_pf_qtag *qtag, u16 vsi_qidx)
 {
 	struct i40e_hw	*hw = &pf->hw;
 	int		error = 0;
 	u32		reg;
 	u16		pf_qidx;
 
 	pf_qidx = ixl_pf_qidx_from_vsi_qidx(qtag, vsi_qidx);
 
 	reg = rd32(hw, I40E_QRX_ENA(pf_qidx));
 	reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
 	wr32(hw, I40E_QRX_ENA(pf_qidx), reg);
 	/* Verify the disable took */
 	for (int j = 0; j < 10; j++) {
 		reg = rd32(hw, I40E_QRX_ENA(pf_qidx));
 		if (!(reg & I40E_QRX_ENA_QENA_STAT_MASK))
 			break;
 		i40e_msec_delay(10);
 	}
 	if (reg & I40E_QRX_ENA_QENA_STAT_MASK) {
 		device_printf(pf->dev, "RX queue %d still enabled!\n",
 		    pf_qidx);
 		error = ETIMEDOUT;
 	}
 
 	return (error);
 }
 
 int
 ixl_disable_ring(struct ixl_pf *pf, struct ixl_pf_qtag *qtag, u16 vsi_qidx)
 {
 	int error = 0;
 
 	error = ixl_disable_tx_ring(pf, qtag, vsi_qidx);
 	/* Called function already prints error message */
 	if (error)
 		return (error);
 	error = ixl_disable_rx_ring(pf, qtag, vsi_qidx);
 	return (error);
 }
 
 int
 ixl_disable_rings(struct ixl_pf *pf, struct ixl_vsi *vsi, struct ixl_pf_qtag *qtag)
 {
 	int error = 0;
 
 	for (int i = 0; i < vsi->num_tx_queues; i++)
 		error = ixl_disable_tx_ring(pf, qtag, i);
 
 	for (int i = 0; i < vsi->num_rx_queues; i++)
 		error = ixl_disable_rx_ring(pf, qtag, i);
 
 	return (error);
 }
 
 static void
 ixl_handle_tx_mdd_event(struct ixl_pf *pf)
 {
 	struct i40e_hw *hw = &pf->hw;
 	device_t dev = pf->dev;
 	struct ixl_vf *vf;
 	bool mdd_detected = false;
 	bool pf_mdd_detected = false;
 	bool vf_mdd_detected = false;
 	u16 vf_num, queue;
 	u8 pf_num, event;
 	u8 pf_mdet_num, vp_mdet_num;
 	u32 reg;
 
 	/* find what triggered the MDD event */
 	reg = rd32(hw, I40E_GL_MDET_TX);
 	if (reg & I40E_GL_MDET_TX_VALID_MASK) {
 		pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >>
 		    I40E_GL_MDET_TX_PF_NUM_SHIFT;
 		vf_num = (reg & I40E_GL_MDET_TX_VF_NUM_MASK) >>
 		    I40E_GL_MDET_TX_VF_NUM_SHIFT;
 		event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >>
 		    I40E_GL_MDET_TX_EVENT_SHIFT;
 		queue = (reg & I40E_GL_MDET_TX_QUEUE_MASK) >>
 		    I40E_GL_MDET_TX_QUEUE_SHIFT;
 		wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
 		mdd_detected = true;
 	}
 
 	if (!mdd_detected)
 		return;
 
 	reg = rd32(hw, I40E_PF_MDET_TX);
 	if (reg & I40E_PF_MDET_TX_VALID_MASK) {
 		wr32(hw, I40E_PF_MDET_TX, 0xFFFF);
 		pf_mdet_num = hw->pf_id;
 		pf_mdd_detected = true;
 	}
 
 	/* Check if MDD was caused by a VF */
 	for (int i = 0; i < pf->num_vfs; i++) {
 		vf = &(pf->vfs[i]);
 		reg = rd32(hw, I40E_VP_MDET_TX(i));
 		if (reg & I40E_VP_MDET_TX_VALID_MASK) {
 			wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF);
 			vp_mdet_num = i;
 			vf->num_mdd_events++;
 			vf_mdd_detected = true;
 		}
 	}
 
 	/* Print out an error message */
 	if (vf_mdd_detected && pf_mdd_detected)
 		device_printf(dev,
 		    "Malicious Driver Detection event %d"
 		    " on TX queue %d, pf number %d (PF-%d), vf number %d (VF-%d)\n",
 		    event, queue, pf_num, pf_mdet_num, vf_num, vp_mdet_num);
 	else if (vf_mdd_detected && !pf_mdd_detected)
 		device_printf(dev,
 		    "Malicious Driver Detection event %d"
 		    " on TX queue %d, pf number %d, vf number %d (VF-%d)\n",
 		    event, queue, pf_num, vf_num, vp_mdet_num);
 	else if (!vf_mdd_detected && pf_mdd_detected)
 		device_printf(dev,
 		    "Malicious Driver Detection event %d"
 		    " on TX queue %d, pf number %d (PF-%d)\n",
 		    event, queue, pf_num, pf_mdet_num);
 	/* Theoretically shouldn't happen */
 	else
 		device_printf(dev,
 		    "TX Malicious Driver Detection event (unknown)\n");
 }
 
 static void
 ixl_handle_rx_mdd_event(struct ixl_pf *pf)
 {
 	struct i40e_hw *hw = &pf->hw;
 	device_t dev = pf->dev;
 	struct ixl_vf *vf;
 	bool mdd_detected = false;
 	bool pf_mdd_detected = false;
 	bool vf_mdd_detected = false;
 	u16 queue;
 	u8 pf_num, event;
 	u8 pf_mdet_num, vp_mdet_num;
 	u32 reg;
 
 	/*
 	 * GL_MDET_RX doesn't contain VF number information, unlike
 	 * GL_MDET_TX.
 	 */
 	reg = rd32(hw, I40E_GL_MDET_RX);
 	if (reg & I40E_GL_MDET_RX_VALID_MASK) {
 		pf_num = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >>
 		    I40E_GL_MDET_RX_FUNCTION_SHIFT;
 		event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >>
 		    I40E_GL_MDET_RX_EVENT_SHIFT;
 		queue = (reg & I40E_GL_MDET_RX_QUEUE_MASK) >>
 		    I40E_GL_MDET_RX_QUEUE_SHIFT;
 		wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
 		mdd_detected = true;
 	}
 
 	if (!mdd_detected)
 		return;
 
 	reg = rd32(hw, I40E_PF_MDET_RX);
 	if (reg & I40E_PF_MDET_RX_VALID_MASK) {
 		wr32(hw, I40E_PF_MDET_RX, 0xFFFF);
 		pf_mdet_num = hw->pf_id;
 		pf_mdd_detected = true;
 	}
 
 	/* Check if MDD was caused by a VF */
 	for (int i = 0; i < pf->num_vfs; i++) {
 		vf = &(pf->vfs[i]);
 		reg = rd32(hw, I40E_VP_MDET_RX(i));
 		if (reg & I40E_VP_MDET_RX_VALID_MASK) {
 			wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF);
 			vp_mdet_num = i;
 			vf->num_mdd_events++;
 			vf_mdd_detected = true;
 		}
 	}
 
 	/* Print out an error message */
 	if (vf_mdd_detected && pf_mdd_detected)
 		device_printf(dev,
 		    "Malicious Driver Detection event %d"
 		    " on RX queue %d, pf number %d (PF-%d), (VF-%d)\n",
 		    event, queue, pf_num, pf_mdet_num, vp_mdet_num);
 	else if (vf_mdd_detected && !pf_mdd_detected)
 		device_printf(dev,
 		    "Malicious Driver Detection event %d"
 		    " on RX queue %d, pf number %d, (VF-%d)\n",
 		    event, queue, pf_num, vp_mdet_num);
 	else if (!vf_mdd_detected && pf_mdd_detected)
 		device_printf(dev,
 		    "Malicious Driver Detection event %d"
 		    " on RX queue %d, pf number %d (PF-%d)\n",
 		    event, queue, pf_num, pf_mdet_num);
 	/* Theoretically shouldn't happen */
 	else
 		device_printf(dev,
 		    "RX Malicious Driver Detection event (unknown)\n");
 }
 
 /**
  * ixl_handle_mdd_event
  *
  * Called from interrupt handler to identify possibly malicious vfs
  * (But also detects events from the PF, as well)
  **/
 void
 ixl_handle_mdd_event(struct ixl_pf *pf)
 {
 	struct i40e_hw *hw = &pf->hw;
 	u32 reg;
 
 	/*
 	 * Handle both TX/RX because it's possible they could
 	 * both trigger in the same interrupt.
 	 */
 	ixl_handle_tx_mdd_event(pf);
 	ixl_handle_rx_mdd_event(pf);
 
 	atomic_clear_32(&pf->state, IXL_PF_STATE_MDD_PENDING);
 
 	/* re-enable mdd interrupt cause */
 	reg = rd32(hw, I40E_PFINT_ICR0_ENA);
 	reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
 	wr32(hw, I40E_PFINT_ICR0_ENA, reg);
 	ixl_flush(hw);
 }
 
 void
 ixl_enable_intr(struct ixl_vsi *vsi)
 {
 	struct i40e_hw		*hw = vsi->hw;
 	struct ixl_rx_queue	*que = vsi->rx_queues;
 
 	if (vsi->shared->isc_intr == IFLIB_INTR_MSIX) {
 		for (int i = 0; i < vsi->num_rx_queues; i++, que++)
 			ixl_enable_queue(hw, que->rxr.me);
 	} else
 		ixl_enable_intr0(hw);
 }
 
 void
 ixl_disable_rings_intr(struct ixl_vsi *vsi)
 {
 	struct i40e_hw		*hw = vsi->hw;
 	struct ixl_rx_queue	*que = vsi->rx_queues;
 
 	for (int i = 0; i < vsi->num_rx_queues; i++, que++)
 		ixl_disable_queue(hw, que->rxr.me);
 }
 
 void
 ixl_enable_intr0(struct i40e_hw *hw)
 {
 	u32		reg;
 
 	/* Use IXL_ITR_NONE so ITR isn't updated here */
 	reg = I40E_PFINT_DYN_CTL0_INTENA_MASK |
 	    I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
 	    (IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
 	wr32(hw, I40E_PFINT_DYN_CTL0, reg);
 }
 
 void
 ixl_disable_intr0(struct i40e_hw *hw)
 {
 	u32		reg;
 
 	reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT;
 	wr32(hw, I40E_PFINT_DYN_CTL0, reg);
 	ixl_flush(hw);
 }
 
 void
 ixl_enable_queue(struct i40e_hw *hw, int id)
 {
 	u32		reg;
 
 	reg = I40E_PFINT_DYN_CTLN_INTENA_MASK |
 	    I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
 	    (IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
 	wr32(hw, I40E_PFINT_DYN_CTLN(id), reg);
 }
 
 void
 ixl_disable_queue(struct i40e_hw *hw, int id)
 {
 	u32		reg;
 
 	reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT;
 	wr32(hw, I40E_PFINT_DYN_CTLN(id), reg);
 }
 
 void
 ixl_update_stats_counters(struct ixl_pf *pf)
 {
 	struct i40e_hw	*hw = &pf->hw;
 	struct ixl_vsi	*vsi = &pf->vsi;
 	struct ixl_vf	*vf;
 
 	struct i40e_hw_port_stats *nsd = &pf->stats;
 	struct i40e_hw_port_stats *osd = &pf->stats_offsets;
 
 	/* Update hw stats */
 	ixl_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
 			   pf->stat_offsets_loaded,
 			   &osd->crc_errors, &nsd->crc_errors);
 	ixl_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
 			   pf->stat_offsets_loaded,
 			   &osd->illegal_bytes, &nsd->illegal_bytes);
 	ixl_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
 			   I40E_GLPRT_GORCL(hw->port),
 			   pf->stat_offsets_loaded,
 			   &osd->eth.rx_bytes, &nsd->eth.rx_bytes);
 	ixl_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
 			   I40E_GLPRT_GOTCL(hw->port),
 			   pf->stat_offsets_loaded,
 			   &osd->eth.tx_bytes, &nsd->eth.tx_bytes);
 	ixl_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
 			   pf->stat_offsets_loaded,
 			   &osd->eth.rx_discards,
 			   &nsd->eth.rx_discards);
 	ixl_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port),
 			   I40E_GLPRT_UPRCL(hw->port),
 			   pf->stat_offsets_loaded,
 			   &osd->eth.rx_unicast,
 			   &nsd->eth.rx_unicast);
 	ixl_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port),
 			   I40E_GLPRT_UPTCL(hw->port),
 			   pf->stat_offsets_loaded,
 			   &osd->eth.tx_unicast,
 			   &nsd->eth.tx_unicast);
 	ixl_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
 			   I40E_GLPRT_MPRCL(hw->port),
 			   pf->stat_offsets_loaded,
 			   &osd->eth.rx_multicast,
 			   &nsd->eth.rx_multicast);
 	ixl_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port),
 			   I40E_GLPRT_MPTCL(hw->port),
 			   pf->stat_offsets_loaded,
 			   &osd->eth.tx_multicast,
 			   &nsd->eth.tx_multicast);
 	ixl_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port),
 			   I40E_GLPRT_BPRCL(hw->port),
 			   pf->stat_offsets_loaded,
 			   &osd->eth.rx_broadcast,
 			   &nsd->eth.rx_broadcast);
 	ixl_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port),
 			   I40E_GLPRT_BPTCL(hw->port),
 			   pf->stat_offsets_loaded,
 			   &osd->eth.tx_broadcast,
 			   &nsd->eth.tx_broadcast);
 
 	ixl_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
 			   pf->stat_offsets_loaded,
 			   &osd->tx_dropped_link_down,
 			   &nsd->tx_dropped_link_down);
 	ixl_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
 			   pf->stat_offsets_loaded,
 			   &osd->mac_local_faults,
 			   &nsd->mac_local_faults);
 	ixl_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
 			   pf->stat_offsets_loaded,
 			   &osd->mac_remote_faults,
 			   &nsd->mac_remote_faults);
 	ixl_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
 			   pf->stat_offsets_loaded,
 			   &osd->rx_length_errors,
 			   &nsd->rx_length_errors);
 
 	/* Flow control (LFC) stats */
 	ixl_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
 			   pf->stat_offsets_loaded,
 			   &osd->link_xon_rx, &nsd->link_xon_rx);
 	ixl_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
 			   pf->stat_offsets_loaded,
 			   &osd->link_xon_tx, &nsd->link_xon_tx);
 	ixl_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
 			   pf->stat_offsets_loaded,
 			   &osd->link_xoff_rx, &nsd->link_xoff_rx);
 	ixl_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
 			   pf->stat_offsets_loaded,
 			   &osd->link_xoff_tx, &nsd->link_xoff_tx);
 
 	/* Packet size stats rx */
 	ixl_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
 			   I40E_GLPRT_PRC64L(hw->port),
 			   pf->stat_offsets_loaded,
 			   &osd->rx_size_64, &nsd->rx_size_64);
 	ixl_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
 			   I40E_GLPRT_PRC127L(hw->port),
 			   pf->stat_offsets_loaded,
 			   &osd->rx_size_127, &nsd->rx_size_127);
 	ixl_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
 			   I40E_GLPRT_PRC255L(hw->port),
 			   pf->stat_offsets_loaded,
 			   &osd->rx_size_255, &nsd->rx_size_255);
 	ixl_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
 			   I40E_GLPRT_PRC511L(hw->port),
 			   pf->stat_offsets_loaded,
 			   &osd->rx_size_511, &nsd->rx_size_511);
 	ixl_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
 			   I40E_GLPRT_PRC1023L(hw->port),
 			   pf->stat_offsets_loaded,
 			   &osd->rx_size_1023, &nsd->rx_size_1023);
 	ixl_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
 			   I40E_GLPRT_PRC1522L(hw->port),
 			   pf->stat_offsets_loaded,
 			   &osd->rx_size_1522, &nsd->rx_size_1522);
 	ixl_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
 			   I40E_GLPRT_PRC9522L(hw->port),
 			   pf->stat_offsets_loaded,
 			   &osd->rx_size_big, &nsd->rx_size_big);
 
 	/* Packet size stats tx */
 	ixl_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
 			   I40E_GLPRT_PTC64L(hw->port),
 			   pf->stat_offsets_loaded,
 			   &osd->tx_size_64, &nsd->tx_size_64);
 	ixl_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
 			   I40E_GLPRT_PTC127L(hw->port),
 			   pf->stat_offsets_loaded,
 			   &osd->tx_size_127, &nsd->tx_size_127);
 	ixl_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
 			   I40E_GLPRT_PTC255L(hw->port),
 			   pf->stat_offsets_loaded,
 			   &osd->tx_size_255, &nsd->tx_size_255);
 	ixl_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
 			   I40E_GLPRT_PTC511L(hw->port),
 			   pf->stat_offsets_loaded,
 			   &osd->tx_size_511, &nsd->tx_size_511);
 	ixl_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
 			   I40E_GLPRT_PTC1023L(hw->port),
 			   pf->stat_offsets_loaded,
 			   &osd->tx_size_1023, &nsd->tx_size_1023);
 	ixl_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
 			   I40E_GLPRT_PTC1522L(hw->port),
 			   pf->stat_offsets_loaded,
 			   &osd->tx_size_1522, &nsd->tx_size_1522);
 	ixl_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
 			   I40E_GLPRT_PTC9522L(hw->port),
 			   pf->stat_offsets_loaded,
 			   &osd->tx_size_big, &nsd->tx_size_big);
 
 	ixl_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
 			   pf->stat_offsets_loaded,
 			   &osd->rx_undersize, &nsd->rx_undersize);
 	ixl_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
 			   pf->stat_offsets_loaded,
 			   &osd->rx_fragments, &nsd->rx_fragments);
 	ixl_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
 			   pf->stat_offsets_loaded,
 			   &osd->rx_oversize, &nsd->rx_oversize);
 	ixl_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
 			   pf->stat_offsets_loaded,
 			   &osd->rx_jabber, &nsd->rx_jabber);
 	pf->stat_offsets_loaded = true;
 	/* End hw stats */
 
 	/* Update vsi stats */
 	ixl_update_vsi_stats(vsi);
 
 	for (int i = 0; i < pf->num_vfs; i++) {
 		vf = &pf->vfs[i];
 		if (vf->vf_flags & VF_FLAG_ENABLED)
 			ixl_update_eth_stats(&pf->vfs[i].vsi);
 	}
 }
 
 int
 ixl_prepare_for_reset(struct ixl_pf *pf, bool is_up)
 {
 	struct i40e_hw *hw = &pf->hw;
 	device_t dev = pf->dev;
 	int error = 0;
 
 	error = i40e_shutdown_lan_hmc(hw);
 	if (error)
 		device_printf(dev,
 		    "Shutdown LAN HMC failed with code %d\n", error);
 
 	ixl_disable_intr0(hw);
 
 	error = i40e_shutdown_adminq(hw);
 	if (error)
 		device_printf(dev,
 		    "Shutdown Admin queue failed with code %d\n", error);
 
 	ixl_pf_qmgr_release(&pf->qmgr, &pf->qtag);
 	return (error);
 }
 
 int
 ixl_rebuild_hw_structs_after_reset(struct ixl_pf *pf)
 {
 	struct i40e_hw *hw = &pf->hw;
 	struct ixl_vsi *vsi = &pf->vsi;
 	device_t dev = pf->dev;
 	int error = 0;
 
 	device_printf(dev, "Rebuilding driver state...\n");
 
 	error = i40e_pf_reset(hw);
 	if (error) {
 		device_printf(dev, "PF reset failure %s\n",
 		    i40e_stat_str(hw, error));
 		goto ixl_rebuild_hw_structs_after_reset_err;
 	}
 
 	/* Setup */
 	error = i40e_init_adminq(hw);
 	if (error != 0 && error != I40E_ERR_FIRMWARE_API_VERSION) {
 		device_printf(dev, "Unable to initialize Admin Queue, error %d\n",
 		    error);
 		goto ixl_rebuild_hw_structs_after_reset_err;
 	}
 
 	i40e_clear_pxe_mode(hw);
 
 	error = ixl_get_hw_capabilities(pf);
 	if (error) {
 		device_printf(dev, "ixl_get_hw_capabilities failed: %d\n", error);
 		goto ixl_rebuild_hw_structs_after_reset_err;
 	}
 
 	error = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
 	    hw->func_caps.num_rx_qp, 0, 0);
 	if (error) {
 		device_printf(dev, "init_lan_hmc failed: %d\n", error);
 		goto ixl_rebuild_hw_structs_after_reset_err;
 	}
 
 	error = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
 	if (error) {
 		device_printf(dev, "configure_lan_hmc failed: %d\n", error);
 		goto ixl_rebuild_hw_structs_after_reset_err;
 	}
 
 	/* reserve a contiguous allocation for the PF's VSI */
 	error = ixl_pf_qmgr_alloc_contiguous(&pf->qmgr, vsi->num_tx_queues, &pf->qtag);
 	if (error) {
 		device_printf(dev, "Failed to reserve queues for PF LAN VSI, error %d\n",
 		    error);
 		/* TODO: error handling */
 	}
 
 	error = ixl_switch_config(pf);
 	if (error) {
 		device_printf(dev, "ixl_rebuild_hw_structs_after_reset: ixl_switch_config() failed: %d\n",
 		     error);
 		error = EIO;
 		goto ixl_rebuild_hw_structs_after_reset_err;
 	}
 
 	error = i40e_aq_set_phy_int_mask(hw, IXL_DEFAULT_PHY_INT_MASK,
 	    NULL);
         if (error) {
 		device_printf(dev, "init: i40e_aq_set_phy_mask() failed: err %d,"
 		    " aq_err %d\n", error, hw->aq.asq_last_status);
 		error = EIO;
 		goto ixl_rebuild_hw_structs_after_reset_err;
 	}
 
 	u8 set_fc_err_mask;
 	error = i40e_set_fc(hw, &set_fc_err_mask, true);
 	if (error) {
 		device_printf(dev, "init: setting link flow control failed; retcode %d,"
 		    " fc_err_mask 0x%02x\n", error, set_fc_err_mask);
 		error = EIO;
 		goto ixl_rebuild_hw_structs_after_reset_err;
 	}
 
 	/* Remove default filters reinstalled by FW on reset */
 	ixl_del_default_hw_filters(vsi);
 
 	/* Determine link state */
 	if (ixl_attach_get_link_status(pf)) {
 		error = EINVAL;
 		/* TODO: error handling */
 	}
 
 	i40e_aq_set_dcb_parameters(hw, TRUE, NULL);
 	ixl_get_fw_lldp_status(pf);
 
 	/* Keep admin queue interrupts active while driver is loaded */
 	if (vsi->shared->isc_intr == IFLIB_INTR_MSIX) {
  		ixl_configure_intr0_msix(pf);
  		ixl_enable_intr0(hw);
 	}
 
 	device_printf(dev, "Rebuilding driver state done.\n");
 	return (0);
 
 ixl_rebuild_hw_structs_after_reset_err:
 	device_printf(dev, "Reload the driver to recover\n");
 	return (error);
 }
 
 void
 ixl_handle_empr_reset(struct ixl_pf *pf)
 {
 	struct ixl_vsi	*vsi = &pf->vsi;
 	struct i40e_hw	*hw = &pf->hw;
 	bool is_up = !!(vsi->ifp->if_drv_flags & IFF_DRV_RUNNING);
 	int count = 0;
 	u32 reg;
 
 	ixl_prepare_for_reset(pf, is_up);
 
 	/* Typically finishes within 3-4 seconds */
 	while (count++ < 100) {
 		reg = rd32(hw, I40E_GLGEN_RSTAT)
 			& I40E_GLGEN_RSTAT_DEVSTATE_MASK;
 		if (reg)
 			i40e_msec_delay(100);
 		else
 			break;
 	}
 	ixl_dbg(pf, IXL_DBG_INFO,
 			"Reset wait count: %d\n", count);
 
 	ixl_rebuild_hw_structs_after_reset(pf);
 
 	atomic_clear_int(&pf->state, IXL_PF_STATE_ADAPTER_RESETTING);
 }
 
 /**
  * Update VSI-specific ethernet statistics counters.
  **/
 void
 ixl_update_eth_stats(struct ixl_vsi *vsi)
 {
 	struct ixl_pf *pf = (struct ixl_pf *)vsi->back;
 	struct i40e_hw *hw = &pf->hw;
 	struct i40e_eth_stats *es;
 	struct i40e_eth_stats *oes;
 	struct i40e_hw_port_stats *nsd;
 	u16 stat_idx = vsi->info.stat_counter_idx;
 
 	es = &vsi->eth_stats;
 	oes = &vsi->eth_stats_offsets;
 	nsd = &pf->stats;
 
 	/* Gather up the stats that the hw collects */
 	ixl_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
 			   vsi->stat_offsets_loaded,
 			   &oes->tx_errors, &es->tx_errors);
 	ixl_stat_update32(hw, I40E_GLV_RDPC(stat_idx),
 			   vsi->stat_offsets_loaded,
 			   &oes->rx_discards, &es->rx_discards);
 
 	ixl_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
 			   I40E_GLV_GORCL(stat_idx),
 			   vsi->stat_offsets_loaded,
 			   &oes->rx_bytes, &es->rx_bytes);
 	ixl_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
 			   I40E_GLV_UPRCL(stat_idx),
 			   vsi->stat_offsets_loaded,
 			   &oes->rx_unicast, &es->rx_unicast);
 	ixl_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
 			   I40E_GLV_MPRCL(stat_idx),
 			   vsi->stat_offsets_loaded,
 			   &oes->rx_multicast, &es->rx_multicast);
 	ixl_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
 			   I40E_GLV_BPRCL(stat_idx),
 			   vsi->stat_offsets_loaded,
 			   &oes->rx_broadcast, &es->rx_broadcast);
 
 	ixl_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
 			   I40E_GLV_GOTCL(stat_idx),
 			   vsi->stat_offsets_loaded,
 			   &oes->tx_bytes, &es->tx_bytes);
 	ixl_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
 			   I40E_GLV_UPTCL(stat_idx),
 			   vsi->stat_offsets_loaded,
 			   &oes->tx_unicast, &es->tx_unicast);
 	ixl_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
 			   I40E_GLV_MPTCL(stat_idx),
 			   vsi->stat_offsets_loaded,
 			   &oes->tx_multicast, &es->tx_multicast);
 	ixl_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
 			   I40E_GLV_BPTCL(stat_idx),
 			   vsi->stat_offsets_loaded,
 			   &oes->tx_broadcast, &es->tx_broadcast);
 	vsi->stat_offsets_loaded = true;
 }
 
 void
 ixl_update_vsi_stats(struct ixl_vsi *vsi)
 {
 	struct ixl_pf		*pf;
 	struct ifnet		*ifp;
 	struct i40e_eth_stats	*es;
 	u64			tx_discards;
 
 	struct i40e_hw_port_stats *nsd;
 
 	pf = vsi->back;
 	ifp = vsi->ifp;
 	es = &vsi->eth_stats;
 	nsd = &pf->stats;
 
 	ixl_update_eth_stats(vsi);
 
 	tx_discards = es->tx_discards + nsd->tx_dropped_link_down;
 
 	/* Update ifnet stats */
 	IXL_SET_IPACKETS(vsi, es->rx_unicast +
 	                   es->rx_multicast +
 			   es->rx_broadcast);
 	IXL_SET_OPACKETS(vsi, es->tx_unicast +
 	                   es->tx_multicast +
 			   es->tx_broadcast);
 	IXL_SET_IBYTES(vsi, es->rx_bytes);
 	IXL_SET_OBYTES(vsi, es->tx_bytes);
 	IXL_SET_IMCASTS(vsi, es->rx_multicast);
 	IXL_SET_OMCASTS(vsi, es->tx_multicast);
 
 	IXL_SET_IERRORS(vsi, nsd->crc_errors + nsd->illegal_bytes +
 	    nsd->rx_undersize + nsd->rx_oversize + nsd->rx_fragments +
 	    nsd->rx_jabber);
 	IXL_SET_OERRORS(vsi, es->tx_errors);
 	IXL_SET_IQDROPS(vsi, es->rx_discards + nsd->eth.rx_discards);
 	IXL_SET_OQDROPS(vsi, tx_discards);
 	IXL_SET_NOPROTO(vsi, es->rx_unknown_protocol);
 	IXL_SET_COLLISIONS(vsi, 0);
 }
 
 /**
  * Reset all of the stats for the given pf
  **/
 void
 ixl_pf_reset_stats(struct ixl_pf *pf)
 {
 	bzero(&pf->stats, sizeof(struct i40e_hw_port_stats));
 	bzero(&pf->stats_offsets, sizeof(struct i40e_hw_port_stats));
 	pf->stat_offsets_loaded = false;
 }
 
 /**
  * Resets all stats of the given vsi
  **/
 void
 ixl_vsi_reset_stats(struct ixl_vsi *vsi)
 {
 	bzero(&vsi->eth_stats, sizeof(struct i40e_eth_stats));
 	bzero(&vsi->eth_stats_offsets, sizeof(struct i40e_eth_stats));
 	vsi->stat_offsets_loaded = false;
 }
 
 /**
  * Read and update a 48 bit stat from the hw
  *
  * Since the device stats are not reset at PFReset, they likely will not
  * be zeroed when the driver starts.  We'll save the first values read
  * and use them as offsets to be subtracted from the raw values in order
  * to report stats that count from zero.
  **/
 void
 ixl_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
 	bool offset_loaded, u64 *offset, u64 *stat)
 {
 	u64 new_data;
 
 #if defined(__FreeBSD__) && (__FreeBSD_version >= 1000000) && defined(__amd64__)
 	new_data = rd64(hw, loreg);
 #else
 	/*
 	 * Use two rd32's instead of one rd64; FreeBSD versions before
 	 * 10 don't support 64-bit bus reads/writes.
 	 */
 	new_data = rd32(hw, loreg);
 	new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
 #endif
 
 	if (!offset_loaded)
 		*offset = new_data;
 	if (new_data >= *offset)
 		*stat = new_data - *offset;
 	else
 		*stat = (new_data + ((u64)1 << 48)) - *offset;
 	*stat &= 0xFFFFFFFFFFFFULL;
 }
 
 /**
  * Read and update a 32 bit stat from the hw
  **/
 void
 ixl_stat_update32(struct i40e_hw *hw, u32 reg,
 	bool offset_loaded, u64 *offset, u64 *stat)
 {
 	u32 new_data;
 
 	new_data = rd32(hw, reg);
 	if (!offset_loaded)
 		*offset = new_data;
 	if (new_data >= *offset)
 		*stat = (u32)(new_data - *offset);
 	else
 		*stat = (u32)((new_data + ((u64)1 << 32)) - *offset);
 }
 
 void
 ixl_add_device_sysctls(struct ixl_pf *pf)
 {
 	device_t dev = pf->dev;
 	struct i40e_hw *hw = &pf->hw;
 
 	struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
 	struct sysctl_oid_list *ctx_list =
 	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev));
 
 	struct sysctl_oid *debug_node;
 	struct sysctl_oid_list *debug_list;
 
 	struct sysctl_oid *fec_node;
 	struct sysctl_oid_list *fec_list;
 
 	/* Set up sysctls */
 	SYSCTL_ADD_PROC(ctx, ctx_list,
 	    OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
 	    pf, 0, ixl_sysctl_set_flowcntl, "I", IXL_SYSCTL_HELP_FC);
 
 	SYSCTL_ADD_PROC(ctx, ctx_list,
 	    OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
 	    pf, 0, ixl_sysctl_set_advertise, "I", IXL_SYSCTL_HELP_SET_ADVERTISE);
 
 	SYSCTL_ADD_PROC(ctx, ctx_list,
 	    OID_AUTO, "supported_speeds", CTLTYPE_INT | CTLFLAG_RD,
 	    pf, 0, ixl_sysctl_supported_speeds, "I", IXL_SYSCTL_HELP_SUPPORTED_SPEED);
 
 	SYSCTL_ADD_PROC(ctx, ctx_list,
 	    OID_AUTO, "current_speed", CTLTYPE_STRING | CTLFLAG_RD,
 	    pf, 0, ixl_sysctl_current_speed, "A", "Current Port Speed");
 
 	SYSCTL_ADD_PROC(ctx, ctx_list,
 	    OID_AUTO, "fw_version", CTLTYPE_STRING | CTLFLAG_RD,
 	    pf, 0, ixl_sysctl_show_fw, "A", "Firmware version");
 
 	SYSCTL_ADD_PROC(ctx, ctx_list,
 	    OID_AUTO, "unallocated_queues", CTLTYPE_INT | CTLFLAG_RD,
 	    pf, 0, ixl_sysctl_unallocated_queues, "I",
 	    "Queues not allocated to a PF or VF");
 
 	SYSCTL_ADD_PROC(ctx, ctx_list,
 	    OID_AUTO, "tx_itr", CTLTYPE_INT | CTLFLAG_RW,
 	    pf, 0, ixl_sysctl_pf_tx_itr, "I",
 	    "Immediately set TX ITR value for all queues");
 
 	SYSCTL_ADD_PROC(ctx, ctx_list,
 	    OID_AUTO, "rx_itr", CTLTYPE_INT | CTLFLAG_RW,
 	    pf, 0, ixl_sysctl_pf_rx_itr, "I",
 	    "Immediately set RX ITR value for all queues");
 
 	SYSCTL_ADD_INT(ctx, ctx_list,
 	    OID_AUTO, "dynamic_rx_itr", CTLFLAG_RW,
 	    &pf->dynamic_rx_itr, 0, "Enable dynamic RX ITR");
 
 	SYSCTL_ADD_INT(ctx, ctx_list,
 	    OID_AUTO, "dynamic_tx_itr", CTLFLAG_RW,
 	    &pf->dynamic_tx_itr, 0, "Enable dynamic TX ITR");
 
 	/* Add FEC sysctls for 25G adapters */
 	if (i40e_is_25G_device(hw->device_id)) {
 		fec_node = SYSCTL_ADD_NODE(ctx, ctx_list,
 		    OID_AUTO, "fec", CTLFLAG_RD, NULL, "FEC Sysctls");
 		fec_list = SYSCTL_CHILDREN(fec_node);
 
 		SYSCTL_ADD_PROC(ctx, fec_list,
 		    OID_AUTO, "fc_ability", CTLTYPE_INT | CTLFLAG_RW,
 		    pf, 0, ixl_sysctl_fec_fc_ability, "I", "FC FEC ability enabled");
 
 		SYSCTL_ADD_PROC(ctx, fec_list,
 		    OID_AUTO, "rs_ability", CTLTYPE_INT | CTLFLAG_RW,
 		    pf, 0, ixl_sysctl_fec_rs_ability, "I", "RS FEC ability enabled");
 
 		SYSCTL_ADD_PROC(ctx, fec_list,
 		    OID_AUTO, "fc_requested", CTLTYPE_INT | CTLFLAG_RW,
 		    pf, 0, ixl_sysctl_fec_fc_request, "I", "FC FEC mode requested on link");
 
 		SYSCTL_ADD_PROC(ctx, fec_list,
 		    OID_AUTO, "rs_requested", CTLTYPE_INT | CTLFLAG_RW,
 		    pf, 0, ixl_sysctl_fec_rs_request, "I", "RS FEC mode requested on link");
 
 		SYSCTL_ADD_PROC(ctx, fec_list,
 		    OID_AUTO, "auto_fec_enabled", CTLTYPE_INT | CTLFLAG_RW,
 		    pf, 0, ixl_sysctl_fec_auto_enable, "I", "Let FW decide FEC ability/request modes");
 	}
 
 	SYSCTL_ADD_PROC(ctx, ctx_list,
 	    OID_AUTO, "fw_lldp", CTLTYPE_INT | CTLFLAG_RW,
 	    pf, 0, ixl_sysctl_fw_lldp, "I", IXL_SYSCTL_HELP_FW_LLDP);
 
 	/* Add sysctls meant to print debug information, but don't list them
 	 * in "sysctl -a" output. */
 	debug_node = SYSCTL_ADD_NODE(ctx, ctx_list,
 	    OID_AUTO, "debug", CTLFLAG_RD | CTLFLAG_SKIP, NULL, "Debug Sysctls");
 	debug_list = SYSCTL_CHILDREN(debug_node);
 
 	SYSCTL_ADD_UINT(ctx, debug_list,
 	    OID_AUTO, "shared_debug_mask", CTLFLAG_RW,
 	    &pf->hw.debug_mask, 0, "Shared code debug message level");
 
 	SYSCTL_ADD_UINT(ctx, debug_list,
 	    OID_AUTO, "core_debug_mask", CTLFLAG_RW,
 	    &pf->dbg_mask, 0, "Non-shared code debug message level");
 
 	SYSCTL_ADD_PROC(ctx, debug_list,
 	    OID_AUTO, "link_status", CTLTYPE_STRING | CTLFLAG_RD,
 	    pf, 0, ixl_sysctl_link_status, "A", IXL_SYSCTL_HELP_LINK_STATUS);
 
 	SYSCTL_ADD_PROC(ctx, debug_list,
 	    OID_AUTO, "phy_abilities", CTLTYPE_STRING | CTLFLAG_RD,
 	    pf, 0, ixl_sysctl_phy_abilities, "A", "PHY Abilities");
 
 	SYSCTL_ADD_PROC(ctx, debug_list,
 	    OID_AUTO, "filter_list", CTLTYPE_STRING | CTLFLAG_RD,
 	    pf, 0, ixl_sysctl_sw_filter_list, "A", "SW Filter List");
 
 	SYSCTL_ADD_PROC(ctx, debug_list,
 	    OID_AUTO, "hw_res_alloc", CTLTYPE_STRING | CTLFLAG_RD,
 	    pf, 0, ixl_sysctl_hw_res_alloc, "A", "HW Resource Allocation");
 
 	SYSCTL_ADD_PROC(ctx, debug_list,
 	    OID_AUTO, "switch_config", CTLTYPE_STRING | CTLFLAG_RD,
 	    pf, 0, ixl_sysctl_switch_config, "A", "HW Switch Configuration");
 
 	SYSCTL_ADD_PROC(ctx, debug_list,
 	    OID_AUTO, "rss_key", CTLTYPE_STRING | CTLFLAG_RD,
 	    pf, 0, ixl_sysctl_hkey, "A", "View RSS key");
 
 	SYSCTL_ADD_PROC(ctx, debug_list,
 	    OID_AUTO, "rss_lut", CTLTYPE_STRING | CTLFLAG_RD,
 	    pf, 0, ixl_sysctl_hlut, "A", "View RSS lookup table");
 
 	SYSCTL_ADD_PROC(ctx, debug_list,
 	    OID_AUTO, "rss_hena", CTLTYPE_ULONG | CTLFLAG_RD,
 	    pf, 0, ixl_sysctl_hena, "LU", "View enabled packet types for RSS");
 
 	SYSCTL_ADD_PROC(ctx, debug_list,
 	    OID_AUTO, "disable_fw_link_management", CTLTYPE_INT | CTLFLAG_WR,
 	    pf, 0, ixl_sysctl_fw_link_management, "I", "Disable FW Link Management");
 
 	SYSCTL_ADD_PROC(ctx, debug_list,
 	    OID_AUTO, "dump_debug_data", CTLTYPE_STRING | CTLFLAG_RD,
 	    pf, 0, ixl_sysctl_dump_debug_data, "A", "Dump Debug Data from FW");
 
 	SYSCTL_ADD_PROC(ctx, debug_list,
 	    OID_AUTO, "do_pf_reset", CTLTYPE_INT | CTLFLAG_WR,
 	    pf, 0, ixl_sysctl_do_pf_reset, "I", "Tell HW to initiate a PF reset");
 
 	SYSCTL_ADD_PROC(ctx, debug_list,
 	    OID_AUTO, "do_core_reset", CTLTYPE_INT | CTLFLAG_WR,
 	    pf, 0, ixl_sysctl_do_core_reset, "I", "Tell HW to initiate a CORE reset");
 
 	SYSCTL_ADD_PROC(ctx, debug_list,
 	    OID_AUTO, "do_global_reset", CTLTYPE_INT | CTLFLAG_WR,
 	    pf, 0, ixl_sysctl_do_global_reset, "I", "Tell HW to initiate a GLOBAL reset");
 
 	SYSCTL_ADD_PROC(ctx, debug_list,
 	    OID_AUTO, "do_emp_reset", CTLTYPE_INT | CTLFLAG_WR,
 	    pf, 0, ixl_sysctl_do_emp_reset, "I",
 	    "(This doesn't work) Tell HW to initiate a EMP (entire firmware) reset");
 
 	SYSCTL_ADD_PROC(ctx, debug_list,
 	    OID_AUTO, "queue_interrupt_table", CTLTYPE_STRING | CTLFLAG_RD,
 	    pf, 0, ixl_sysctl_queue_interrupt_table, "A", "View MSI-X indices for TX/RX queues");
 
 	if (pf->has_i2c) {
 		SYSCTL_ADD_PROC(ctx, debug_list,
 		    OID_AUTO, "read_i2c_byte", CTLTYPE_INT | CTLFLAG_RW,
 		    pf, 0, ixl_sysctl_read_i2c_byte, "I", IXL_SYSCTL_HELP_READ_I2C);
 
 		SYSCTL_ADD_PROC(ctx, debug_list,
 		    OID_AUTO, "write_i2c_byte", CTLTYPE_INT | CTLFLAG_RW,
 		    pf, 0, ixl_sysctl_write_i2c_byte, "I", IXL_SYSCTL_HELP_WRITE_I2C);
 
 		SYSCTL_ADD_PROC(ctx, debug_list,
 		    OID_AUTO, "read_i2c_diag_data", CTLTYPE_STRING | CTLFLAG_RD,
 		    pf, 0, ixl_sysctl_read_i2c_diag_data, "A", "Dump selected diagnostic data from FW");
 	}
 }
 
 /*
  * Primarily for finding out how many queues can be assigned to VFs,
  * at runtime.
  */
 static int
 ixl_sysctl_unallocated_queues(SYSCTL_HANDLER_ARGS)
 {
 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
 	int queues;
 
 	queues = (int)ixl_pf_qmgr_get_num_free(&pf->qmgr);
 
 	return sysctl_handle_int(oidp, NULL, queues, req);
 }
 
 /*
 ** Set flow control using sysctl:
 ** 	0 - off
 **	1 - rx pause
 **	2 - tx pause
 **	3 - full
 */
 int
 ixl_sysctl_set_flowcntl(SYSCTL_HANDLER_ARGS)
 {
 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
 	struct i40e_hw *hw = &pf->hw;
 	device_t dev = pf->dev;
 	int requested_fc, error = 0;
 	enum i40e_status_code aq_error = 0;
 	u8 fc_aq_err = 0;
 
 	/* Get request */
 	requested_fc = pf->fc;
 	error = sysctl_handle_int(oidp, &requested_fc, 0, req);
 	if ((error) || (req->newptr == NULL))
 		return (error);
 	if (requested_fc < 0 || requested_fc > 3) {
 		device_printf(dev,
 		    "Invalid fc mode; valid modes are 0 through 3\n");
 		return (EINVAL);
 	}
 
 	/* Set fc ability for port */
 	hw->fc.requested_mode = requested_fc;
 	aq_error = i40e_set_fc(hw, &fc_aq_err, TRUE);
 	if (aq_error) {
 		device_printf(dev,
 		    "%s: Error setting new fc mode %d; fc_err %#x\n",
 		    __func__, aq_error, fc_aq_err);
 		return (EIO);
 	}
 	pf->fc = requested_fc;
 
 	return (0);
 }
 
 char *
 ixl_aq_speed_to_str(enum i40e_aq_link_speed link_speed)
 {
 	int index;
 
 	char *speeds[] = {
 		"Unknown",
 		"100 Mbps",
 		"1 Gbps",
 		"10 Gbps",
 		"40 Gbps",
 		"20 Gbps",
 		"25 Gbps",
 	};
 
 	switch (link_speed) {
 	case I40E_LINK_SPEED_100MB:
 		index = 1;
 		break;
 	case I40E_LINK_SPEED_1GB:
 		index = 2;
 		break;
 	case I40E_LINK_SPEED_10GB:
 		index = 3;
 		break;
 	case I40E_LINK_SPEED_40GB:
 		index = 4;
 		break;
 	case I40E_LINK_SPEED_20GB:
 		index = 5;
 		break;
 	case I40E_LINK_SPEED_25GB:
 		index = 6;
 		break;
 	case I40E_LINK_SPEED_UNKNOWN:
 	default:
 		index = 0;
 		break;
 	}
 
 	return speeds[index];
 }
 
 int
 ixl_sysctl_current_speed(SYSCTL_HANDLER_ARGS)
 {
 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
 	struct i40e_hw *hw = &pf->hw;
 	int error = 0;
 
 	ixl_update_link_status(pf);
 
 	error = sysctl_handle_string(oidp,
 	    ixl_aq_speed_to_str(hw->phy.link_info.link_speed),
 	    8, req);
 	return (error);
 }
 
 /*
  * Converts 8-bit speeds value to and from sysctl flags and
  * Admin Queue flags.
  */
 static u8
 ixl_convert_sysctl_aq_link_speed(u8 speeds, bool to_aq)
 {
 	static u16 speedmap[6] = {
 		(I40E_LINK_SPEED_100MB | (0x1 << 8)),
 		(I40E_LINK_SPEED_1GB   | (0x2 << 8)),
 		(I40E_LINK_SPEED_10GB  | (0x4 << 8)),
 		(I40E_LINK_SPEED_20GB  | (0x8 << 8)),
 		(I40E_LINK_SPEED_25GB  | (0x10 << 8)),
 		(I40E_LINK_SPEED_40GB  | (0x20 << 8))
 	};
 	u8 retval = 0;
 
 	for (int i = 0; i < 6; i++) {
 		if (to_aq)
 			retval |= (speeds & (speedmap[i] >> 8)) ? (speedmap[i] & 0xff) : 0;
 		else
 			retval |= (speeds & speedmap[i]) ? (speedmap[i] >> 8) : 0;
 	}
 
 	return (retval);
 }
 
 int
 ixl_set_advertised_speeds(struct ixl_pf *pf, int speeds, bool from_aq)
 {
 	struct i40e_hw *hw = &pf->hw;
 	device_t dev = pf->dev;
 	struct i40e_aq_get_phy_abilities_resp abilities;
 	struct i40e_aq_set_phy_config config;
 	enum i40e_status_code aq_error = 0;
 
 	/* Get current capability information */
 	aq_error = i40e_aq_get_phy_capabilities(hw,
 	    FALSE, FALSE, &abilities, NULL);
 	if (aq_error) {
 		device_printf(dev,
 		    "%s: Error getting phy capabilities %d,"
 		    " aq error: %d\n", __func__, aq_error,
 		    hw->aq.asq_last_status);
 		return (EIO);
 	}
 
 	/* Prepare new config */
 	bzero(&config, sizeof(config));
 	if (from_aq)
 		config.link_speed = speeds;
 	else
 		config.link_speed = ixl_convert_sysctl_aq_link_speed(speeds, true);
 	config.phy_type = abilities.phy_type;
 	config.phy_type_ext = abilities.phy_type_ext;
 	config.abilities = abilities.abilities
 	    | I40E_AQ_PHY_ENABLE_ATOMIC_LINK;
 	config.eee_capability = abilities.eee_capability;
 	config.eeer = abilities.eeer_val;
 	config.low_power_ctrl = abilities.d3_lpan;
 	config.fec_config = (abilities.fec_cfg_curr_mod_ext_info & 0x1e);
 
 	/* Do aq command & restart link */
 	aq_error = i40e_aq_set_phy_config(hw, &config, NULL);
 	if (aq_error) {
 		device_printf(dev,
 		    "%s: Error setting new phy config %d,"
 		    " aq error: %d\n", __func__, aq_error,
 		    hw->aq.asq_last_status);
 		return (EIO);
 	}
 
 	return (0);
 }
 
 /*
 ** Supported link speedsL
 **	Flags:
 **	 0x1 - 100 Mb
 **	 0x2 - 1G
 **	 0x4 - 10G
 **	 0x8 - 20G
 **	0x10 - 25G
 **	0x20 - 40G
 */
 static int
 ixl_sysctl_supported_speeds(SYSCTL_HANDLER_ARGS)
 {
 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
 	int supported = ixl_convert_sysctl_aq_link_speed(pf->supported_speeds, false);
 
 	return sysctl_handle_int(oidp, NULL, supported, req);
 }
 
 /*
 ** Control link advertise speed:
 **	Flags:
 **	 0x1 - advertise 100 Mb
 **	 0x2 - advertise 1G
 **	 0x4 - advertise 10G
 **	 0x8 - advertise 20G
 **	0x10 - advertise 25G
 **	0x20 - advertise 40G
 **
 **	Set to 0 to disable link
 */
 int
 ixl_sysctl_set_advertise(SYSCTL_HANDLER_ARGS)
 {
 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
 	device_t dev = pf->dev;
 	u8 converted_speeds;
 	int requested_ls = 0;
 	int error = 0;
 
 	/* Read in new mode */
 	requested_ls = pf->advertised_speed;
 	error = sysctl_handle_int(oidp, &requested_ls, 0, req);
 	if ((error) || (req->newptr == NULL))
 		return (error);
 
 	/* Error out if bits outside of possible flag range are set */
 	if ((requested_ls & ~((u8)0x3F)) != 0) {
 		device_printf(dev, "Input advertised speed out of range; "
 		    "valid flags are: 0x%02x\n",
 		    ixl_convert_sysctl_aq_link_speed(pf->supported_speeds, false));
 		return (EINVAL);
 	}
 
 	/* Check if adapter supports input value */
 	converted_speeds = ixl_convert_sysctl_aq_link_speed((u8)requested_ls, true);
 	if ((converted_speeds | pf->supported_speeds) != pf->supported_speeds) {
 		device_printf(dev, "Invalid advertised speed; "
 		    "valid flags are: 0x%02x\n",
 		    ixl_convert_sysctl_aq_link_speed(pf->supported_speeds, false));
 		return (EINVAL);
 	}
 
 	error = ixl_set_advertised_speeds(pf, requested_ls, false);
 	if (error)
 		return (error);
 
 	pf->advertised_speed = requested_ls;
 	ixl_update_link_status(pf);
 	return (0);
 }
 
 /*
 ** Get the width and transaction speed of
 ** the bus this adapter is plugged into.
 */
 void
 ixl_get_bus_info(struct ixl_pf *pf)
 {
 	struct i40e_hw *hw = &pf->hw;
 	device_t dev = pf->dev;
         u16 link;
         u32 offset, num_ports;
 	u64 max_speed;
 
 	/* Some devices don't use PCIE */
 	if (hw->mac.type == I40E_MAC_X722)
 		return;
 
         /* Read PCI Express Capabilities Link Status Register */
         pci_find_cap(dev, PCIY_EXPRESS, &offset);
         link = pci_read_config(dev, offset + PCIER_LINK_STA, 2);
 
 	/* Fill out hw struct with PCIE info */
 	i40e_set_pci_config_data(hw, link);
 
 	/* Use info to print out bandwidth messages */
         device_printf(dev,"PCI Express Bus: Speed %s %s\n",
             ((hw->bus.speed == i40e_bus_speed_8000) ? "8.0GT/s":
             (hw->bus.speed == i40e_bus_speed_5000) ? "5.0GT/s":
             (hw->bus.speed == i40e_bus_speed_2500) ? "2.5GT/s":"Unknown"),
             (hw->bus.width == i40e_bus_width_pcie_x8) ? "Width x8" :
             (hw->bus.width == i40e_bus_width_pcie_x4) ? "Width x4" :
             (hw->bus.width == i40e_bus_width_pcie_x2) ? "Width x2" :
             (hw->bus.width == i40e_bus_width_pcie_x1) ? "Width x1" :
             ("Unknown"));
 
 	/*
 	 * If adapter is in slot with maximum supported speed,
 	 * no warning message needs to be printed out.
 	 */
 	if (hw->bus.speed >= i40e_bus_speed_8000
 	    && hw->bus.width >= i40e_bus_width_pcie_x8)
 		return;
 
 	num_ports = bitcount32(hw->func_caps.valid_functions);
 	max_speed = ixl_max_aq_speed_to_value(pf->supported_speeds) / 1000000;
 
 	if ((num_ports * max_speed) > hw->bus.speed * hw->bus.width) {
                 device_printf(dev, "PCI-Express bandwidth available"
                     " for this device may be insufficient for"
                     " optimal performance.\n");
                 device_printf(dev, "Please move the device to a different"
 		    " PCI-e link with more lanes and/or higher"
 		    " transfer rate.\n");
         }
 }
 
 static int
 ixl_sysctl_show_fw(SYSCTL_HANDLER_ARGS)
 {
 	struct ixl_pf	*pf = (struct ixl_pf *)arg1;
 	struct i40e_hw	*hw = &pf->hw;
 	struct sbuf	*sbuf;
 
 	sbuf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
 	ixl_nvm_version_str(hw, sbuf);
 	sbuf_finish(sbuf);
 	sbuf_delete(sbuf);
 
 	return (0);
 }
 
 void
 ixl_print_nvm_cmd(device_t dev, struct i40e_nvm_access *nvma)
 {
 	if ((nvma->command == I40E_NVM_READ) &&
 	    ((nvma->config & 0xFF) == 0xF) &&
 	    (((nvma->config & 0xF00) >> 8) == 0xF) &&
 	    (nvma->offset == 0) &&
 	    (nvma->data_size == 1)) {
 		// device_printf(dev, "- Get Driver Status Command\n");
 	}
 	else if (nvma->command == I40E_NVM_READ) {
 
 	}
 	else {
 		switch (nvma->command) {
 		case 0xB:
 			device_printf(dev, "- command: I40E_NVM_READ\n");
 			break;
 		case 0xC:
 			device_printf(dev, "- command: I40E_NVM_WRITE\n");
 			break;
 		default:
 			device_printf(dev, "- command: unknown 0x%08x\n", nvma->command);
 			break;
 		}
 
 		device_printf(dev, "- config (ptr)  : 0x%02x\n", nvma->config & 0xFF);
 		device_printf(dev, "- config (flags): 0x%01x\n", (nvma->config & 0xF00) >> 8);
 		device_printf(dev, "- offset : 0x%08x\n", nvma->offset);
 		device_printf(dev, "- data_s : 0x%08x\n", nvma->data_size);
 	}
 }
 
 int
 ixl_handle_nvmupd_cmd(struct ixl_pf *pf, struct ifdrv *ifd)
 {
 	struct i40e_hw *hw = &pf->hw;
 	struct i40e_nvm_access *nvma;
 	device_t dev = pf->dev;
 	enum i40e_status_code status = 0;
 	size_t nvma_size, ifd_len, exp_len;
 	int err, perrno;
 
 	DEBUGFUNC("ixl_handle_nvmupd_cmd");
 
 	/* Sanity checks */
 	nvma_size = sizeof(struct i40e_nvm_access);
 	ifd_len = ifd->ifd_len;
 
 	if (ifd_len < nvma_size ||
 	    ifd->ifd_data == NULL) {
 		device_printf(dev, "%s: incorrect ifdrv length or data pointer\n",
 		    __func__);
 		device_printf(dev, "%s: ifdrv length: %zu, sizeof(struct i40e_nvm_access): %zu\n",
 		    __func__, ifd_len, nvma_size);
 		device_printf(dev, "%s: data pointer: %p\n", __func__,
 		    ifd->ifd_data);
 		return (EINVAL);
 	}
 
 	nvma = malloc(ifd_len, M_DEVBUF, M_WAITOK);
 	err = copyin(ifd->ifd_data, nvma, ifd_len);
 	if (err) {
 		device_printf(dev, "%s: Cannot get request from user space\n",
 		    __func__);
 		free(nvma, M_DEVBUF);
 		return (err);
 	}
 
 	if (pf->dbg_mask & IXL_DBG_NVMUPD)
 		ixl_print_nvm_cmd(dev, nvma);
 
 	if (pf->state & IXL_PF_STATE_ADAPTER_RESETTING) {
 		int count = 0;
 		while (count++ < 100) {
 			i40e_msec_delay(100);
 			if (!(pf->state & IXL_PF_STATE_ADAPTER_RESETTING))
 				break;
 		}
 	}
 
 	if (pf->state & IXL_PF_STATE_ADAPTER_RESETTING) {
 		free(nvma, M_DEVBUF);
 		return (-EBUSY);
 	}
 
 	if (nvma->data_size < 1 || nvma->data_size > 4096) {
 		device_printf(dev, "%s: invalid request, data size not in supported range\n",
 		    __func__);
 		free(nvma, M_DEVBUF);
 		return (EINVAL);
 	}
 
 	/*
 	 * Older versions of the NVM update tool don't set ifd_len to the size
 	 * of the entire buffer passed to the ioctl. Check the data_size field
 	 * in the contained i40e_nvm_access struct and ensure everything is
 	 * copied in from userspace.
 	 */
 	exp_len = nvma_size + nvma->data_size - 1; /* One byte is kept in struct */
 
 	if (ifd_len < exp_len) {
 		ifd_len = exp_len;
 		nvma = realloc(nvma, ifd_len, M_DEVBUF, M_WAITOK);
 		err = copyin(ifd->ifd_data, nvma, ifd_len);
 		if (err) {
 			device_printf(dev, "%s: Cannot get request from user space\n",
 					__func__);
 			free(nvma, M_DEVBUF);
 			return (err);
 		}
 	}
 
 	// TODO: Might need a different lock here
 	// IXL_PF_LOCK(pf);
 	status = i40e_nvmupd_command(hw, nvma, nvma->data, &perrno);
 	// IXL_PF_UNLOCK(pf);
 
 	err = copyout(nvma, ifd->ifd_data, ifd_len);
 	free(nvma, M_DEVBUF);
 	if (err) {
 		device_printf(dev, "%s: Cannot return data to user space\n",
 				__func__);
 		return (err);
 	}
 
 	/* Let the nvmupdate report errors, show them only when debug is enabled */
 	if (status != 0 && (pf->dbg_mask & IXL_DBG_NVMUPD) != 0)
 		device_printf(dev, "i40e_nvmupd_command status %s, perrno %d\n",
 		    i40e_stat_str(hw, status), perrno);
 
 	/*
 	 * -EPERM is actually ERESTART, which the kernel interprets as it needing
 	 * to run this ioctl again. So use -EACCES for -EPERM instead.
 	 */
 	if (perrno == -EPERM)
 		return (-EACCES);
 	else
 		return (perrno);
 }
 
 int
 ixl_find_i2c_interface(struct ixl_pf *pf)
 {
 	struct i40e_hw *hw = &pf->hw;
 	bool i2c_en, port_matched;
 	u32 reg;
 
 	for (int i = 0; i < 4; i++) {
 		reg = rd32(hw, I40E_GLGEN_MDIO_I2C_SEL(i));
 		i2c_en = (reg & I40E_GLGEN_MDIO_I2C_SEL_MDIO_I2C_SEL_MASK);
 		port_matched = ((reg & I40E_GLGEN_MDIO_I2C_SEL_PHY_PORT_NUM_MASK)
 		    >> I40E_GLGEN_MDIO_I2C_SEL_PHY_PORT_NUM_SHIFT)
 		    & BIT(hw->port);
 		if (i2c_en && port_matched)
 			return (i);
 	}
 
 	return (-1);
 }
 
 static char *
 ixl_phy_type_string(u32 bit_pos, bool ext)
 {
 	static char * phy_types_str[32] = {
 		"SGMII",
 		"1000BASE-KX",
 		"10GBASE-KX4",
 		"10GBASE-KR",
 		"40GBASE-KR4",
 		"XAUI",
 		"XFI",
 		"SFI",
 		"XLAUI",
 		"XLPPI",
 		"40GBASE-CR4",
 		"10GBASE-CR1",
 		"SFP+ Active DA",
 		"QSFP+ Active DA",
 		"Reserved (14)",
 		"Reserved (15)",
 		"Reserved (16)",
 		"100BASE-TX",
 		"1000BASE-T",
 		"10GBASE-T",
 		"10GBASE-SR",
 		"10GBASE-LR",
 		"10GBASE-SFP+Cu",
 		"10GBASE-CR1",
 		"40GBASE-CR4",
 		"40GBASE-SR4",
 		"40GBASE-LR4",
 		"1000BASE-SX",
 		"1000BASE-LX",
 		"1000BASE-T Optical",
 		"20GBASE-KR2",
 		"Reserved (31)"
 	};
 	static char * ext_phy_types_str[8] = {
 		"25GBASE-KR",
 		"25GBASE-CR",
 		"25GBASE-SR",
 		"25GBASE-LR",
 		"25GBASE-AOC",
 		"25GBASE-ACC",
 		"Reserved (6)",
 		"Reserved (7)"
 	};
 
 	if (ext && bit_pos > 7) return "Invalid_Ext";
 	if (bit_pos > 31) return "Invalid";
 
 	return (ext) ? ext_phy_types_str[bit_pos] : phy_types_str[bit_pos];
 }
 
 /* TODO: ERJ: I don't this is necessary anymore. */
 int
 ixl_aq_get_link_status(struct ixl_pf *pf, struct i40e_aqc_get_link_status *link_status)
 {
 	device_t dev = pf->dev;
 	struct i40e_hw *hw = &pf->hw;
 	struct i40e_aq_desc desc;
 	enum i40e_status_code status;
 
 	struct i40e_aqc_get_link_status *aq_link_status =
 		(struct i40e_aqc_get_link_status *)&desc.params.raw;
 
 	i40e_fill_default_direct_cmd_desc(&desc, i40e_aqc_opc_get_link_status);
 	link_status->command_flags = CPU_TO_LE16(I40E_AQ_LSE_ENABLE);
 	status = i40e_asq_send_command(hw, &desc, NULL, 0, NULL);
 	if (status) {
 		device_printf(dev,
 		    "%s: i40e_aqc_opc_get_link_status status %s, aq error %s\n",
 		    __func__, i40e_stat_str(hw, status),
 		    i40e_aq_str(hw, hw->aq.asq_last_status));
 		return (EIO);
 	}
 
 	bcopy(aq_link_status, link_status, sizeof(struct i40e_aqc_get_link_status));
 	return (0);
 }
 
 static char *
 ixl_phy_type_string_ls(u8 val)
 {
 	if (val >= 0x1F)
 		return ixl_phy_type_string(val - 0x1F, true);
 	else
 		return ixl_phy_type_string(val, false);
 }
 
 static int
 ixl_sysctl_link_status(SYSCTL_HANDLER_ARGS)
 {
 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
 	device_t dev = pf->dev;
 	struct sbuf *buf;
 	int error = 0;
 
 	buf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
 	if (!buf) {
 		device_printf(dev, "Could not allocate sbuf for sysctl output.\n");
 		return (ENOMEM);
 	}
 
 	struct i40e_aqc_get_link_status link_status;
 	error = ixl_aq_get_link_status(pf, &link_status);
 	if (error) {
 		sbuf_delete(buf);
 		return (error);
 	}
 
 	sbuf_printf(buf, "\n"
 	    "PHY Type : 0x%02x<%s>\n"
 	    "Speed    : 0x%02x\n"
 	    "Link info: 0x%02x\n"
 	    "AN info  : 0x%02x\n"
 	    "Ext info : 0x%02x\n"
 	    "Loopback : 0x%02x\n"
 	    "Max Frame: %d\n"
 	    "Config   : 0x%02x\n"
 	    "Power    : 0x%02x",
 	    link_status.phy_type,
 	    ixl_phy_type_string_ls(link_status.phy_type),
 	    link_status.link_speed,
 	    link_status.link_info,
 	    link_status.an_info,
 	    link_status.ext_info,
 	    link_status.loopback,
 	    link_status.max_frame_size,
 	    link_status.config,
 	    link_status.power_desc);
 
 	error = sbuf_finish(buf);
 	if (error)
 		device_printf(dev, "Error finishing sbuf: %d\n", error);
 
 	sbuf_delete(buf);
 	return (error);
 }
 
 static int
 ixl_sysctl_phy_abilities(SYSCTL_HANDLER_ARGS)
 {
 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
 	struct i40e_hw *hw = &pf->hw;
 	device_t dev = pf->dev;
 	enum i40e_status_code status;
 	struct i40e_aq_get_phy_abilities_resp abilities;
 	struct sbuf *buf;
 	int error = 0;
 
 	buf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
 	if (!buf) {
 		device_printf(dev, "Could not allocate sbuf for sysctl output.\n");
 		return (ENOMEM);
 	}
 
 	status = i40e_aq_get_phy_capabilities(hw,
 	    FALSE, FALSE, &abilities, NULL);
 	if (status) {
 		device_printf(dev,
 		    "%s: i40e_aq_get_phy_capabilities() status %s, aq error %s\n",
 		    __func__, i40e_stat_str(hw, status),
 		    i40e_aq_str(hw, hw->aq.asq_last_status));
 		sbuf_delete(buf);
 		return (EIO);
 	}
 
 	sbuf_printf(buf, "\n"
 	    "PHY Type : %08x",
 	    abilities.phy_type);
 
 	if (abilities.phy_type != 0) {
 		sbuf_printf(buf, "<");
 		for (int i = 0; i < 32; i++)
 			if ((1 << i) & abilities.phy_type)
 				sbuf_printf(buf, "%s,", ixl_phy_type_string(i, false));
 		sbuf_printf(buf, ">\n");
 	}
 
 	sbuf_printf(buf, "PHY Ext  : %02x",
 	    abilities.phy_type_ext);
 
 	if (abilities.phy_type_ext != 0) {
 		sbuf_printf(buf, "<");
 		for (int i = 0; i < 4; i++)
 			if ((1 << i) & abilities.phy_type_ext)
 				sbuf_printf(buf, "%s,", ixl_phy_type_string(i, true));
 		sbuf_printf(buf, ">");
 	}
 	sbuf_printf(buf, "\n");
 
 	sbuf_printf(buf,
 	    "Speed    : %02x\n"
 	    "Abilities: %02x\n"
 	    "EEE cap  : %04x\n"
 	    "EEER reg : %08x\n"
 	    "D3 Lpan  : %02x\n"
 	    "ID       : %02x %02x %02x %02x\n"
 	    "ModType  : %02x %02x %02x\n"
 	    "ModType E: %01x\n"
 	    "FEC Cfg  : %02x\n"
 	    "Ext CC   : %02x",
 	    abilities.link_speed,
 	    abilities.abilities, abilities.eee_capability,
 	    abilities.eeer_val, abilities.d3_lpan,
 	    abilities.phy_id[0], abilities.phy_id[1],
 	    abilities.phy_id[2], abilities.phy_id[3],
 	    abilities.module_type[0], abilities.module_type[1],
 	    abilities.module_type[2], (abilities.fec_cfg_curr_mod_ext_info & 0xe0) >> 5,
 	    abilities.fec_cfg_curr_mod_ext_info & 0x1F,
 	    abilities.ext_comp_code);
 
 	error = sbuf_finish(buf);
 	if (error)
 		device_printf(dev, "Error finishing sbuf: %d\n", error);
 
 	sbuf_delete(buf);
 	return (error);
 }
 
 static int
 ixl_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS)
 {
 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
 	struct ixl_vsi *vsi = &pf->vsi;
 	struct ixl_mac_filter *f;
 	device_t dev = pf->dev;
 	int error = 0, ftl_len = 0, ftl_counter = 0;
 
 	struct sbuf *buf;
 
 	buf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
 	if (!buf) {
 		device_printf(dev, "Could not allocate sbuf for output.\n");
 		return (ENOMEM);
 	}
 
 	sbuf_printf(buf, "\n");
 
 	/* Print MAC filters */
 	sbuf_printf(buf, "PF Filters:\n");
 	SLIST_FOREACH(f, &vsi->ftl, next)
 		ftl_len++;
 
 	if (ftl_len < 1)
 		sbuf_printf(buf, "(none)\n");
 	else {
 		SLIST_FOREACH(f, &vsi->ftl, next) {
 			sbuf_printf(buf,
 			    MAC_FORMAT ", vlan %4d, flags %#06x",
 			    MAC_FORMAT_ARGS(f->macaddr), f->vlan, f->flags);
 			/* don't print '\n' for last entry */
 			if (++ftl_counter != ftl_len)
 				sbuf_printf(buf, "\n");
 		}
 	}
 
 #ifdef PCI_IOV
 	/* TODO: Give each VF its own filter list sysctl */
 	struct ixl_vf *vf;
 	if (pf->num_vfs > 0) {
 		sbuf_printf(buf, "\n\n");
 		for (int i = 0; i < pf->num_vfs; i++) {
 			vf = &pf->vfs[i];
 			if (!(vf->vf_flags & VF_FLAG_ENABLED))
 				continue;
 
 			vsi = &vf->vsi;
 			ftl_len = 0, ftl_counter = 0;
 			sbuf_printf(buf, "VF-%d Filters:\n", vf->vf_num);
 			SLIST_FOREACH(f, &vsi->ftl, next)
 				ftl_len++;
 
 			if (ftl_len < 1)
 				sbuf_printf(buf, "(none)\n");
 			else {
 				SLIST_FOREACH(f, &vsi->ftl, next) {
 					sbuf_printf(buf,
 					    MAC_FORMAT ", vlan %4d, flags %#06x\n",
 					    MAC_FORMAT_ARGS(f->macaddr), f->vlan, f->flags);
 				}
 			}
 		}
 	}
 #endif
 
 	error = sbuf_finish(buf);
 	if (error)
 		device_printf(dev, "Error finishing sbuf: %d\n", error);
 	sbuf_delete(buf);
 
 	return (error);
 }
 
 #define IXL_SW_RES_SIZE 0x14
 int
 ixl_res_alloc_cmp(const void *a, const void *b)
 {
 	const struct i40e_aqc_switch_resource_alloc_element_resp *one, *two;
 	one = (const struct i40e_aqc_switch_resource_alloc_element_resp *)a;
 	two = (const struct i40e_aqc_switch_resource_alloc_element_resp *)b;
 
 	return ((int)one->resource_type - (int)two->resource_type);
 }
 
 /*
  * Longest string length: 25
  */
 char *
 ixl_switch_res_type_string(u8 type)
 {
 	// TODO: This should be changed to static const
 	char * ixl_switch_res_type_strings[0x14] = {
 		"VEB",
 		"VSI",
 		"Perfect Match MAC address",
 		"S-tag",
 		"(Reserved)",
 		"Multicast hash entry",
 		"Unicast hash entry",
 		"VLAN",
 		"VSI List entry",
 		"(Reserved)",
 		"VLAN Statistic Pool",
 		"Mirror Rule",
 		"Queue Set",
 		"Inner VLAN Forward filter",
 		"(Reserved)",
 		"Inner MAC",
 		"IP",
 		"GRE/VN1 Key",
 		"VN2 Key",
 		"Tunneling Port"
 	};
 
 	if (type < 0x14)
 		return ixl_switch_res_type_strings[type];
 	else
 		return "(Reserved)";
 }
 
 static int
 ixl_sysctl_hw_res_alloc(SYSCTL_HANDLER_ARGS)
 {
 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
 	struct i40e_hw *hw = &pf->hw;
 	device_t dev = pf->dev;
 	struct sbuf *buf;
 	enum i40e_status_code status;
 	int error = 0;
 
 	u8 num_entries;
 	struct i40e_aqc_switch_resource_alloc_element_resp resp[IXL_SW_RES_SIZE];
 
 	buf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
 	if (!buf) {
 		device_printf(dev, "Could not allocate sbuf for output.\n");
 		return (ENOMEM);
 	}
 
 	bzero(resp, sizeof(resp));
 	status = i40e_aq_get_switch_resource_alloc(hw, &num_entries,
 				resp,
 				IXL_SW_RES_SIZE,
 				NULL);
 	if (status) {
 		device_printf(dev,
 		    "%s: get_switch_resource_alloc() error %s, aq error %s\n",
 		    __func__, i40e_stat_str(hw, status),
 		    i40e_aq_str(hw, hw->aq.asq_last_status));
 		sbuf_delete(buf);
 		return (error);
 	}
 
 	/* Sort entries by type for display */
 	qsort(resp, num_entries,
 	    sizeof(struct i40e_aqc_switch_resource_alloc_element_resp),
 	    &ixl_res_alloc_cmp);
 
 	sbuf_cat(buf, "\n");
 	sbuf_printf(buf, "# of entries: %d\n", num_entries);
 	sbuf_printf(buf,
 	    "                     Type | Guaranteed | Total | Used   | Un-allocated\n"
 	    "                          | (this)     | (all) | (this) | (all)       \n");
 	for (int i = 0; i < num_entries; i++) {
 		sbuf_printf(buf,
 		    "%25s | %10d   %5d   %6d   %12d",
 		    ixl_switch_res_type_string(resp[i].resource_type),
 		    resp[i].guaranteed,
 		    resp[i].total,
 		    resp[i].used,
 		    resp[i].total_unalloced);
 		if (i < num_entries - 1)
 			sbuf_cat(buf, "\n");
 	}
 
 	error = sbuf_finish(buf);
 	if (error)
 		device_printf(dev, "Error finishing sbuf: %d\n", error);
 
 	sbuf_delete(buf);
 	return (error);
 }
 
 /*
 ** Caller must init and delete sbuf; this function will clear and
 ** finish it for caller.
 */
 char *
 ixl_switch_element_string(struct sbuf *s,
     struct i40e_aqc_switch_config_element_resp *element)
 {
 	sbuf_clear(s);
 
 	switch (element->element_type) {
 	case I40E_AQ_SW_ELEM_TYPE_MAC:
 		sbuf_printf(s, "MAC %3d", element->element_info);
 		break;
 	case I40E_AQ_SW_ELEM_TYPE_PF:
 		sbuf_printf(s, "PF  %3d", element->element_info);
 		break;
 	case I40E_AQ_SW_ELEM_TYPE_VF:
 		sbuf_printf(s, "VF  %3d", element->element_info);
 		break;
 	case I40E_AQ_SW_ELEM_TYPE_EMP:
 		sbuf_cat(s, "EMP");
 		break;
 	case I40E_AQ_SW_ELEM_TYPE_BMC:
 		sbuf_cat(s, "BMC");
 		break;
 	case I40E_AQ_SW_ELEM_TYPE_PV:
 		sbuf_cat(s, "PV");
 		break;
 	case I40E_AQ_SW_ELEM_TYPE_VEB:
 		sbuf_cat(s, "VEB");
 		break;
 	case I40E_AQ_SW_ELEM_TYPE_PA:
 		sbuf_cat(s, "PA");
 		break;
 	case I40E_AQ_SW_ELEM_TYPE_VSI:
 		sbuf_printf(s, "VSI %3d", element->element_info);
 		break;
 	default:
 		sbuf_cat(s, "?");
 		break;
 	}
 
 	sbuf_finish(s);
 	return sbuf_data(s);
 }
 
 static int
 ixl_sysctl_switch_config(SYSCTL_HANDLER_ARGS)
 {
 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
 	struct i40e_hw *hw = &pf->hw;
 	device_t dev = pf->dev;
 	struct sbuf *buf;
 	struct sbuf *nmbuf;
 	enum i40e_status_code status;
 	int error = 0;
 	u16 next = 0;
 	u8 aq_buf[I40E_AQ_LARGE_BUF];
 
 	struct i40e_aqc_get_switch_config_resp *sw_config;
 	sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
 
 	buf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
 	if (!buf) {
 		device_printf(dev, "Could not allocate sbuf for sysctl output.\n");
 		return (ENOMEM);
 	}
 
 	status = i40e_aq_get_switch_config(hw, sw_config,
 	    sizeof(aq_buf), &next, NULL);
 	if (status) {
 		device_printf(dev,
 		    "%s: aq_get_switch_config() error %s, aq error %s\n",
 		    __func__, i40e_stat_str(hw, status),
 		    i40e_aq_str(hw, hw->aq.asq_last_status));
 		sbuf_delete(buf);
 		return error;
 	}
 	if (next)
 		device_printf(dev, "%s: TODO: get more config with SEID %d\n",
 		    __func__, next);
 
 	nmbuf = sbuf_new_auto();
 	if (!nmbuf) {
 		device_printf(dev, "Could not allocate sbuf for name output.\n");
 		sbuf_delete(buf);
 		return (ENOMEM);
 	}
 
 	sbuf_cat(buf, "\n");
 	/* Assuming <= 255 elements in switch */
 	sbuf_printf(buf, "# of reported elements: %d\n", sw_config->header.num_reported);
 	sbuf_printf(buf, "total # of elements: %d\n", sw_config->header.num_total);
 	/* Exclude:
 	** Revision -- all elements are revision 1 for now
 	*/
 	sbuf_printf(buf,
 	    "SEID (  Name  ) |  Uplink  | Downlink | Conn Type\n"
 	    "                |          |          | (uplink)\n");
 	for (int i = 0; i < sw_config->header.num_reported; i++) {
 		// "%4d (%8s) | %8s   %8s   %#8x",
 		sbuf_printf(buf, "%4d", sw_config->element[i].seid);
 		sbuf_cat(buf, " ");
 		sbuf_printf(buf, "(%8s)", ixl_switch_element_string(nmbuf,
 		    &sw_config->element[i]));
 		sbuf_cat(buf, " | ");
 		sbuf_printf(buf, "%8d", sw_config->element[i].uplink_seid);
 		sbuf_cat(buf, "   ");
 		sbuf_printf(buf, "%8d", sw_config->element[i].downlink_seid);
 		sbuf_cat(buf, "   ");
 		sbuf_printf(buf, "%#8x", sw_config->element[i].connection_type);
 		if (i < sw_config->header.num_reported - 1)
 			sbuf_cat(buf, "\n");
 	}
 	sbuf_delete(nmbuf);
 
 	error = sbuf_finish(buf);
 	if (error)
 		device_printf(dev, "Error finishing sbuf: %d\n", error);
 
 	sbuf_delete(buf);
 
 	return (error);
 }
 
 static int
 ixl_sysctl_hkey(SYSCTL_HANDLER_ARGS)
 {
 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
 	struct i40e_hw *hw = &pf->hw;
 	device_t dev = pf->dev;
 	struct sbuf *buf;
 	int error = 0;
 	enum i40e_status_code status;
 	u32 reg;
 
 	struct i40e_aqc_get_set_rss_key_data key_data;
 
 	buf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
 	if (!buf) {
 		device_printf(dev, "Could not allocate sbuf for output.\n");
 		return (ENOMEM);
 	}
 
 	bzero(key_data.standard_rss_key, sizeof(key_data.standard_rss_key));
 
 	sbuf_cat(buf, "\n");
 	if (hw->mac.type == I40E_MAC_X722) {
 		status = i40e_aq_get_rss_key(hw, pf->vsi.vsi_num, &key_data);
 		if (status)
 			device_printf(dev, "i40e_aq_get_rss_key status %s, error %s\n",
 			    i40e_stat_str(hw, status), i40e_aq_str(hw, hw->aq.asq_last_status));
 	} else {
 		for (int i = 0; i < IXL_RSS_KEY_SIZE_REG; i++) {
 			reg = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i));
 			bcopy(&reg, ((caddr_t)&key_data) + (i << 2), 4);
 		}
 	}
 
 	ixl_sbuf_print_bytes(buf, (u8 *)&key_data, sizeof(key_data), 0, true);
 
 	error = sbuf_finish(buf);
 	if (error)
 		device_printf(dev, "Error finishing sbuf: %d\n", error);
 	sbuf_delete(buf);
 
 	return (error);
 }
 
 static void
 ixl_sbuf_print_bytes(struct sbuf *sb, u8 *buf, int length, int label_offset, bool text)
 {
 	int i, j, k, width;
 	char c;
 
 	if (length < 1 || buf == NULL) return;
 
 	int byte_stride = 16;
 	int lines = length / byte_stride;
 	int rem = length % byte_stride;
 	if (rem > 0)
 		lines++;
 
 	for (i = 0; i < lines; i++) {
 		width = (rem > 0 && i == lines - 1)
 		    ? rem : byte_stride;
 
 		sbuf_printf(sb, "%4d | ", label_offset + i * byte_stride);
 
 		for (j = 0; j < width; j++)
 			sbuf_printf(sb, "%02x ", buf[i * byte_stride + j]);
 
 		if (width < byte_stride) {
 			for (k = 0; k < (byte_stride - width); k++)
 				sbuf_printf(sb, "   ");
 		}
 
 		if (!text) {
 			sbuf_printf(sb, "\n");
 			continue;
 		}
 
 		for (j = 0; j < width; j++) {
 			c = (char)buf[i * byte_stride + j];
 			if (c < 32 || c > 126)
 				sbuf_printf(sb, ".");
 			else
 				sbuf_printf(sb, "%c", c);
 
 			if (j == width - 1)
 				sbuf_printf(sb, "\n");
 		}
 	}
 }
 
 static int
 ixl_sysctl_hlut(SYSCTL_HANDLER_ARGS)
 {
 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
 	struct i40e_hw *hw = &pf->hw;
 	device_t dev = pf->dev;
 	struct sbuf *buf;
 	int error = 0;
 	enum i40e_status_code status;
 	u8 hlut[512];
 	u32 reg;
 
 	buf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
 	if (!buf) {
 		device_printf(dev, "Could not allocate sbuf for output.\n");
 		return (ENOMEM);
 	}
 
 	bzero(hlut, sizeof(hlut));
 	sbuf_cat(buf, "\n");
 	if (hw->mac.type == I40E_MAC_X722) {
 		status = i40e_aq_get_rss_lut(hw, pf->vsi.vsi_num, TRUE, hlut, sizeof(hlut));
 		if (status)
 			device_printf(dev, "i40e_aq_get_rss_lut status %s, error %s\n",
 			    i40e_stat_str(hw, status), i40e_aq_str(hw, hw->aq.asq_last_status));
 	} else {
 		for (int i = 0; i < hw->func_caps.rss_table_size >> 2; i++) {
 			reg = rd32(hw, I40E_PFQF_HLUT(i));
 			bcopy(&reg, &hlut[i << 2], 4);
 		}
 	}
 	ixl_sbuf_print_bytes(buf, hlut, 512, 0, false);
 
 	error = sbuf_finish(buf);
 	if (error)
 		device_printf(dev, "Error finishing sbuf: %d\n", error);
 	sbuf_delete(buf);
 
 	return (error);
 }
 
 static int
 ixl_sysctl_hena(SYSCTL_HANDLER_ARGS)
 {
 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
 	struct i40e_hw *hw = &pf->hw;
 	u64 hena;
 
 	hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) |
 	    ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32);
 
 	return sysctl_handle_long(oidp, NULL, hena, req);
 }
 
 /*
  * Sysctl to disable firmware's link management
  *
  * 1 - Disable link management on this port
  * 0 - Re-enable link management
  *
  * On normal NVMs, firmware manages link by default.
  */
 static int
 ixl_sysctl_fw_link_management(SYSCTL_HANDLER_ARGS)
 {
 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
 	struct i40e_hw *hw = &pf->hw;
 	device_t dev = pf->dev;
 	int requested_mode = -1;
 	enum i40e_status_code status = 0;
 	int error = 0;
 
 	/* Read in new mode */
 	error = sysctl_handle_int(oidp, &requested_mode, 0, req);
 	if ((error) || (req->newptr == NULL))
 		return (error);
 	/* Check for sane value */
 	if (requested_mode < 0 || requested_mode > 1) {
 		device_printf(dev, "Valid modes are 0 or 1\n");
 		return (EINVAL);
 	}
 
 	/* Set new mode */
 	status = i40e_aq_set_phy_debug(hw, !!(requested_mode) << 4, NULL);
 	if (status) {
 		device_printf(dev,
 		    "%s: Error setting new phy debug mode %s,"
 		    " aq error: %s\n", __func__, i40e_stat_str(hw, status),
 		    i40e_aq_str(hw, hw->aq.asq_last_status));
 		return (EIO);
 	}
 
 	return (0);
 }
 
 /*
  * Read some diagnostic data from an SFP module
  * Bytes 96-99, 102-105 from device address 0xA2
  */
 static int
 ixl_sysctl_read_i2c_diag_data(SYSCTL_HANDLER_ARGS)
 {
 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
 	device_t dev = pf->dev;
 	struct sbuf *sbuf;
 	int error = 0;
 	u8 output;
 
 	error = pf->read_i2c_byte(pf, 0, 0xA0, &output);
 	if (error) {
 		device_printf(dev, "Error reading from i2c\n");
 		return (error);
 	}
 	if (output != 0x3) {
 		device_printf(dev, "Module is not SFP/SFP+/SFP28 (%02X)\n", output);
 		return (EIO);
 	}
 
 	pf->read_i2c_byte(pf, 92, 0xA0, &output);
 	if (!(output & 0x60)) {
 		device_printf(dev, "Module doesn't support diagnostics: %02X\n", output);
 		return (EIO);
 	}
 
 	sbuf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
 
 	for (u8 offset = 96; offset < 100; offset++) {
 		pf->read_i2c_byte(pf, offset, 0xA2, &output);
 		sbuf_printf(sbuf, "%02X ", output);
 	}
 	for (u8 offset = 102; offset < 106; offset++) {
 		pf->read_i2c_byte(pf, offset, 0xA2, &output);
 		sbuf_printf(sbuf, "%02X ", output);
 	}
 
 	sbuf_finish(sbuf);
 	sbuf_delete(sbuf);
 
 	return (0);
 }
 
 /*
  * Sysctl to read a byte from I2C bus.
  *
  * Input: 32-bit value:
  * 	bits 0-7:   device address (0xA0 or 0xA2)
  * 	bits 8-15:  offset (0-255)
  *	bits 16-31: unused
  * Output: 8-bit value read
  */
 static int
 ixl_sysctl_read_i2c_byte(SYSCTL_HANDLER_ARGS)
 {
 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
 	device_t dev = pf->dev;
 	int input = -1, error = 0;
 	u8 dev_addr, offset, output;
 
 	/* Read in I2C read parameters */
 	error = sysctl_handle_int(oidp, &input, 0, req);
 	if ((error) || (req->newptr == NULL))
 		return (error);
 	/* Validate device address */
 	dev_addr = input & 0xFF;
 	if (dev_addr != 0xA0 && dev_addr != 0xA2) {
 		return (EINVAL);
 	}
 	offset = (input >> 8) & 0xFF;
 
 	error = pf->read_i2c_byte(pf, offset, dev_addr, &output);
 	if (error)
 		return (error);
 
 	device_printf(dev, "%02X\n", output);
 	return (0);
 }
 
 /*
  * Sysctl to write a byte to the I2C bus.
  *
  * Input: 32-bit value:
  * 	bits 0-7:   device address (0xA0 or 0xA2)
  * 	bits 8-15:  offset (0-255)
  *	bits 16-23: value to write
  *	bits 24-31: unused
  * Output: 8-bit value written
  */
 static int
 ixl_sysctl_write_i2c_byte(SYSCTL_HANDLER_ARGS)
 {
 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
 	device_t dev = pf->dev;
 	int input = -1, error = 0;
 	u8 dev_addr, offset, value;
 
 	/* Read in I2C write parameters */
 	error = sysctl_handle_int(oidp, &input, 0, req);
 	if ((error) || (req->newptr == NULL))
 		return (error);
 	/* Validate device address */
 	dev_addr = input & 0xFF;
 	if (dev_addr != 0xA0 && dev_addr != 0xA2) {
 		return (EINVAL);
 	}
 	offset = (input >> 8) & 0xFF;
 	value = (input >> 16) & 0xFF;
 
 	error = pf->write_i2c_byte(pf, offset, dev_addr, value);
 	if (error)
 		return (error);
 
 	device_printf(dev, "%02X written\n", value);
 	return (0);
 }
 
 static int
 ixl_get_fec_config(struct ixl_pf *pf, struct i40e_aq_get_phy_abilities_resp *abilities,
     u8 bit_pos, int *is_set)
 {
 	device_t dev = pf->dev;
 	struct i40e_hw *hw = &pf->hw;
 	enum i40e_status_code status;
 
 	status = i40e_aq_get_phy_capabilities(hw,
 	    FALSE, FALSE, abilities, NULL);
 	if (status) {
 		device_printf(dev,
 		    "%s: i40e_aq_get_phy_capabilities() status %s, aq error %s\n",
 		    __func__, i40e_stat_str(hw, status),
 		    i40e_aq_str(hw, hw->aq.asq_last_status));
 		return (EIO);
 	}
 
 	*is_set = !!(abilities->fec_cfg_curr_mod_ext_info & bit_pos);
 	return (0);
 }
 
 static int
 ixl_set_fec_config(struct ixl_pf *pf, struct i40e_aq_get_phy_abilities_resp *abilities,
     u8 bit_pos, int set)
 {
 	device_t dev = pf->dev;
 	struct i40e_hw *hw = &pf->hw;
 	struct i40e_aq_set_phy_config config;
 	enum i40e_status_code status;
 
 	/* Set new PHY config */
 	memset(&config, 0, sizeof(config));
 	config.fec_config = abilities->fec_cfg_curr_mod_ext_info & ~(bit_pos);
 	if (set)
 		config.fec_config |= bit_pos;
 	if (config.fec_config != abilities->fec_cfg_curr_mod_ext_info) {
 		config.abilities |= I40E_AQ_PHY_ENABLE_ATOMIC_LINK;
 		config.phy_type = abilities->phy_type;
 		config.phy_type_ext = abilities->phy_type_ext;
 		config.link_speed = abilities->link_speed;
 		config.eee_capability = abilities->eee_capability;
 		config.eeer = abilities->eeer_val;
 		config.low_power_ctrl = abilities->d3_lpan;
 		status = i40e_aq_set_phy_config(hw, &config, NULL);
 
 		if (status) {
 			device_printf(dev,
 			    "%s: i40e_aq_set_phy_config() status %s, aq error %s\n",
 			    __func__, i40e_stat_str(hw, status),
 			    i40e_aq_str(hw, hw->aq.asq_last_status));
 			return (EIO);
 		}
 	}
 
 	return (0);
 }
 
 static int
 ixl_sysctl_fec_fc_ability(SYSCTL_HANDLER_ARGS)
 {
 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
 	int mode, error = 0;
 
 	struct i40e_aq_get_phy_abilities_resp abilities;
 	error = ixl_get_fec_config(pf, &abilities, I40E_AQ_ENABLE_FEC_KR, &mode);
 	if (error)
 		return (error);
 	/* Read in new mode */
 	error = sysctl_handle_int(oidp, &mode, 0, req);
 	if ((error) || (req->newptr == NULL))
 		return (error);
 
 	return ixl_set_fec_config(pf, &abilities, I40E_AQ_SET_FEC_ABILITY_KR, !!(mode));
 }
 
 static int
 ixl_sysctl_fec_rs_ability(SYSCTL_HANDLER_ARGS)
 {
 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
 	int mode, error = 0;
 
 	struct i40e_aq_get_phy_abilities_resp abilities;
 	error = ixl_get_fec_config(pf, &abilities, I40E_AQ_ENABLE_FEC_RS, &mode);
 	if (error)
 		return (error);
 	/* Read in new mode */
 	error = sysctl_handle_int(oidp, &mode, 0, req);
 	if ((error) || (req->newptr == NULL))
 		return (error);
 
 	return ixl_set_fec_config(pf, &abilities, I40E_AQ_SET_FEC_ABILITY_RS, !!(mode));
 }
 
 static int
 ixl_sysctl_fec_fc_request(SYSCTL_HANDLER_ARGS)
 {
 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
 	int mode, error = 0;
 
 	struct i40e_aq_get_phy_abilities_resp abilities;
 	error = ixl_get_fec_config(pf, &abilities, I40E_AQ_REQUEST_FEC_KR, &mode);
 	if (error)
 		return (error);
 	/* Read in new mode */
 	error = sysctl_handle_int(oidp, &mode, 0, req);
 	if ((error) || (req->newptr == NULL))
 		return (error);
 
 	return ixl_set_fec_config(pf, &abilities, I40E_AQ_SET_FEC_REQUEST_KR, !!(mode));
 }
 
 static int
 ixl_sysctl_fec_rs_request(SYSCTL_HANDLER_ARGS)
 {
 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
 	int mode, error = 0;
 
 	struct i40e_aq_get_phy_abilities_resp abilities;
 	error = ixl_get_fec_config(pf, &abilities, I40E_AQ_REQUEST_FEC_RS, &mode);
 	if (error)
 		return (error);
 	/* Read in new mode */
 	error = sysctl_handle_int(oidp, &mode, 0, req);
 	if ((error) || (req->newptr == NULL))
 		return (error);
 
 	return ixl_set_fec_config(pf, &abilities, I40E_AQ_SET_FEC_REQUEST_RS, !!(mode));
 }
 
 static int
 ixl_sysctl_fec_auto_enable(SYSCTL_HANDLER_ARGS)
 {
 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
 	int mode, error = 0;
 
 	struct i40e_aq_get_phy_abilities_resp abilities;
 	error = ixl_get_fec_config(pf, &abilities, I40E_AQ_ENABLE_FEC_AUTO, &mode);
 	if (error)
 		return (error);
 	/* Read in new mode */
 	error = sysctl_handle_int(oidp, &mode, 0, req);
 	if ((error) || (req->newptr == NULL))
 		return (error);
 
 	return ixl_set_fec_config(pf, &abilities, I40E_AQ_SET_FEC_AUTO, !!(mode));
 }
 
 static int
 ixl_sysctl_dump_debug_data(SYSCTL_HANDLER_ARGS)
 {
 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
 	struct i40e_hw *hw = &pf->hw;
 	device_t dev = pf->dev;
 	struct sbuf *buf;
 	int error = 0;
 	enum i40e_status_code status;
 
 	buf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
 	if (!buf) {
 		device_printf(dev, "Could not allocate sbuf for output.\n");
 		return (ENOMEM);
 	}
 
 	u8 *final_buff;
 	/* This amount is only necessary if reading the entire cluster into memory */
 #define IXL_FINAL_BUFF_SIZE	(1280 * 1024)
 	final_buff = malloc(IXL_FINAL_BUFF_SIZE, M_DEVBUF, M_WAITOK);
 	if (final_buff == NULL) {
 		device_printf(dev, "Could not allocate memory for output.\n");
 		goto out;
 	}
 	int final_buff_len = 0;
 
 	u8 cluster_id = 1;
 	bool more = true;
 
 	u8 dump_buf[4096];
 	u16 curr_buff_size = 4096;
 	u8 curr_next_table = 0;
 	u32 curr_next_index = 0;
 
 	u16 ret_buff_size;
 	u8 ret_next_table;
 	u32 ret_next_index;
 
 	sbuf_cat(buf, "\n");
 
 	while (more) {
 		status = i40e_aq_debug_dump(hw, cluster_id, curr_next_table, curr_next_index, curr_buff_size,
 		    dump_buf, &ret_buff_size, &ret_next_table, &ret_next_index, NULL);
 		if (status) {
 			device_printf(dev, "i40e_aq_debug_dump status %s, error %s\n",
 			    i40e_stat_str(hw, status), i40e_aq_str(hw, hw->aq.asq_last_status));
 			goto free_out;
 		}
 
 		/* copy info out of temp buffer */
 		bcopy(dump_buf, (caddr_t)final_buff + final_buff_len, ret_buff_size);
 		final_buff_len += ret_buff_size;
 
 		if (ret_next_table != curr_next_table) {
 			/* We're done with the current table; we can dump out read data. */
 			sbuf_printf(buf, "%d:", curr_next_table);
 			int bytes_printed = 0;
 			while (bytes_printed <= final_buff_len) {
 				sbuf_printf(buf, "%16D", ((caddr_t)final_buff + bytes_printed), "");
 				bytes_printed += 16;
 			}
 				sbuf_cat(buf, "\n");
 
 			/* The entire cluster has been read; we're finished */
 			if (ret_next_table == 0xFF)
 				break;
 
 			/* Otherwise clear the output buffer and continue reading */
 			bzero(final_buff, IXL_FINAL_BUFF_SIZE);
 			final_buff_len = 0;
 		}
 
 		if (ret_next_index == 0xFFFFFFFF)
 			ret_next_index = 0;
 
 		bzero(dump_buf, sizeof(dump_buf));
 		curr_next_table = ret_next_table;
 		curr_next_index = ret_next_index;
 	}
 
 free_out:
 	free(final_buff, M_DEVBUF);
 out:
 	error = sbuf_finish(buf);
 	if (error)
 		device_printf(dev, "Error finishing sbuf: %d\n", error);
 	sbuf_delete(buf);
 
 	return (error);
 }
 
 static int
 ixl_sysctl_fw_lldp(SYSCTL_HANDLER_ARGS)
 {
 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
 	struct i40e_hw *hw = &pf->hw;
 	device_t dev = pf->dev;
 	int error = 0;
 	int state, new_state;
 	enum i40e_status_code status;
 	state = new_state = ((pf->state & IXL_PF_STATE_FW_LLDP_DISABLED) == 0);
 
 	/* Read in new mode */
 	error = sysctl_handle_int(oidp, &new_state, 0, req);
 	if ((error) || (req->newptr == NULL))
 		return (error);
 
 	/* Already in requested state */
 	if (new_state == state)
 		return (error);
 
 	if (new_state == 0) {
 		if (hw->mac.type == I40E_MAC_X722 || hw->func_caps.npar_enable != 0) {
 			device_printf(dev, "Disabling FW LLDP agent is not supported on this device\n");
 			return (EINVAL);
 		}
 
 		if (pf->hw.aq.api_maj_ver < 1 ||
 		    (pf->hw.aq.api_maj_ver == 1 &&
 		    pf->hw.aq.api_min_ver < 7)) {
 			device_printf(dev, "Disabling FW LLDP agent is not supported in this FW version. Please update FW to enable this feature.\n");
 			return (EINVAL);
 		}
 
 		i40e_aq_stop_lldp(&pf->hw, true, NULL);
 		i40e_aq_set_dcb_parameters(&pf->hw, true, NULL);
 		atomic_set_int(&pf->state, IXL_PF_STATE_FW_LLDP_DISABLED);
 	} else {
 		status = i40e_aq_start_lldp(&pf->hw, NULL);
 		if (status != I40E_SUCCESS && hw->aq.asq_last_status == I40E_AQ_RC_EEXIST)
 			device_printf(dev, "FW LLDP agent is already running\n");
 		atomic_clear_int(&pf->state, IXL_PF_STATE_FW_LLDP_DISABLED);
 	}
 
 	return (0);
 }
 
 /*
  * Get FW LLDP Agent status
  */
 int
 ixl_get_fw_lldp_status(struct ixl_pf *pf)
 {
 	enum i40e_status_code ret = I40E_SUCCESS;
 	struct i40e_lldp_variables lldp_cfg;
 	struct i40e_hw *hw = &pf->hw;
 	u8 adminstatus = 0;
 
 	ret = i40e_read_lldp_cfg(hw, &lldp_cfg);
 	if (ret)
 		return ret;
 
 	/* Get the LLDP AdminStatus for the current port */
 	adminstatus = lldp_cfg.adminstatus >> (hw->port * 4);
 	adminstatus &= 0xf;
 
 	/* Check if LLDP agent is disabled */
 	if (!adminstatus) {
 		device_printf(pf->dev, "FW LLDP agent is disabled for this PF.\n");
 		atomic_set_int(&pf->state, IXL_PF_STATE_FW_LLDP_DISABLED);
 	} else
 		atomic_clear_int(&pf->state, IXL_PF_STATE_FW_LLDP_DISABLED);
 
 	return (0);
 }
 
 int
 ixl_attach_get_link_status(struct ixl_pf *pf)
 {
 	struct i40e_hw *hw = &pf->hw;
 	device_t dev = pf->dev;
 	int error = 0;
 
 	if (((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver < 33)) ||
 	    (hw->aq.fw_maj_ver < 4)) {
 		i40e_msec_delay(75);
 		error = i40e_aq_set_link_restart_an(hw, TRUE, NULL);
 		if (error) {
 			device_printf(dev, "link restart failed, aq_err=%d\n",
 			    pf->hw.aq.asq_last_status);
 			return error;
 		}
 	}
 
 	/* Determine link state */
 	hw->phy.get_link_info = TRUE;
 	i40e_get_link_status(hw, &pf->link_up);
 	return (0);
 }
 
 static int
 ixl_sysctl_do_pf_reset(SYSCTL_HANDLER_ARGS)
 {
 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
 	int requested = 0, error = 0;
 
 	/* Read in new mode */
 	error = sysctl_handle_int(oidp, &requested, 0, req);
 	if ((error) || (req->newptr == NULL))
 		return (error);
 
 	/* Initiate the PF reset later in the admin task */
 	atomic_set_32(&pf->state, IXL_PF_STATE_PF_RESET_REQ);
 
 	return (error);
 }
 
 static int
 ixl_sysctl_do_core_reset(SYSCTL_HANDLER_ARGS)
 {
 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
 	struct i40e_hw *hw = &pf->hw;
 	int requested = 0, error = 0;
 
 	/* Read in new mode */
 	error = sysctl_handle_int(oidp, &requested, 0, req);
 	if ((error) || (req->newptr == NULL))
 		return (error);
 
 	wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK);
 
 	return (error);
 }
 
 static int
 ixl_sysctl_do_global_reset(SYSCTL_HANDLER_ARGS)
 {
 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
 	struct i40e_hw *hw = &pf->hw;
 	int requested = 0, error = 0;
 
 	/* Read in new mode */
 	error = sysctl_handle_int(oidp, &requested, 0, req);
 	if ((error) || (req->newptr == NULL))
 		return (error);
 
 	wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_GLOBR_MASK);
 
 	return (error);
 }
 
 static int
 ixl_sysctl_do_emp_reset(SYSCTL_HANDLER_ARGS)
 {
 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
 	struct i40e_hw *hw = &pf->hw;
 	int requested = 0, error = 0;
 
 	/* Read in new mode */
 	error = sysctl_handle_int(oidp, &requested, 0, req);
 	if ((error) || (req->newptr == NULL))
 		return (error);
 
 	/* TODO: Find out how to bypass this */
 	if (!(rd32(hw, 0x000B818C) & 0x1)) {
 		device_printf(pf->dev, "SW not allowed to initiate EMPR\n");
 		error = EINVAL;
 	} else
 		wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_EMPFWR_MASK);
 
 	return (error);
 }
 
 /*
  * Print out mapping of TX queue indexes and Rx queue indexes
  * to MSI-X vectors.
  */
 static int
 ixl_sysctl_queue_interrupt_table(SYSCTL_HANDLER_ARGS)
 {
 	struct ixl_pf *pf = (struct ixl_pf *)arg1;
 	struct ixl_vsi *vsi = &pf->vsi;
 	device_t dev = pf->dev;
 	struct sbuf *buf;
 	int error = 0;
 
 	struct ixl_rx_queue *rx_que = vsi->rx_queues;
 	struct ixl_tx_queue *tx_que = vsi->tx_queues;
 
 	buf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
 	if (!buf) {
 		device_printf(dev, "Could not allocate sbuf for output.\n");
 		return (ENOMEM);
 	}
 
 	sbuf_cat(buf, "\n");
 	for (int i = 0; i < vsi->num_rx_queues; i++) {
 		rx_que = &vsi->rx_queues[i];
 		sbuf_printf(buf, "(rxq %3d): %d\n", i, rx_que->msix);
 	}
 	for (int i = 0; i < vsi->num_tx_queues; i++) {
 		tx_que = &vsi->tx_queues[i];
 		sbuf_printf(buf, "(txq %3d): %d\n", i, tx_que->msix);
 	}
 
 	error = sbuf_finish(buf);
 	if (error)
 		device_printf(dev, "Error finishing sbuf: %d\n", error);
 	sbuf_delete(buf);
 
 	return (error);
 }
Index: head/sys/net/iflib.c
===================================================================
--- head/sys/net/iflib.c	(revision 345304)
+++ head/sys/net/iflib.c	(revision 345305)
@@ -1,6547 +1,6569 @@
 /*-
  * Copyright (c) 2014-2018, Matthew Macy <mmacy@mattmacy.io>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
  *
  *  1. Redistributions of source code must retain the above copyright notice,
  *     this list of conditions and the following disclaimer.
  *
  *  2. Neither the name of Matthew Macy nor the names of its
  *     contributors may be used to endorse or promote products derived from
  *     this software without specific prior written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include "opt_inet.h"
 #include "opt_inet6.h"
 #include "opt_acpi.h"
 #include "opt_sched.h"
 
 #include <sys/param.h>
 #include <sys/types.h>
 #include <sys/bus.h>
 #include <sys/eventhandler.h>
 #include <sys/jail.h>
 #include <sys/kernel.h>
 #include <sys/lock.h>
 #include <sys/md5.h>
 #include <sys/mutex.h>
 #include <sys/module.h>
 #include <sys/kobj.h>
 #include <sys/rman.h>
 #include <sys/proc.h>
 #include <sys/sbuf.h>
 #include <sys/smp.h>
 #include <sys/socket.h>
 #include <sys/sockio.h>
 #include <sys/sysctl.h>
 #include <sys/syslog.h>
 #include <sys/taskqueue.h>
 #include <sys/limits.h>
 
 #include <net/if.h>
 #include <net/if_var.h>
 #include <net/if_types.h>
 #include <net/if_media.h>
 #include <net/bpf.h>
 #include <net/ethernet.h>
 #include <net/mp_ring.h>
 #include <net/vnet.h>
 
 #include <netinet/in.h>
 #include <netinet/in_pcb.h>
 #include <netinet/tcp_lro.h>
 #include <netinet/in_systm.h>
 #include <netinet/if_ether.h>
 #include <netinet/ip.h>
 #include <netinet/ip6.h>
 #include <netinet/tcp.h>
 #include <netinet/ip_var.h>
 #include <netinet/netdump/netdump.h>
 #include <netinet6/ip6_var.h>
 
 #include <machine/bus.h>
 #include <machine/in_cksum.h>
 
 #include <vm/vm.h>
 #include <vm/pmap.h>
 
 #include <dev/led/led.h>
 #include <dev/pci/pcireg.h>
 #include <dev/pci/pcivar.h>
 #include <dev/pci/pci_private.h>
 
 #include <net/iflib.h>
 #include <net/iflib_private.h>
 
 #include "ifdi_if.h"
 
 #ifdef PCI_IOV
 #include <dev/pci/pci_iov.h>
 #endif
 
 #include <sys/bitstring.h>
 /*
  * enable accounting of every mbuf as it comes in to and goes out of
  * iflib's software descriptor references
  */
 #define MEMORY_LOGGING 0
 /*
  * Enable mbuf vectors for compressing long mbuf chains
  */
 
 /*
  * NB:
  * - Prefetching in tx cleaning should perhaps be a tunable. The distance ahead
  *   we prefetch needs to be determined by the time spent in m_free vis a vis
  *   the cost of a prefetch. This will of course vary based on the workload:
  *      - NFLX's m_free path is dominated by vm-based M_EXT manipulation which
  *        is quite expensive, thus suggesting very little prefetch.
  *      - small packet forwarding which is just returning a single mbuf to
  *        UMA will typically be very fast vis a vis the cost of a memory
  *        access.
  */
 
 
 /*
  * File organization:
  *  - private structures
  *  - iflib private utility functions
  *  - ifnet functions
  *  - vlan registry and other exported functions
  *  - iflib public core functions
  *
  *
  */
 MALLOC_DEFINE(M_IFLIB, "iflib", "ifnet library");
 
 struct iflib_txq;
 typedef struct iflib_txq *iflib_txq_t;
 struct iflib_rxq;
 typedef struct iflib_rxq *iflib_rxq_t;
 struct iflib_fl;
 typedef struct iflib_fl *iflib_fl_t;
 
 struct iflib_ctx;
 
 static void iru_init(if_rxd_update_t iru, iflib_rxq_t rxq, uint8_t flid);
 static void iflib_timer(void *arg);
 
 typedef struct iflib_filter_info {
 	driver_filter_t *ifi_filter;
 	void *ifi_filter_arg;
 	struct grouptask *ifi_task;
 	void *ifi_ctx;
 } *iflib_filter_info_t;
 
 struct iflib_ctx {
 	KOBJ_FIELDS;
 	/*
 	 * Pointer to hardware driver's softc
 	 */
 	void *ifc_softc;
 	device_t ifc_dev;
 	if_t ifc_ifp;
 
 	cpuset_t ifc_cpus;
 	if_shared_ctx_t ifc_sctx;
 	struct if_softc_ctx ifc_softc_ctx;
 
 	struct sx ifc_ctx_sx;
 	struct mtx ifc_state_mtx;
 
 	iflib_txq_t ifc_txqs;
 	iflib_rxq_t ifc_rxqs;
 	uint32_t ifc_if_flags;
 	uint32_t ifc_flags;
 	uint32_t ifc_max_fl_buf_size;
+	uint32_t ifc_rx_mbuf_sz;
 
 	int ifc_link_state;
 	int ifc_link_irq;
 	int ifc_watchdog_events;
 	struct cdev *ifc_led_dev;
 	struct resource *ifc_msix_mem;
 
 	struct if_irq ifc_legacy_irq;
 	struct grouptask ifc_admin_task;
 	struct grouptask ifc_vflr_task;
 	struct iflib_filter_info ifc_filter_info;
 	struct ifmedia	ifc_media;
 
 	struct sysctl_oid *ifc_sysctl_node;
 	uint16_t ifc_sysctl_ntxqs;
 	uint16_t ifc_sysctl_nrxqs;
 	uint16_t ifc_sysctl_qs_eq_override;
 	uint16_t ifc_sysctl_rx_budget;
 	uint16_t ifc_sysctl_tx_abdicate;
 
 	qidx_t ifc_sysctl_ntxds[8];
 	qidx_t ifc_sysctl_nrxds[8];
 	struct if_txrx ifc_txrx;
 #define isc_txd_encap  ifc_txrx.ift_txd_encap
 #define isc_txd_flush  ifc_txrx.ift_txd_flush
 #define isc_txd_credits_update  ifc_txrx.ift_txd_credits_update
 #define isc_rxd_available ifc_txrx.ift_rxd_available
 #define isc_rxd_pkt_get ifc_txrx.ift_rxd_pkt_get
 #define isc_rxd_refill ifc_txrx.ift_rxd_refill
 #define isc_rxd_flush ifc_txrx.ift_rxd_flush
 #define isc_rxd_refill ifc_txrx.ift_rxd_refill
 #define isc_rxd_refill ifc_txrx.ift_rxd_refill
 #define isc_legacy_intr ifc_txrx.ift_legacy_intr
 	eventhandler_tag ifc_vlan_attach_event;
 	eventhandler_tag ifc_vlan_detach_event;
 	uint8_t ifc_mac[ETHER_ADDR_LEN];
 	char ifc_mtx_name[16];
 };
 
 
 void *
 iflib_get_softc(if_ctx_t ctx)
 {
 
 	return (ctx->ifc_softc);
 }
 
 device_t
 iflib_get_dev(if_ctx_t ctx)
 {
 
 	return (ctx->ifc_dev);
 }
 
 if_t
 iflib_get_ifp(if_ctx_t ctx)
 {
 
 	return (ctx->ifc_ifp);
 }
 
 struct ifmedia *
 iflib_get_media(if_ctx_t ctx)
 {
 
 	return (&ctx->ifc_media);
 }
 
 uint32_t
 iflib_get_flags(if_ctx_t ctx)
 {
 	return (ctx->ifc_flags);
 }
 
 void
 iflib_set_mac(if_ctx_t ctx, uint8_t mac[ETHER_ADDR_LEN])
 {
 
 	bcopy(mac, ctx->ifc_mac, ETHER_ADDR_LEN);
 }
 
 if_softc_ctx_t
 iflib_get_softc_ctx(if_ctx_t ctx)
 {
 
 	return (&ctx->ifc_softc_ctx);
 }
 
 if_shared_ctx_t
 iflib_get_sctx(if_ctx_t ctx)
 {
 
 	return (ctx->ifc_sctx);
 }
 
 #define IP_ALIGNED(m) ((((uintptr_t)(m)->m_data) & 0x3) == 0x2)
 #define CACHE_PTR_INCREMENT (CACHE_LINE_SIZE/sizeof(void*))
 #define CACHE_PTR_NEXT(ptr) ((void *)(((uintptr_t)(ptr)+CACHE_LINE_SIZE-1) & (CACHE_LINE_SIZE-1)))
 
 #define LINK_ACTIVE(ctx) ((ctx)->ifc_link_state == LINK_STATE_UP)
 #define CTX_IS_VF(ctx) ((ctx)->ifc_sctx->isc_flags & IFLIB_IS_VF)
 
 typedef struct iflib_sw_rx_desc_array {
 	bus_dmamap_t	*ifsd_map;         /* bus_dma maps for packet */
 	struct mbuf	**ifsd_m;           /* pkthdr mbufs */
 	caddr_t		*ifsd_cl;          /* direct cluster pointer for rx */
 	bus_addr_t	*ifsd_ba;          /* bus addr of cluster for rx */
 } iflib_rxsd_array_t;
 
 typedef struct iflib_sw_tx_desc_array {
 	bus_dmamap_t    *ifsd_map;         /* bus_dma maps for packet */
 	bus_dmamap_t	*ifsd_tso_map;     /* bus_dma maps for TSO packet */
 	struct mbuf    **ifsd_m;           /* pkthdr mbufs */
 } if_txsd_vec_t;
 
 
 /* magic number that should be high enough for any hardware */
 #define IFLIB_MAX_TX_SEGS		128
 #define IFLIB_RX_COPY_THRESH		128
 #define IFLIB_MAX_RX_REFRESH		32
 /* The minimum descriptors per second before we start coalescing */
 #define IFLIB_MIN_DESC_SEC		16384
 #define IFLIB_DEFAULT_TX_UPDATE_FREQ	16
 #define IFLIB_QUEUE_IDLE		0
 #define IFLIB_QUEUE_HUNG		1
 #define IFLIB_QUEUE_WORKING		2
 /* maximum number of txqs that can share an rx interrupt */
 #define IFLIB_MAX_TX_SHARED_INTR	4
 
 /* this should really scale with ring size - this is a fairly arbitrary value */
 #define TX_BATCH_SIZE			32
 
 #define IFLIB_RESTART_BUDGET		8
 
 
 #define CSUM_OFFLOAD		(CSUM_IP_TSO|CSUM_IP6_TSO|CSUM_IP| \
 				 CSUM_IP_UDP|CSUM_IP_TCP|CSUM_IP_SCTP| \
 				 CSUM_IP6_UDP|CSUM_IP6_TCP|CSUM_IP6_SCTP)
 struct iflib_txq {
 	qidx_t		ift_in_use;
 	qidx_t		ift_cidx;
 	qidx_t		ift_cidx_processed;
 	qidx_t		ift_pidx;
 	uint8_t		ift_gen;
 	uint8_t		ift_br_offset;
 	uint16_t	ift_npending;
 	uint16_t	ift_db_pending;
 	uint16_t	ift_rs_pending;
 	/* implicit pad */
 	uint8_t		ift_txd_size[8];
 	uint64_t	ift_processed;
 	uint64_t	ift_cleaned;
 	uint64_t	ift_cleaned_prev;
 #if MEMORY_LOGGING
 	uint64_t	ift_enqueued;
 	uint64_t	ift_dequeued;
 #endif
 	uint64_t	ift_no_tx_dma_setup;
 	uint64_t	ift_no_desc_avail;
 	uint64_t	ift_mbuf_defrag_failed;
 	uint64_t	ift_mbuf_defrag;
 	uint64_t	ift_map_failed;
 	uint64_t	ift_txd_encap_efbig;
 	uint64_t	ift_pullups;
 	uint64_t	ift_last_timer_tick;
 
 	struct mtx	ift_mtx;
 	struct mtx	ift_db_mtx;
 
 	/* constant values */
 	if_ctx_t	ift_ctx;
 	struct ifmp_ring        *ift_br;
 	struct grouptask	ift_task;
 	qidx_t		ift_size;
 	uint16_t	ift_id;
 	struct callout	ift_timer;
 
 	if_txsd_vec_t	ift_sds;
 	uint8_t		ift_qstatus;
 	uint8_t		ift_closed;
 	uint8_t		ift_update_freq;
 	struct iflib_filter_info ift_filter_info;
 	bus_dma_tag_t	ift_buf_tag;
 	bus_dma_tag_t	ift_tso_buf_tag;
 	iflib_dma_info_t	ift_ifdi;
 #define MTX_NAME_LEN 16
 	char                    ift_mtx_name[MTX_NAME_LEN];
 	char                    ift_db_mtx_name[MTX_NAME_LEN];
 	bus_dma_segment_t	ift_segs[IFLIB_MAX_TX_SEGS]  __aligned(CACHE_LINE_SIZE);
 #ifdef IFLIB_DIAGNOSTICS
 	uint64_t ift_cpu_exec_count[256];
 #endif
 } __aligned(CACHE_LINE_SIZE);
 
 struct iflib_fl {
 	qidx_t		ifl_cidx;
 	qidx_t		ifl_pidx;
 	qidx_t		ifl_credits;
 	uint8_t		ifl_gen;
 	uint8_t		ifl_rxd_size;
 #if MEMORY_LOGGING
 	uint64_t	ifl_m_enqueued;
 	uint64_t	ifl_m_dequeued;
 	uint64_t	ifl_cl_enqueued;
 	uint64_t	ifl_cl_dequeued;
 #endif
 	/* implicit pad */
 
 	bitstr_t 	*ifl_rx_bitmap;
 	qidx_t		ifl_fragidx;
 	/* constant */
 	qidx_t		ifl_size;
 	uint16_t	ifl_buf_size;
 	uint16_t	ifl_cltype;
 	uma_zone_t	ifl_zone;
 	iflib_rxsd_array_t	ifl_sds;
 	iflib_rxq_t	ifl_rxq;
 	uint8_t		ifl_id;
 	bus_dma_tag_t	ifl_buf_tag;
 	iflib_dma_info_t	ifl_ifdi;
 	uint64_t	ifl_bus_addrs[IFLIB_MAX_RX_REFRESH] __aligned(CACHE_LINE_SIZE);
 	caddr_t		ifl_vm_addrs[IFLIB_MAX_RX_REFRESH];
 	qidx_t	ifl_rxd_idxs[IFLIB_MAX_RX_REFRESH];
 }  __aligned(CACHE_LINE_SIZE);
 
 static inline qidx_t
 get_inuse(int size, qidx_t cidx, qidx_t pidx, uint8_t gen)
 {
 	qidx_t used;
 
 	if (pidx > cidx)
 		used = pidx - cidx;
 	else if (pidx < cidx)
 		used = size - cidx + pidx;
 	else if (gen == 0 && pidx == cidx)
 		used = 0;
 	else if (gen == 1 && pidx == cidx)
 		used = size;
 	else
 		panic("bad state");
 
 	return (used);
 }
 
 #define TXQ_AVAIL(txq) (txq->ift_size - get_inuse(txq->ift_size, txq->ift_cidx, txq->ift_pidx, txq->ift_gen))
 
 #define IDXDIFF(head, tail, wrap) \
 	((head) >= (tail) ? (head) - (tail) : (wrap) - (tail) + (head))
 
 struct iflib_rxq {
 	/* If there is a separate completion queue -
 	 * these are the cq cidx and pidx. Otherwise
 	 * these are unused.
 	 */
 	qidx_t		ifr_size;
 	qidx_t		ifr_cq_cidx;
 	qidx_t		ifr_cq_pidx;
 	uint8_t		ifr_cq_gen;
 	uint8_t		ifr_fl_offset;
 
 	if_ctx_t	ifr_ctx;
 	iflib_fl_t	ifr_fl;
 	uint64_t	ifr_rx_irq;
 	uint16_t	ifr_id;
 	uint8_t		ifr_lro_enabled;
 	uint8_t		ifr_nfl;
 	uint8_t		ifr_ntxqirq;
 	uint8_t		ifr_txqid[IFLIB_MAX_TX_SHARED_INTR];
 	struct lro_ctrl			ifr_lc;
 	struct grouptask        ifr_task;
 	struct iflib_filter_info ifr_filter_info;
 	iflib_dma_info_t		ifr_ifdi;
 
 	/* dynamically allocate if any drivers need a value substantially larger than this */
 	struct if_rxd_frag	ifr_frags[IFLIB_MAX_RX_SEGS] __aligned(CACHE_LINE_SIZE);
 #ifdef IFLIB_DIAGNOSTICS
 	uint64_t ifr_cpu_exec_count[256];
 #endif
 }  __aligned(CACHE_LINE_SIZE);
 
 typedef struct if_rxsd {
 	caddr_t *ifsd_cl;
 	struct mbuf **ifsd_m;
 	iflib_fl_t ifsd_fl;
 	qidx_t ifsd_cidx;
 } *if_rxsd_t;
 
 /* multiple of word size */
 #ifdef __LP64__
 #define PKT_INFO_SIZE	6
 #define RXD_INFO_SIZE	5
 #define PKT_TYPE uint64_t
 #else
 #define PKT_INFO_SIZE	11
 #define RXD_INFO_SIZE	8
 #define PKT_TYPE uint32_t
 #endif
 #define PKT_LOOP_BOUND  ((PKT_INFO_SIZE/3)*3)
 #define RXD_LOOP_BOUND  ((RXD_INFO_SIZE/4)*4)
 
 typedef struct if_pkt_info_pad {
 	PKT_TYPE pkt_val[PKT_INFO_SIZE];
 } *if_pkt_info_pad_t;
 typedef struct if_rxd_info_pad {
 	PKT_TYPE rxd_val[RXD_INFO_SIZE];
 } *if_rxd_info_pad_t;
 
 CTASSERT(sizeof(struct if_pkt_info_pad) == sizeof(struct if_pkt_info));
 CTASSERT(sizeof(struct if_rxd_info_pad) == sizeof(struct if_rxd_info));
 
 
 static inline void
 pkt_info_zero(if_pkt_info_t pi)
 {
 	if_pkt_info_pad_t pi_pad;
 
 	pi_pad = (if_pkt_info_pad_t)pi;
 	pi_pad->pkt_val[0] = 0; pi_pad->pkt_val[1] = 0; pi_pad->pkt_val[2] = 0;
 	pi_pad->pkt_val[3] = 0; pi_pad->pkt_val[4] = 0; pi_pad->pkt_val[5] = 0;
 #ifndef __LP64__
 	pi_pad->pkt_val[6] = 0; pi_pad->pkt_val[7] = 0; pi_pad->pkt_val[8] = 0;
 	pi_pad->pkt_val[9] = 0; pi_pad->pkt_val[10] = 0;
 #endif	
 }
 
 static device_method_t iflib_pseudo_methods[] = {
 	DEVMETHOD(device_attach, noop_attach),
 	DEVMETHOD(device_detach, iflib_pseudo_detach),
 	DEVMETHOD_END
 };
 
 driver_t iflib_pseudodriver = {
 	"iflib_pseudo", iflib_pseudo_methods, sizeof(struct iflib_ctx),
 };
 
 static inline void
 rxd_info_zero(if_rxd_info_t ri)
 {
 	if_rxd_info_pad_t ri_pad;
 	int i;
 
 	ri_pad = (if_rxd_info_pad_t)ri;
 	for (i = 0; i < RXD_LOOP_BOUND; i += 4) {
 		ri_pad->rxd_val[i] = 0;
 		ri_pad->rxd_val[i+1] = 0;
 		ri_pad->rxd_val[i+2] = 0;
 		ri_pad->rxd_val[i+3] = 0;
 	}
 #ifdef __LP64__
 	ri_pad->rxd_val[RXD_INFO_SIZE-1] = 0;
 #endif
 }
 
 /*
  * Only allow a single packet to take up most 1/nth of the tx ring
  */
 #define MAX_SINGLE_PACKET_FRACTION 12
 #define IF_BAD_DMA (bus_addr_t)-1
 
 #define CTX_ACTIVE(ctx) ((if_getdrvflags((ctx)->ifc_ifp) & IFF_DRV_RUNNING))
 
 #define CTX_LOCK_INIT(_sc)  sx_init(&(_sc)->ifc_ctx_sx, "iflib ctx lock")
 #define CTX_LOCK(ctx) sx_xlock(&(ctx)->ifc_ctx_sx)
 #define CTX_UNLOCK(ctx) sx_xunlock(&(ctx)->ifc_ctx_sx)
 #define CTX_LOCK_DESTROY(ctx) sx_destroy(&(ctx)->ifc_ctx_sx)
 
 
 #define STATE_LOCK_INIT(_sc, _name)  mtx_init(&(_sc)->ifc_state_mtx, _name, "iflib state lock", MTX_DEF)
 #define STATE_LOCK(ctx) mtx_lock(&(ctx)->ifc_state_mtx)
 #define STATE_UNLOCK(ctx) mtx_unlock(&(ctx)->ifc_state_mtx)
 #define STATE_LOCK_DESTROY(ctx) mtx_destroy(&(ctx)->ifc_state_mtx)
 
 
 
 #define CALLOUT_LOCK(txq)	mtx_lock(&txq->ift_mtx)
 #define CALLOUT_UNLOCK(txq) 	mtx_unlock(&txq->ift_mtx)
 
 void
 iflib_set_detach(if_ctx_t ctx)
 {
 	STATE_LOCK(ctx);
 	ctx->ifc_flags |= IFC_IN_DETACH;
 	STATE_UNLOCK(ctx);
 }
 
 /* Our boot-time initialization hook */
 static int	iflib_module_event_handler(module_t, int, void *);
 
 static moduledata_t iflib_moduledata = {
 	"iflib",
 	iflib_module_event_handler,
 	NULL
 };
 
 DECLARE_MODULE(iflib, iflib_moduledata, SI_SUB_INIT_IF, SI_ORDER_ANY);
 MODULE_VERSION(iflib, 1);
 
 MODULE_DEPEND(iflib, pci, 1, 1, 1);
 MODULE_DEPEND(iflib, ether, 1, 1, 1);
 
 TASKQGROUP_DEFINE(if_io_tqg, mp_ncpus, 1);
 TASKQGROUP_DEFINE(if_config_tqg, 1, 1);
 
 #ifndef IFLIB_DEBUG_COUNTERS
 #ifdef INVARIANTS
 #define IFLIB_DEBUG_COUNTERS 1
 #else
 #define IFLIB_DEBUG_COUNTERS 0
 #endif /* !INVARIANTS */
 #endif
 
 static SYSCTL_NODE(_net, OID_AUTO, iflib, CTLFLAG_RD, 0,
                    "iflib driver parameters");
 
 /*
  * XXX need to ensure that this can't accidentally cause the head to be moved backwards 
  */
 static int iflib_min_tx_latency = 0;
 SYSCTL_INT(_net_iflib, OID_AUTO, min_tx_latency, CTLFLAG_RW,
 		   &iflib_min_tx_latency, 0, "minimize transmit latency at the possible expense of throughput");
 static int iflib_no_tx_batch = 0;
 SYSCTL_INT(_net_iflib, OID_AUTO, no_tx_batch, CTLFLAG_RW,
 		   &iflib_no_tx_batch, 0, "minimize transmit latency at the possible expense of throughput");
 
 
 #if IFLIB_DEBUG_COUNTERS
 
 static int iflib_tx_seen;
 static int iflib_tx_sent;
 static int iflib_tx_encap;
 static int iflib_rx_allocs;
 static int iflib_fl_refills;
 static int iflib_fl_refills_large;
 static int iflib_tx_frees;
 
 SYSCTL_INT(_net_iflib, OID_AUTO, tx_seen, CTLFLAG_RD,
 		   &iflib_tx_seen, 0, "# tx mbufs seen");
 SYSCTL_INT(_net_iflib, OID_AUTO, tx_sent, CTLFLAG_RD,
 		   &iflib_tx_sent, 0, "# tx mbufs sent");
 SYSCTL_INT(_net_iflib, OID_AUTO, tx_encap, CTLFLAG_RD,
 		   &iflib_tx_encap, 0, "# tx mbufs encapped");
 SYSCTL_INT(_net_iflib, OID_AUTO, tx_frees, CTLFLAG_RD,
 		   &iflib_tx_frees, 0, "# tx frees");
 SYSCTL_INT(_net_iflib, OID_AUTO, rx_allocs, CTLFLAG_RD,
 		   &iflib_rx_allocs, 0, "# rx allocations");
 SYSCTL_INT(_net_iflib, OID_AUTO, fl_refills, CTLFLAG_RD,
 		   &iflib_fl_refills, 0, "# refills");
 SYSCTL_INT(_net_iflib, OID_AUTO, fl_refills_large, CTLFLAG_RD,
 		   &iflib_fl_refills_large, 0, "# large refills");
 
 
 static int iflib_txq_drain_flushing;
 static int iflib_txq_drain_oactive;
 static int iflib_txq_drain_notready;
 
 SYSCTL_INT(_net_iflib, OID_AUTO, txq_drain_flushing, CTLFLAG_RD,
 		   &iflib_txq_drain_flushing, 0, "# drain flushes");
 SYSCTL_INT(_net_iflib, OID_AUTO, txq_drain_oactive, CTLFLAG_RD,
 		   &iflib_txq_drain_oactive, 0, "# drain oactives");
 SYSCTL_INT(_net_iflib, OID_AUTO, txq_drain_notready, CTLFLAG_RD,
 		   &iflib_txq_drain_notready, 0, "# drain notready");
 
 
 static int iflib_encap_load_mbuf_fail;
 static int iflib_encap_pad_mbuf_fail;
 static int iflib_encap_txq_avail_fail;
 static int iflib_encap_txd_encap_fail;
 
 SYSCTL_INT(_net_iflib, OID_AUTO, encap_load_mbuf_fail, CTLFLAG_RD,
 		   &iflib_encap_load_mbuf_fail, 0, "# busdma load failures");
 SYSCTL_INT(_net_iflib, OID_AUTO, encap_pad_mbuf_fail, CTLFLAG_RD,
 		   &iflib_encap_pad_mbuf_fail, 0, "# runt frame pad failures");
 SYSCTL_INT(_net_iflib, OID_AUTO, encap_txq_avail_fail, CTLFLAG_RD,
 		   &iflib_encap_txq_avail_fail, 0, "# txq avail failures");
 SYSCTL_INT(_net_iflib, OID_AUTO, encap_txd_encap_fail, CTLFLAG_RD,
 		   &iflib_encap_txd_encap_fail, 0, "# driver encap failures");
 
 static int iflib_task_fn_rxs;
 static int iflib_rx_intr_enables;
 static int iflib_fast_intrs;
 static int iflib_rx_unavail;
 static int iflib_rx_ctx_inactive;
 static int iflib_rx_if_input;
 static int iflib_rx_mbuf_null;
 static int iflib_rxd_flush;
 
 static int iflib_verbose_debug;
 
 SYSCTL_INT(_net_iflib, OID_AUTO, task_fn_rx, CTLFLAG_RD,
 		   &iflib_task_fn_rxs, 0, "# task_fn_rx calls");
 SYSCTL_INT(_net_iflib, OID_AUTO, rx_intr_enables, CTLFLAG_RD,
 		   &iflib_rx_intr_enables, 0, "# rx intr enables");
 SYSCTL_INT(_net_iflib, OID_AUTO, fast_intrs, CTLFLAG_RD,
 		   &iflib_fast_intrs, 0, "# fast_intr calls");
 SYSCTL_INT(_net_iflib, OID_AUTO, rx_unavail, CTLFLAG_RD,
 		   &iflib_rx_unavail, 0, "# times rxeof called with no available data");
 SYSCTL_INT(_net_iflib, OID_AUTO, rx_ctx_inactive, CTLFLAG_RD,
 		   &iflib_rx_ctx_inactive, 0, "# times rxeof called with inactive context");
 SYSCTL_INT(_net_iflib, OID_AUTO, rx_if_input, CTLFLAG_RD,
 		   &iflib_rx_if_input, 0, "# times rxeof called if_input");
 SYSCTL_INT(_net_iflib, OID_AUTO, rx_mbuf_null, CTLFLAG_RD,
 		   &iflib_rx_mbuf_null, 0, "# times rxeof got null mbuf");
 SYSCTL_INT(_net_iflib, OID_AUTO, rxd_flush, CTLFLAG_RD,
 	         &iflib_rxd_flush, 0, "# times rxd_flush called");
 SYSCTL_INT(_net_iflib, OID_AUTO, verbose_debug, CTLFLAG_RW,
 		   &iflib_verbose_debug, 0, "enable verbose debugging");
 
 #define DBG_COUNTER_INC(name) atomic_add_int(&(iflib_ ## name), 1)
 static void
 iflib_debug_reset(void)
 {
 	iflib_tx_seen = iflib_tx_sent = iflib_tx_encap = iflib_rx_allocs =
 		iflib_fl_refills = iflib_fl_refills_large = iflib_tx_frees =
 		iflib_txq_drain_flushing = iflib_txq_drain_oactive =
 		iflib_txq_drain_notready =
 		iflib_encap_load_mbuf_fail = iflib_encap_pad_mbuf_fail =
 		iflib_encap_txq_avail_fail = iflib_encap_txd_encap_fail =
 		iflib_task_fn_rxs = iflib_rx_intr_enables = iflib_fast_intrs =
 		iflib_rx_unavail =
 		iflib_rx_ctx_inactive = iflib_rx_if_input =
 		iflib_rx_mbuf_null = iflib_rxd_flush = 0;
 }
 
 #else
 #define DBG_COUNTER_INC(name)
 static void iflib_debug_reset(void) {}
 #endif
 
 #define IFLIB_DEBUG 0
 
 static void iflib_tx_structures_free(if_ctx_t ctx);
 static void iflib_rx_structures_free(if_ctx_t ctx);
 static int iflib_queues_alloc(if_ctx_t ctx);
 static int iflib_tx_credits_update(if_ctx_t ctx, iflib_txq_t txq);
 static int iflib_rxd_avail(if_ctx_t ctx, iflib_rxq_t rxq, qidx_t cidx, qidx_t budget);
 static int iflib_qset_structures_setup(if_ctx_t ctx);
 static int iflib_msix_init(if_ctx_t ctx);
 static int iflib_legacy_setup(if_ctx_t ctx, driver_filter_t filter, void *filterarg, int *rid, const char *str);
 static void iflib_txq_check_drain(iflib_txq_t txq, int budget);
 static uint32_t iflib_txq_can_drain(struct ifmp_ring *);
 #ifdef ALTQ
 static void iflib_altq_if_start(if_t ifp);
 static int iflib_altq_if_transmit(if_t ifp, struct mbuf *m);
 #endif
 static int iflib_register(if_ctx_t);
 static void iflib_init_locked(if_ctx_t ctx);
 static void iflib_add_device_sysctl_pre(if_ctx_t ctx);
 static void iflib_add_device_sysctl_post(if_ctx_t ctx);
 static void iflib_ifmp_purge(iflib_txq_t txq);
 static void _iflib_pre_assert(if_softc_ctx_t scctx);
 static void iflib_if_init_locked(if_ctx_t ctx);
 static void iflib_free_intr_mem(if_ctx_t ctx);
 #ifndef __NO_STRICT_ALIGNMENT
 static struct mbuf * iflib_fixup_rx(struct mbuf *m);
 #endif
 
 NETDUMP_DEFINE(iflib);
 
 #ifdef DEV_NETMAP
 #include <sys/selinfo.h>
 #include <net/netmap.h>
 #include <dev/netmap/netmap_kern.h>
 
 MODULE_DEPEND(iflib, netmap, 1, 1, 1);
 
 static int netmap_fl_refill(iflib_rxq_t rxq, struct netmap_kring *kring, uint32_t nm_i, bool init);
 
 /*
  * device-specific sysctl variables:
  *
  * iflib_crcstrip: 0: keep CRC in rx frames (default), 1: strip it.
  *	During regular operations the CRC is stripped, but on some
  *	hardware reception of frames not multiple of 64 is slower,
  *	so using crcstrip=0 helps in benchmarks.
  *
  * iflib_rx_miss, iflib_rx_miss_bufs:
  *	count packets that might be missed due to lost interrupts.
  */
 SYSCTL_DECL(_dev_netmap);
 /*
  * The xl driver by default strips CRCs and we do not override it.
  */
 
 int iflib_crcstrip = 1;
 SYSCTL_INT(_dev_netmap, OID_AUTO, iflib_crcstrip,
     CTLFLAG_RW, &iflib_crcstrip, 1, "strip CRC on rx frames");
 
 int iflib_rx_miss, iflib_rx_miss_bufs;
 SYSCTL_INT(_dev_netmap, OID_AUTO, iflib_rx_miss,
     CTLFLAG_RW, &iflib_rx_miss, 0, "potentially missed rx intr");
 SYSCTL_INT(_dev_netmap, OID_AUTO, iflib_rx_miss_bufs,
     CTLFLAG_RW, &iflib_rx_miss_bufs, 0, "potentially missed rx intr bufs");
 
 /*
  * Register/unregister. We are already under netmap lock.
  * Only called on the first register or the last unregister.
  */
 static int
 iflib_netmap_register(struct netmap_adapter *na, int onoff)
 {
 	struct ifnet *ifp = na->ifp;
 	if_ctx_t ctx = ifp->if_softc;
 	int status;
 
 	CTX_LOCK(ctx);
 	IFDI_INTR_DISABLE(ctx);
 
 	/* Tell the stack that the interface is no longer active */
 	ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
 
 	if (!CTX_IS_VF(ctx))
 		IFDI_CRCSTRIP_SET(ctx, onoff, iflib_crcstrip);
 
 	/* enable or disable flags and callbacks in na and ifp */
 	if (onoff) {
 		nm_set_native_flags(na);
 	} else {
 		nm_clear_native_flags(na);
 	}
 	iflib_stop(ctx);
 	iflib_init_locked(ctx);
 	IFDI_CRCSTRIP_SET(ctx, onoff, iflib_crcstrip); // XXX why twice ?
 	status = ifp->if_drv_flags & IFF_DRV_RUNNING ? 0 : 1;
 	if (status)
 		nm_clear_native_flags(na);
 	CTX_UNLOCK(ctx);
 	return (status);
 }
 
 static int
 netmap_fl_refill(iflib_rxq_t rxq, struct netmap_kring *kring, uint32_t nm_i, bool init)
 {
 	struct netmap_adapter *na = kring->na;
 	u_int const lim = kring->nkr_num_slots - 1;
 	u_int head = kring->rhead;
 	struct netmap_ring *ring = kring->ring;
 	bus_dmamap_t *map;
 	struct if_rxd_update iru;
 	if_ctx_t ctx = rxq->ifr_ctx;
 	iflib_fl_t fl = &rxq->ifr_fl[0];
 	uint32_t refill_pidx, nic_i;
 #if IFLIB_DEBUG_COUNTERS
 	int rf_count = 0;
 #endif
 
 	if (nm_i == head && __predict_true(!init))
 		return 0;
 	iru_init(&iru, rxq, 0 /* flid */);
 	map = fl->ifl_sds.ifsd_map;
 	refill_pidx = netmap_idx_k2n(kring, nm_i);
 	/*
 	 * IMPORTANT: we must leave one free slot in the ring,
 	 * so move head back by one unit
 	 */
 	head = nm_prev(head, lim);
 	nic_i = UINT_MAX;
 	DBG_COUNTER_INC(fl_refills);
 	while (nm_i != head) {
 #if IFLIB_DEBUG_COUNTERS
 		if (++rf_count == 9)
 			DBG_COUNTER_INC(fl_refills_large);
 #endif
 		for (int tmp_pidx = 0; tmp_pidx < IFLIB_MAX_RX_REFRESH && nm_i != head; tmp_pidx++) {
 			struct netmap_slot *slot = &ring->slot[nm_i];
 			void *addr = PNMB(na, slot, &fl->ifl_bus_addrs[tmp_pidx]);
 			uint32_t nic_i_dma = refill_pidx;
 			nic_i = netmap_idx_k2n(kring, nm_i);
 
 			MPASS(tmp_pidx < IFLIB_MAX_RX_REFRESH);
 
 			if (addr == NETMAP_BUF_BASE(na)) /* bad buf */
 			        return netmap_ring_reinit(kring);
 
 			fl->ifl_vm_addrs[tmp_pidx] = addr;
 			if (__predict_false(init)) {
 				netmap_load_map(na, fl->ifl_buf_tag,
 				    map[nic_i], addr);
 			} else if (slot->flags & NS_BUF_CHANGED) {
 				/* buffer has changed, reload map */
 				netmap_reload_map(na, fl->ifl_buf_tag,
 				    map[nic_i], addr);
 			}
 			slot->flags &= ~NS_BUF_CHANGED;
 
 			nm_i = nm_next(nm_i, lim);
 			fl->ifl_rxd_idxs[tmp_pidx] = nic_i = nm_next(nic_i, lim);
 			if (nm_i != head && tmp_pidx < IFLIB_MAX_RX_REFRESH-1)
 				continue;
 
 			iru.iru_pidx = refill_pidx;
 			iru.iru_count = tmp_pidx+1;
 			ctx->isc_rxd_refill(ctx->ifc_softc, &iru);
 			refill_pidx = nic_i;
 			for (int n = 0; n < iru.iru_count; n++) {
 				bus_dmamap_sync(fl->ifl_buf_tag, map[nic_i_dma],
 						BUS_DMASYNC_PREREAD);
 				/* XXX - change this to not use the netmap func*/
 				nic_i_dma = nm_next(nic_i_dma, lim);
 			}
 		}
 	}
 	kring->nr_hwcur = head;
 
 	bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map,
 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 	if (__predict_true(nic_i != UINT_MAX)) {
 		ctx->isc_rxd_flush(ctx->ifc_softc, rxq->ifr_id, fl->ifl_id, nic_i);
 		DBG_COUNTER_INC(rxd_flush);
 	}
 	return (0);
 }
 
 /*
  * Reconcile kernel and user view of the transmit ring.
  *
  * All information is in the kring.
  * Userspace wants to send packets up to the one before kring->rhead,
  * kernel knows kring->nr_hwcur is the first unsent packet.
  *
  * Here we push packets out (as many as possible), and possibly
  * reclaim buffers from previously completed transmission.
  *
  * The caller (netmap) guarantees that there is only one instance
  * running at any time. Any interference with other driver
  * methods should be handled by the individual drivers.
  */
 static int
 iflib_netmap_txsync(struct netmap_kring *kring, int flags)
 {
 	struct netmap_adapter *na = kring->na;
 	struct ifnet *ifp = na->ifp;
 	struct netmap_ring *ring = kring->ring;
 	u_int nm_i;	/* index into the netmap kring */
 	u_int nic_i;	/* index into the NIC ring */
 	u_int n;
 	u_int const lim = kring->nkr_num_slots - 1;
 	u_int const head = kring->rhead;
 	struct if_pkt_info pi;
 
 	/*
 	 * interrupts on every tx packet are expensive so request
 	 * them every half ring, or where NS_REPORT is set
 	 */
 	u_int report_frequency = kring->nkr_num_slots >> 1;
 	/* device-specific */
 	if_ctx_t ctx = ifp->if_softc;
 	iflib_txq_t txq = &ctx->ifc_txqs[kring->ring_id];
 
 	bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,
 	    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 
 	/*
 	 * First part: process new packets to send.
 	 * nm_i is the current index in the netmap kring,
 	 * nic_i is the corresponding index in the NIC ring.
 	 *
 	 * If we have packets to send (nm_i != head)
 	 * iterate over the netmap ring, fetch length and update
 	 * the corresponding slot in the NIC ring. Some drivers also
 	 * need to update the buffer's physical address in the NIC slot
 	 * even NS_BUF_CHANGED is not set (PNMB computes the addresses).
 	 *
 	 * The netmap_reload_map() calls is especially expensive,
 	 * even when (as in this case) the tag is 0, so do only
 	 * when the buffer has actually changed.
 	 *
 	 * If possible do not set the report/intr bit on all slots,
 	 * but only a few times per ring or when NS_REPORT is set.
 	 *
 	 * Finally, on 10G and faster drivers, it might be useful
 	 * to prefetch the next slot and txr entry.
 	 */
 
 	nm_i = kring->nr_hwcur;
 	if (nm_i != head) {	/* we have new packets to send */
 		pkt_info_zero(&pi);
 		pi.ipi_segs = txq->ift_segs;
 		pi.ipi_qsidx = kring->ring_id;
 		nic_i = netmap_idx_k2n(kring, nm_i);
 
 		__builtin_prefetch(&ring->slot[nm_i]);
 		__builtin_prefetch(&txq->ift_sds.ifsd_m[nic_i]);
 		__builtin_prefetch(&txq->ift_sds.ifsd_map[nic_i]);
 
 		for (n = 0; nm_i != head; n++) {
 			struct netmap_slot *slot = &ring->slot[nm_i];
 			u_int len = slot->len;
 			uint64_t paddr;
 			void *addr = PNMB(na, slot, &paddr);
 			int flags = (slot->flags & NS_REPORT ||
 				nic_i == 0 || nic_i == report_frequency) ?
 				IPI_TX_INTR : 0;
 
 			/* device-specific */
 			pi.ipi_len = len;
 			pi.ipi_segs[0].ds_addr = paddr;
 			pi.ipi_segs[0].ds_len = len;
 			pi.ipi_nsegs = 1;
 			pi.ipi_ndescs = 0;
 			pi.ipi_pidx = nic_i;
 			pi.ipi_flags = flags;
 
 			/* Fill the slot in the NIC ring. */
 			ctx->isc_txd_encap(ctx->ifc_softc, &pi);
 			DBG_COUNTER_INC(tx_encap);
 
 			/* prefetch for next round */
 			__builtin_prefetch(&ring->slot[nm_i + 1]);
 			__builtin_prefetch(&txq->ift_sds.ifsd_m[nic_i + 1]);
 			__builtin_prefetch(&txq->ift_sds.ifsd_map[nic_i + 1]);
 
 			NM_CHECK_ADDR_LEN(na, addr, len);
 
 			if (slot->flags & NS_BUF_CHANGED) {
 				/* buffer has changed, reload map */
 				netmap_reload_map(na, txq->ift_buf_tag,
 				    txq->ift_sds.ifsd_map[nic_i], addr);
 			}
 			/* make sure changes to the buffer are synced */
 			bus_dmamap_sync(txq->ift_buf_tag,
 			    txq->ift_sds.ifsd_map[nic_i],
 			    BUS_DMASYNC_PREWRITE);
 
 			slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED);
 			nm_i = nm_next(nm_i, lim);
 			nic_i = nm_next(nic_i, lim);
 		}
 		kring->nr_hwcur = nm_i;
 
 		/* synchronize the NIC ring */
 		bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,
 		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 
 		/* (re)start the tx unit up to slot nic_i (excluded) */
 		ctx->isc_txd_flush(ctx->ifc_softc, txq->ift_id, nic_i);
 	}
 
 	/*
 	 * Second part: reclaim buffers for completed transmissions.
 	 *
 	 * If there are unclaimed buffers, attempt to reclaim them.
 	 * If none are reclaimed, and TX IRQs are not in use, do an initial
 	 * minimal delay, then trigger the tx handler which will spin in the
 	 * group task queue.
 	 */
 	if (kring->nr_hwtail != nm_prev(kring->nr_hwcur, lim)) {
 		if (iflib_tx_credits_update(ctx, txq)) {
 			/* some tx completed, increment avail */
 			nic_i = txq->ift_cidx_processed;
 			kring->nr_hwtail = nm_prev(netmap_idx_n2k(kring, nic_i), lim);
 		}
 	}
 	if (!(ctx->ifc_flags & IFC_NETMAP_TX_IRQ))
 		if (kring->nr_hwtail != nm_prev(kring->nr_hwcur, lim)) {
 			callout_reset_on(&txq->ift_timer, hz < 2000 ? 1 : hz / 1000,
 			    iflib_timer, txq, txq->ift_timer.c_cpu);
 	}
 	return (0);
 }
 
 /*
  * Reconcile kernel and user view of the receive ring.
  * Same as for the txsync, this routine must be efficient.
  * The caller guarantees a single invocations, but races against
  * the rest of the driver should be handled here.
  *
  * On call, kring->rhead is the first packet that userspace wants
  * to keep, and kring->rcur is the wakeup point.
  * The kernel has previously reported packets up to kring->rtail.
  *
  * If (flags & NAF_FORCE_READ) also check for incoming packets irrespective
  * of whether or not we received an interrupt.
  */
 static int
 iflib_netmap_rxsync(struct netmap_kring *kring, int flags)
 {
 	struct netmap_adapter *na = kring->na;
 	struct netmap_ring *ring = kring->ring;
 	iflib_fl_t fl;
 	uint32_t nm_i;	/* index into the netmap ring */
 	uint32_t nic_i;	/* index into the NIC ring */
 	u_int i, n;
 	u_int const lim = kring->nkr_num_slots - 1;
 	u_int const head = kring->rhead;
 	int force_update = (flags & NAF_FORCE_READ) || kring->nr_kflags & NKR_PENDINTR;
 	struct if_rxd_info ri;
 
 	struct ifnet *ifp = na->ifp;
 	if_ctx_t ctx = ifp->if_softc;
 	iflib_rxq_t rxq = &ctx->ifc_rxqs[kring->ring_id];
 	if (head > lim)
 		return netmap_ring_reinit(kring);
 
 	/*
 	 * XXX netmap_fl_refill() only ever (re)fills free list 0 so far.
 	 */
 
 	for (i = 0, fl = rxq->ifr_fl; i < rxq->ifr_nfl; i++, fl++) {
 		bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map,
 		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 	}
 
 	/*
 	 * First part: import newly received packets.
 	 *
 	 * nm_i is the index of the next free slot in the netmap ring,
 	 * nic_i is the index of the next received packet in the NIC ring,
 	 * and they may differ in case if_init() has been called while
 	 * in netmap mode. For the receive ring we have
 	 *
 	 *	nic_i = rxr->next_check;
 	 *	nm_i = kring->nr_hwtail (previous)
 	 * and
 	 *	nm_i == (nic_i + kring->nkr_hwofs) % ring_size
 	 *
 	 * rxr->next_check is set to 0 on a ring reinit
 	 */
 	if (netmap_no_pendintr || force_update) {
 		int crclen = iflib_crcstrip ? 0 : 4;
 		int error, avail;
 
 		for (i = 0; i < rxq->ifr_nfl; i++) {
 			fl = &rxq->ifr_fl[i];
 			nic_i = fl->ifl_cidx;
 			nm_i = netmap_idx_n2k(kring, nic_i);
 			avail = ctx->isc_rxd_available(ctx->ifc_softc,
 			    rxq->ifr_id, nic_i, USHRT_MAX);
 			for (n = 0; avail > 0; n++, avail--) {
 				rxd_info_zero(&ri);
 				ri.iri_frags = rxq->ifr_frags;
 				ri.iri_qsidx = kring->ring_id;
 				ri.iri_ifp = ctx->ifc_ifp;
 				ri.iri_cidx = nic_i;
 
 				error = ctx->isc_rxd_pkt_get(ctx->ifc_softc, &ri);
 				ring->slot[nm_i].len = error ? 0 : ri.iri_len - crclen;
 				ring->slot[nm_i].flags = 0;
 				bus_dmamap_sync(fl->ifl_buf_tag,
 				    fl->ifl_sds.ifsd_map[nic_i], BUS_DMASYNC_POSTREAD);
 				nm_i = nm_next(nm_i, lim);
 				nic_i = nm_next(nic_i, lim);
 			}
 			if (n) { /* update the state variables */
 				if (netmap_no_pendintr && !force_update) {
 					/* diagnostics */
 					iflib_rx_miss ++;
 					iflib_rx_miss_bufs += n;
 				}
 				fl->ifl_cidx = nic_i;
 				kring->nr_hwtail = nm_i;
 			}
 			kring->nr_kflags &= ~NKR_PENDINTR;
 		}
 	}
 	/*
 	 * Second part: skip past packets that userspace has released.
 	 * (kring->nr_hwcur to head excluded),
 	 * and make the buffers available for reception.
 	 * As usual nm_i is the index in the netmap ring,
 	 * nic_i is the index in the NIC ring, and
 	 * nm_i == (nic_i + kring->nkr_hwofs) % ring_size
 	 */
 	/* XXX not sure how this will work with multiple free lists */
 	nm_i = kring->nr_hwcur;
 
 	return (netmap_fl_refill(rxq, kring, nm_i, false));
 }
 
 static void
 iflib_netmap_intr(struct netmap_adapter *na, int onoff)
 {
 	struct ifnet *ifp = na->ifp;
 	if_ctx_t ctx = ifp->if_softc;
 
 	CTX_LOCK(ctx);
 	if (onoff) {
 		IFDI_INTR_ENABLE(ctx);
 	} else {
 		IFDI_INTR_DISABLE(ctx);
 	}
 	CTX_UNLOCK(ctx);
 }
 
 
 static int
 iflib_netmap_attach(if_ctx_t ctx)
 {
 	struct netmap_adapter na;
 	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
 
 	bzero(&na, sizeof(na));
 
 	na.ifp = ctx->ifc_ifp;
 	na.na_flags = NAF_BDG_MAYSLEEP;
 	MPASS(ctx->ifc_softc_ctx.isc_ntxqsets);
 	MPASS(ctx->ifc_softc_ctx.isc_nrxqsets);
 
 	na.num_tx_desc = scctx->isc_ntxd[0];
 	na.num_rx_desc = scctx->isc_nrxd[0];
 	na.nm_txsync = iflib_netmap_txsync;
 	na.nm_rxsync = iflib_netmap_rxsync;
 	na.nm_register = iflib_netmap_register;
 	na.nm_intr = iflib_netmap_intr;
 	na.num_tx_rings = ctx->ifc_softc_ctx.isc_ntxqsets;
 	na.num_rx_rings = ctx->ifc_softc_ctx.isc_nrxqsets;
 	return (netmap_attach(&na));
 }
 
 static void
 iflib_netmap_txq_init(if_ctx_t ctx, iflib_txq_t txq)
 {
 	struct netmap_adapter *na = NA(ctx->ifc_ifp);
 	struct netmap_slot *slot;
 
 	slot = netmap_reset(na, NR_TX, txq->ift_id, 0);
 	if (slot == NULL)
 		return;
 	for (int i = 0; i < ctx->ifc_softc_ctx.isc_ntxd[0]; i++) {
 
 		/*
 		 * In netmap mode, set the map for the packet buffer.
 		 * NOTE: Some drivers (not this one) also need to set
 		 * the physical buffer address in the NIC ring.
 		 * netmap_idx_n2k() maps a nic index, i, into the corresponding
 		 * netmap slot index, si
 		 */
 		int si = netmap_idx_n2k(na->tx_rings[txq->ift_id], i);
 		netmap_load_map(na, txq->ift_buf_tag, txq->ift_sds.ifsd_map[i],
 		    NMB(na, slot + si));
 	}
 }
 
 static void
 iflib_netmap_rxq_init(if_ctx_t ctx, iflib_rxq_t rxq)
 {
 	struct netmap_adapter *na = NA(ctx->ifc_ifp);
 	struct netmap_kring *kring = na->rx_rings[rxq->ifr_id];
 	struct netmap_slot *slot;
 	uint32_t nm_i;
 
 	slot = netmap_reset(na, NR_RX, rxq->ifr_id, 0);
 	if (slot == NULL)
 		return;
 	nm_i = netmap_idx_n2k(kring, 0);
 	netmap_fl_refill(rxq, kring, nm_i, true);
 }
 
 static void
 iflib_netmap_timer_adjust(if_ctx_t ctx, iflib_txq_t txq, uint32_t *reset_on)
 {
 	struct netmap_kring *kring;
 	uint16_t txqid;
 
 	txqid = txq->ift_id;
 	kring = NA(ctx->ifc_ifp)->tx_rings[txqid];
 
 	if (kring->nr_hwcur != nm_next(kring->nr_hwtail, kring->nkr_num_slots - 1)) {
 		bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,
 		    BUS_DMASYNC_POSTREAD);
 		if (ctx->isc_txd_credits_update(ctx->ifc_softc, txqid, false))
 			netmap_tx_irq(ctx->ifc_ifp, txqid);
 		if (!(ctx->ifc_flags & IFC_NETMAP_TX_IRQ)) {
 			if (hz < 2000)
 				*reset_on = 1;
 			else
 				*reset_on = hz / 1000;
 		}
 	}
 }
 
 #define iflib_netmap_detach(ifp) netmap_detach(ifp)
 
 #else
 #define iflib_netmap_txq_init(ctx, txq)
 #define iflib_netmap_rxq_init(ctx, rxq)
 #define iflib_netmap_detach(ifp)
 
 #define iflib_netmap_attach(ctx) (0)
 #define netmap_rx_irq(ifp, qid, budget) (0)
 #define netmap_tx_irq(ifp, qid) do {} while (0)
 #define iflib_netmap_timer_adjust(ctx, txq, reset_on)
 
 #endif
 
 #if defined(__i386__) || defined(__amd64__)
 static __inline void
 prefetch(void *x)
 {
 	__asm volatile("prefetcht0 %0" :: "m" (*(unsigned long *)x));
 }
 static __inline void
 prefetch2cachelines(void *x)
 {
 	__asm volatile("prefetcht0 %0" :: "m" (*(unsigned long *)x));
 #if (CACHE_LINE_SIZE < 128)
 	__asm volatile("prefetcht0 %0" :: "m" (*(((unsigned long *)x)+CACHE_LINE_SIZE/(sizeof(unsigned long)))));
 #endif
 }
 #else
 #define prefetch(x)
 #define prefetch2cachelines(x)
 #endif
 
 static void
 iflib_gen_mac(if_ctx_t ctx)
 {
 	struct thread *td;
 	MD5_CTX mdctx;
 	char uuid[HOSTUUIDLEN+1];
 	char buf[HOSTUUIDLEN+16];
 	uint8_t *mac;
 	unsigned char digest[16];
 
 	td = curthread;
 	mac = ctx->ifc_mac;
 	uuid[HOSTUUIDLEN] = 0;
 	bcopy(td->td_ucred->cr_prison->pr_hostuuid, uuid, HOSTUUIDLEN);
 	snprintf(buf, HOSTUUIDLEN+16, "%s-%s", uuid, device_get_nameunit(ctx->ifc_dev));
 	/*
 	 * Generate a pseudo-random, deterministic MAC
 	 * address based on the UUID and unit number.
 	 * The FreeBSD Foundation OUI of 58-9C-FC is used.
 	 */
 	MD5Init(&mdctx);
 	MD5Update(&mdctx, buf, strlen(buf));
 	MD5Final(digest, &mdctx);
 
 	mac[0] = 0x58;
 	mac[1] = 0x9C;
 	mac[2] = 0xFC;
 	mac[3] = digest[0];
 	mac[4] = digest[1];
 	mac[5] = digest[2];
 }
 
 static void
 iru_init(if_rxd_update_t iru, iflib_rxq_t rxq, uint8_t flid)
 {
 	iflib_fl_t fl;
 
 	fl = &rxq->ifr_fl[flid];
 	iru->iru_paddrs = fl->ifl_bus_addrs;
 	iru->iru_vaddrs = &fl->ifl_vm_addrs[0];
 	iru->iru_idxs = fl->ifl_rxd_idxs;
 	iru->iru_qsidx = rxq->ifr_id;
 	iru->iru_buf_size = fl->ifl_buf_size;
 	iru->iru_flidx = fl->ifl_id;
 }
 
 static void
 _iflib_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int err)
 {
 	if (err)
 		return;
 	*(bus_addr_t *) arg = segs[0].ds_addr;
 }
 
 int
 iflib_dma_alloc_align(if_ctx_t ctx, int size, int align, iflib_dma_info_t dma, int mapflags)
 {
 	int err;
 	device_t dev = ctx->ifc_dev;
 
 	err = bus_dma_tag_create(bus_get_dma_tag(dev),	/* parent */
 				align, 0,		/* alignment, bounds */
 				BUS_SPACE_MAXADDR,	/* lowaddr */
 				BUS_SPACE_MAXADDR,	/* highaddr */
 				NULL, NULL,		/* filter, filterarg */
 				size,			/* maxsize */
 				1,			/* nsegments */
 				size,			/* maxsegsize */
 				BUS_DMA_ALLOCNOW,	/* flags */
 				NULL,			/* lockfunc */
 				NULL,			/* lockarg */
 				&dma->idi_tag);
 	if (err) {
 		device_printf(dev,
 		    "%s: bus_dma_tag_create failed: %d\n",
 		    __func__, err);
 		goto fail_0;
 	}
 
 	err = bus_dmamem_alloc(dma->idi_tag, (void**) &dma->idi_vaddr,
 	    BUS_DMA_NOWAIT | BUS_DMA_COHERENT | BUS_DMA_ZERO, &dma->idi_map);
 	if (err) {
 		device_printf(dev,
 		    "%s: bus_dmamem_alloc(%ju) failed: %d\n",
 		    __func__, (uintmax_t)size, err);
 		goto fail_1;
 	}
 
 	dma->idi_paddr = IF_BAD_DMA;
 	err = bus_dmamap_load(dma->idi_tag, dma->idi_map, dma->idi_vaddr,
 	    size, _iflib_dmamap_cb, &dma->idi_paddr, mapflags | BUS_DMA_NOWAIT);
 	if (err || dma->idi_paddr == IF_BAD_DMA) {
 		device_printf(dev,
 		    "%s: bus_dmamap_load failed: %d\n",
 		    __func__, err);
 		goto fail_2;
 	}
 
 	dma->idi_size = size;
 	return (0);
 
 fail_2:
 	bus_dmamem_free(dma->idi_tag, dma->idi_vaddr, dma->idi_map);
 fail_1:
 	bus_dma_tag_destroy(dma->idi_tag);
 fail_0:
 	dma->idi_tag = NULL;
 
 	return (err);
 }
 
 int
 iflib_dma_alloc(if_ctx_t ctx, int size, iflib_dma_info_t dma, int mapflags)
 {
 	if_shared_ctx_t sctx = ctx->ifc_sctx;
 
 	KASSERT(sctx->isc_q_align != 0, ("alignment value not initialized"));
 
 	return (iflib_dma_alloc_align(ctx, size, sctx->isc_q_align, dma, mapflags));
 }
 
 int
 iflib_dma_alloc_multi(if_ctx_t ctx, int *sizes, iflib_dma_info_t *dmalist, int mapflags, int count)
 {
 	int i, err;
 	iflib_dma_info_t *dmaiter;
 
 	dmaiter = dmalist;
 	for (i = 0; i < count; i++, dmaiter++) {
 		if ((err = iflib_dma_alloc(ctx, sizes[i], *dmaiter, mapflags)) != 0)
 			break;
 	}
 	if (err)
 		iflib_dma_free_multi(dmalist, i);
 	return (err);
 }
 
 void
 iflib_dma_free(iflib_dma_info_t dma)
 {
 	if (dma->idi_tag == NULL)
 		return;
 	if (dma->idi_paddr != IF_BAD_DMA) {
 		bus_dmamap_sync(dma->idi_tag, dma->idi_map,
 		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 		bus_dmamap_unload(dma->idi_tag, dma->idi_map);
 		dma->idi_paddr = IF_BAD_DMA;
 	}
 	if (dma->idi_vaddr != NULL) {
 		bus_dmamem_free(dma->idi_tag, dma->idi_vaddr, dma->idi_map);
 		dma->idi_vaddr = NULL;
 	}
 	bus_dma_tag_destroy(dma->idi_tag);
 	dma->idi_tag = NULL;
 }
 
 void
 iflib_dma_free_multi(iflib_dma_info_t *dmalist, int count)
 {
 	int i;
 	iflib_dma_info_t *dmaiter = dmalist;
 
 	for (i = 0; i < count; i++, dmaiter++)
 		iflib_dma_free(*dmaiter);
 }
 
 #ifdef EARLY_AP_STARTUP
 static const int iflib_started = 1;
 #else
 /*
  * We used to abuse the smp_started flag to decide if the queues have been
  * fully initialized (by late taskqgroup_adjust() calls in a SYSINIT()).
  * That gave bad races, since the SYSINIT() runs strictly after smp_started
  * is set.  Run a SYSINIT() strictly after that to just set a usable
  * completion flag.
  */
 
 static int iflib_started;
 
 static void
 iflib_record_started(void *arg)
 {
 	iflib_started = 1;
 }
 
 SYSINIT(iflib_record_started, SI_SUB_SMP + 1, SI_ORDER_FIRST,
 	iflib_record_started, NULL);
 #endif
 
 static int
 iflib_fast_intr(void *arg)
 {
 	iflib_filter_info_t info = arg;
 	struct grouptask *gtask = info->ifi_task;
 	int result;
 
 	if (!iflib_started)
 		return (FILTER_STRAY);
 
 	DBG_COUNTER_INC(fast_intrs);
 	if (info->ifi_filter != NULL) {
 		result = info->ifi_filter(info->ifi_filter_arg);
 		if ((result & FILTER_SCHEDULE_THREAD) == 0)
 			return (result);
 	}
 
 	GROUPTASK_ENQUEUE(gtask);
 	return (FILTER_HANDLED);
 }
 
 static int
 iflib_fast_intr_rxtx(void *arg)
 {
 	iflib_filter_info_t info = arg;
 	struct grouptask *gtask = info->ifi_task;
 	if_ctx_t ctx;
 	iflib_rxq_t rxq = (iflib_rxq_t)info->ifi_ctx;
 	iflib_txq_t txq;
 	void *sc;
 	int i, cidx, result;
 	qidx_t txqid;
 
 	if (!iflib_started)
 		return (FILTER_STRAY);
 
 	DBG_COUNTER_INC(fast_intrs);
 	if (info->ifi_filter != NULL) {
 		result = info->ifi_filter(info->ifi_filter_arg);
 		if ((result & FILTER_SCHEDULE_THREAD) == 0)
 			return (result);
 	}
 
 	ctx = rxq->ifr_ctx;
 	sc = ctx->ifc_softc;
 	MPASS(rxq->ifr_ntxqirq);
 	for (i = 0; i < rxq->ifr_ntxqirq; i++) {
 		txqid = rxq->ifr_txqid[i];
 		txq = &ctx->ifc_txqs[txqid];
 		bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,
 		    BUS_DMASYNC_POSTREAD);
 		if (!ctx->isc_txd_credits_update(sc, txqid, false)) {
 			IFDI_TX_QUEUE_INTR_ENABLE(ctx, txqid);
 			continue;
 		}
 		GROUPTASK_ENQUEUE(&txq->ift_task);
 	}
 	if (ctx->ifc_sctx->isc_flags & IFLIB_HAS_RXCQ)
 		cidx = rxq->ifr_cq_cidx;
 	else
 		cidx = rxq->ifr_fl[0].ifl_cidx;
 	if (iflib_rxd_avail(ctx, rxq, cidx, 1))
 		GROUPTASK_ENQUEUE(gtask);
 	else {
 		IFDI_RX_QUEUE_INTR_ENABLE(ctx, rxq->ifr_id);
 		DBG_COUNTER_INC(rx_intr_enables);
 	}
 	return (FILTER_HANDLED);
 }
 
 
 static int
 iflib_fast_intr_ctx(void *arg)
 {
 	iflib_filter_info_t info = arg;
 	struct grouptask *gtask = info->ifi_task;
 	int result;
 
 	if (!iflib_started)
 		return (FILTER_STRAY);
 
 	DBG_COUNTER_INC(fast_intrs);
 	if (info->ifi_filter != NULL) {
 		result = info->ifi_filter(info->ifi_filter_arg);
 		if ((result & FILTER_SCHEDULE_THREAD) == 0)
 			return (result);
 	}
 
 	GROUPTASK_ENQUEUE(gtask);
 	return (FILTER_HANDLED);
 }
 
 static int
 _iflib_irq_alloc(if_ctx_t ctx, if_irq_t irq, int rid,
 		 driver_filter_t filter, driver_intr_t handler, void *arg,
 		 const char *name)
 {
 	int rc, flags;
 	struct resource *res;
 	void *tag = NULL;
 	device_t dev = ctx->ifc_dev;
 
 	flags = RF_ACTIVE;
 	if (ctx->ifc_flags & IFC_LEGACY)
 		flags |= RF_SHAREABLE;
 	MPASS(rid < 512);
 	irq->ii_rid = rid;
 	res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &irq->ii_rid, flags);
 	if (res == NULL) {
 		device_printf(dev,
 		    "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
 		return (ENOMEM);
 	}
 	irq->ii_res = res;
 	KASSERT(filter == NULL || handler == NULL, ("filter and handler can't both be non-NULL"));
 	rc = bus_setup_intr(dev, res, INTR_MPSAFE | INTR_TYPE_NET,
 						filter, handler, arg, &tag);
 	if (rc != 0) {
 		device_printf(dev,
 		    "failed to setup interrupt for rid %d, name %s: %d\n",
 					  rid, name ? name : "unknown", rc);
 		return (rc);
 	} else if (name)
 		bus_describe_intr(dev, res, tag, "%s", name);
 
 	irq->ii_tag = tag;
 	return (0);
 }
 
 
 /*********************************************************************
  *
  *  Allocate DMA resources for TX buffers as well as memory for the TX
  *  mbuf map.  TX DMA maps (non-TSO/TSO) and TX mbuf map are kept in a
  *  iflib_sw_tx_desc_array structure, storing all the information that
  *  is needed to transmit a packet on the wire.  This is called only
  *  once at attach, setup is done every reset.
  *
  **********************************************************************/
 static int
 iflib_txsd_alloc(iflib_txq_t txq)
 {
 	if_ctx_t ctx = txq->ift_ctx;
 	if_shared_ctx_t sctx = ctx->ifc_sctx;
 	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
 	device_t dev = ctx->ifc_dev;
 	bus_size_t tsomaxsize;
 	int err, nsegments, ntsosegments;
 	bool tso;
 
 	nsegments = scctx->isc_tx_nsegments;
 	ntsosegments = scctx->isc_tx_tso_segments_max;
 	tsomaxsize = scctx->isc_tx_tso_size_max;
 	if (if_getcapabilities(ctx->ifc_ifp) & IFCAP_VLAN_MTU)
 		tsomaxsize += sizeof(struct ether_vlan_header);
 	MPASS(scctx->isc_ntxd[0] > 0);
 	MPASS(scctx->isc_ntxd[txq->ift_br_offset] > 0);
 	MPASS(nsegments > 0);
 	if (if_getcapabilities(ctx->ifc_ifp) & IFCAP_TSO) {
 		MPASS(ntsosegments > 0);
 		MPASS(sctx->isc_tso_maxsize >= tsomaxsize);
 	}
 
 	/*
 	 * Set up DMA tags for TX buffers.
 	 */
 	if ((err = bus_dma_tag_create(bus_get_dma_tag(dev),
 			       1, 0,			/* alignment, bounds */
 			       BUS_SPACE_MAXADDR,	/* lowaddr */
 			       BUS_SPACE_MAXADDR,	/* highaddr */
 			       NULL, NULL,		/* filter, filterarg */
 			       sctx->isc_tx_maxsize,		/* maxsize */
 			       nsegments,	/* nsegments */
 			       sctx->isc_tx_maxsegsize,	/* maxsegsize */
 			       0,			/* flags */
 			       NULL,			/* lockfunc */
 			       NULL,			/* lockfuncarg */
 			       &txq->ift_buf_tag))) {
 		device_printf(dev,"Unable to allocate TX DMA tag: %d\n", err);
 		device_printf(dev,"maxsize: %ju nsegments: %d maxsegsize: %ju\n",
 		    (uintmax_t)sctx->isc_tx_maxsize, nsegments, (uintmax_t)sctx->isc_tx_maxsegsize);
 		goto fail;
 	}
 	tso = (if_getcapabilities(ctx->ifc_ifp) & IFCAP_TSO) != 0;
 	if (tso && (err = bus_dma_tag_create(bus_get_dma_tag(dev),
 			       1, 0,			/* alignment, bounds */
 			       BUS_SPACE_MAXADDR,	/* lowaddr */
 			       BUS_SPACE_MAXADDR,	/* highaddr */
 			       NULL, NULL,		/* filter, filterarg */
 			       tsomaxsize,		/* maxsize */
 			       ntsosegments,	/* nsegments */
 			       sctx->isc_tso_maxsegsize,/* maxsegsize */
 			       0,			/* flags */
 			       NULL,			/* lockfunc */
 			       NULL,			/* lockfuncarg */
 			       &txq->ift_tso_buf_tag))) {
 		device_printf(dev, "Unable to allocate TSO TX DMA tag: %d\n",
 		    err);
 		goto fail;
 	}
 
 	/* Allocate memory for the TX mbuf map. */
 	if (!(txq->ift_sds.ifsd_m =
 	    (struct mbuf **) malloc(sizeof(struct mbuf *) *
 	    scctx->isc_ntxd[txq->ift_br_offset], M_IFLIB, M_NOWAIT | M_ZERO))) {
 		device_printf(dev, "Unable to allocate TX mbuf map memory\n");
 		err = ENOMEM;
 		goto fail;
 	}
 
 	/*
 	 * Create the DMA maps for TX buffers.
 	 */
 	if ((txq->ift_sds.ifsd_map = (bus_dmamap_t *)malloc(
 	    sizeof(bus_dmamap_t) * scctx->isc_ntxd[txq->ift_br_offset],
 	    M_IFLIB, M_NOWAIT | M_ZERO)) == NULL) {
 		device_printf(dev,
 		    "Unable to allocate TX buffer DMA map memory\n");
 		err = ENOMEM;
 		goto fail;
 	}
 	if (tso && (txq->ift_sds.ifsd_tso_map = (bus_dmamap_t *)malloc(
 	    sizeof(bus_dmamap_t) * scctx->isc_ntxd[txq->ift_br_offset],
 	    M_IFLIB, M_NOWAIT | M_ZERO)) == NULL) {
 		device_printf(dev,
 		    "Unable to allocate TSO TX buffer map memory\n");
 		err = ENOMEM;
 		goto fail;
 	}
 	for (int i = 0; i < scctx->isc_ntxd[txq->ift_br_offset]; i++) {
 		err = bus_dmamap_create(txq->ift_buf_tag, 0,
 		    &txq->ift_sds.ifsd_map[i]);
 		if (err != 0) {
 			device_printf(dev, "Unable to create TX DMA map\n");
 			goto fail;
 		}
 		if (!tso)
 			continue;
 		err = bus_dmamap_create(txq->ift_tso_buf_tag, 0,
 		    &txq->ift_sds.ifsd_tso_map[i]);
 		if (err != 0) {
 			device_printf(dev, "Unable to create TSO TX DMA map\n");
 			goto fail;
 		}
 	}
 	return (0);
 fail:
 	/* We free all, it handles case where we are in the middle */
 	iflib_tx_structures_free(ctx);
 	return (err);
 }
 
 static void
 iflib_txsd_destroy(if_ctx_t ctx, iflib_txq_t txq, int i)
 {
 	bus_dmamap_t map;
 
 	map = NULL;
 	if (txq->ift_sds.ifsd_map != NULL)
 		map = txq->ift_sds.ifsd_map[i];
 	if (map != NULL) {
 		bus_dmamap_sync(txq->ift_buf_tag, map, BUS_DMASYNC_POSTWRITE);
 		bus_dmamap_unload(txq->ift_buf_tag, map);
 		bus_dmamap_destroy(txq->ift_buf_tag, map);
 		txq->ift_sds.ifsd_map[i] = NULL;
 	}
 
 	map = NULL;
 	if (txq->ift_sds.ifsd_tso_map != NULL)
 		map = txq->ift_sds.ifsd_tso_map[i];
 	if (map != NULL) {
 		bus_dmamap_sync(txq->ift_tso_buf_tag, map,
 		    BUS_DMASYNC_POSTWRITE);
 		bus_dmamap_unload(txq->ift_tso_buf_tag, map);
 		bus_dmamap_destroy(txq->ift_tso_buf_tag, map);
 		txq->ift_sds.ifsd_tso_map[i] = NULL;
 	}
 }
 
 static void
 iflib_txq_destroy(iflib_txq_t txq)
 {
 	if_ctx_t ctx = txq->ift_ctx;
 
 	for (int i = 0; i < txq->ift_size; i++)
 		iflib_txsd_destroy(ctx, txq, i);
 	if (txq->ift_sds.ifsd_map != NULL) {
 		free(txq->ift_sds.ifsd_map, M_IFLIB);
 		txq->ift_sds.ifsd_map = NULL;
 	}
 	if (txq->ift_sds.ifsd_tso_map != NULL) {
 		free(txq->ift_sds.ifsd_tso_map, M_IFLIB);
 		txq->ift_sds.ifsd_tso_map = NULL;
 	}
 	if (txq->ift_sds.ifsd_m != NULL) {
 		free(txq->ift_sds.ifsd_m, M_IFLIB);
 		txq->ift_sds.ifsd_m = NULL;
 	}
 	if (txq->ift_buf_tag != NULL) {
 		bus_dma_tag_destroy(txq->ift_buf_tag);
 		txq->ift_buf_tag = NULL;
 	}
 	if (txq->ift_tso_buf_tag != NULL) {
 		bus_dma_tag_destroy(txq->ift_tso_buf_tag);
 		txq->ift_tso_buf_tag = NULL;
 	}
 }
 
 static void
 iflib_txsd_free(if_ctx_t ctx, iflib_txq_t txq, int i)
 {
 	struct mbuf **mp;
 
 	mp = &txq->ift_sds.ifsd_m[i];
 	if (*mp == NULL)
 		return;
 
 	if (txq->ift_sds.ifsd_map != NULL) {
 		bus_dmamap_sync(txq->ift_buf_tag,
 		    txq->ift_sds.ifsd_map[i], BUS_DMASYNC_POSTWRITE);
 		bus_dmamap_unload(txq->ift_buf_tag, txq->ift_sds.ifsd_map[i]);
 	}
 	if (txq->ift_sds.ifsd_tso_map != NULL) {
 		bus_dmamap_sync(txq->ift_tso_buf_tag,
 		    txq->ift_sds.ifsd_tso_map[i], BUS_DMASYNC_POSTWRITE);
 		bus_dmamap_unload(txq->ift_tso_buf_tag,
 		    txq->ift_sds.ifsd_tso_map[i]);
 	}
 	m_free(*mp);
 	DBG_COUNTER_INC(tx_frees);
 	*mp = NULL;
 }
 
 static int
 iflib_txq_setup(iflib_txq_t txq)
 {
 	if_ctx_t ctx = txq->ift_ctx;
 	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
 	if_shared_ctx_t sctx = ctx->ifc_sctx;
 	iflib_dma_info_t di;
 	int i;
 
 	/* Set number of descriptors available */
 	txq->ift_qstatus = IFLIB_QUEUE_IDLE;
 	/* XXX make configurable */
 	txq->ift_update_freq = IFLIB_DEFAULT_TX_UPDATE_FREQ;
 
 	/* Reset indices */
 	txq->ift_cidx_processed = 0;
 	txq->ift_pidx = txq->ift_cidx = txq->ift_npending = 0;
 	txq->ift_size = scctx->isc_ntxd[txq->ift_br_offset];
 
 	for (i = 0, di = txq->ift_ifdi; i < sctx->isc_ntxqs; i++, di++)
 		bzero((void *)di->idi_vaddr, di->idi_size);
 
 	IFDI_TXQ_SETUP(ctx, txq->ift_id);
 	for (i = 0, di = txq->ift_ifdi; i < sctx->isc_ntxqs; i++, di++)
 		bus_dmamap_sync(di->idi_tag, di->idi_map,
 		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 	return (0);
 }
 
 /*********************************************************************
  *
  *  Allocate DMA resources for RX buffers as well as memory for the RX
  *  mbuf map, direct RX cluster pointer map and RX cluster bus address
  *  map.  RX DMA map, RX mbuf map, direct RX cluster pointer map and
  *  RX cluster map are kept in a iflib_sw_rx_desc_array structure.
  *  Since we use use one entry in iflib_sw_rx_desc_array per received
  *  packet, the maximum number of entries we'll need is equal to the
  *  number of hardware receive descriptors that we've allocated.
  *
  **********************************************************************/
 static int
 iflib_rxsd_alloc(iflib_rxq_t rxq)
 {
 	if_ctx_t ctx = rxq->ifr_ctx;
 	if_shared_ctx_t sctx = ctx->ifc_sctx;
 	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
 	device_t dev = ctx->ifc_dev;
 	iflib_fl_t fl;
 	int			err;
 
 	MPASS(scctx->isc_nrxd[0] > 0);
 	MPASS(scctx->isc_nrxd[rxq->ifr_fl_offset] > 0);
 
 	fl = rxq->ifr_fl;
 	for (int i = 0; i <  rxq->ifr_nfl; i++, fl++) {
 		fl->ifl_size = scctx->isc_nrxd[rxq->ifr_fl_offset]; /* this isn't necessarily the same */
 		/* Set up DMA tag for RX buffers. */
 		err = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
 					 1, 0,			/* alignment, bounds */
 					 BUS_SPACE_MAXADDR,	/* lowaddr */
 					 BUS_SPACE_MAXADDR,	/* highaddr */
 					 NULL, NULL,		/* filter, filterarg */
 					 sctx->isc_rx_maxsize,	/* maxsize */
 					 sctx->isc_rx_nsegments,	/* nsegments */
 					 sctx->isc_rx_maxsegsize,	/* maxsegsize */
 					 0,			/* flags */
 					 NULL,			/* lockfunc */
 					 NULL,			/* lockarg */
 					 &fl->ifl_buf_tag);
 		if (err) {
 			device_printf(dev,
 			    "Unable to allocate RX DMA tag: %d\n", err);
 			goto fail;
 		}
 
 		/* Allocate memory for the RX mbuf map. */
 		if (!(fl->ifl_sds.ifsd_m =
 		      (struct mbuf **) malloc(sizeof(struct mbuf *) *
 					      scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT | M_ZERO))) {
 			device_printf(dev,
 			    "Unable to allocate RX mbuf map memory\n");
 			err = ENOMEM;
 			goto fail;
 		}
 
 		/* Allocate memory for the direct RX cluster pointer map. */
 		if (!(fl->ifl_sds.ifsd_cl =
 		      (caddr_t *) malloc(sizeof(caddr_t) *
 					      scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT | M_ZERO))) {
 			device_printf(dev,
 			    "Unable to allocate RX cluster map memory\n");
 			err = ENOMEM;
 			goto fail;
 		}
 
 		/* Allocate memory for the RX cluster bus address map. */
 		if (!(fl->ifl_sds.ifsd_ba =
 		      (bus_addr_t *) malloc(sizeof(bus_addr_t) *
 					      scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT | M_ZERO))) {
 			device_printf(dev,
 			    "Unable to allocate RX bus address map memory\n");
 			err = ENOMEM;
 			goto fail;
 		}
 
 		/*
 		 * Create the DMA maps for RX buffers.
 		 */
 		if (!(fl->ifl_sds.ifsd_map =
 		      (bus_dmamap_t *) malloc(sizeof(bus_dmamap_t) * scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT | M_ZERO))) {
 			device_printf(dev,
 			    "Unable to allocate RX buffer DMA map memory\n");
 			err = ENOMEM;
 			goto fail;
 		}
 		for (int i = 0; i < scctx->isc_nrxd[rxq->ifr_fl_offset]; i++) {
 			err = bus_dmamap_create(fl->ifl_buf_tag, 0,
 			    &fl->ifl_sds.ifsd_map[i]);
 			if (err != 0) {
 				device_printf(dev, "Unable to create RX buffer DMA map\n");
 				goto fail;
 			}
 		}
 	}
 	return (0);
 
 fail:
 	iflib_rx_structures_free(ctx);
 	return (err);
 }
 
 
 /*
  * Internal service routines
  */
 
 struct rxq_refill_cb_arg {
 	int               error;
 	bus_dma_segment_t seg;
 	int               nseg;
 };
 
 static void
 _rxq_refill_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 {
 	struct rxq_refill_cb_arg *cb_arg = arg;
 
 	cb_arg->error = error;
 	cb_arg->seg = segs[0];
 	cb_arg->nseg = nseg;
 }
 
 /**
  *	rxq_refill - refill an rxq  free-buffer list
  *	@ctx: the iflib context
  *	@rxq: the free-list to refill
  *	@n: the number of new buffers to allocate
  *
  *	(Re)populate an rxq free-buffer list with up to @n new packet buffers.
  *	The caller must assure that @n does not exceed the queue's capacity.
  */
 static void
 _iflib_fl_refill(if_ctx_t ctx, iflib_fl_t fl, int count)
 {
 	struct if_rxd_update iru;
 	struct rxq_refill_cb_arg cb_arg;
 	struct mbuf *m;
 	caddr_t cl, *sd_cl;
 	struct mbuf **sd_m;
 	bus_dmamap_t *sd_map;
 	bus_addr_t bus_addr, *sd_ba;
 	int err, frag_idx, i, idx, n, pidx;
 	qidx_t credits;
 
 	sd_m = fl->ifl_sds.ifsd_m;
 	sd_map = fl->ifl_sds.ifsd_map;
 	sd_cl = fl->ifl_sds.ifsd_cl;
 	sd_ba = fl->ifl_sds.ifsd_ba;
 	pidx = fl->ifl_pidx;
 	idx = pidx;
 	frag_idx = fl->ifl_fragidx;
 	credits = fl->ifl_credits;
 
 	i = 0;
 	n = count;
 	MPASS(n > 0);
 	MPASS(credits + n <= fl->ifl_size);
 
 	if (pidx < fl->ifl_cidx)
 		MPASS(pidx + n <= fl->ifl_cidx);
 	if (pidx == fl->ifl_cidx && (credits < fl->ifl_size))
 		MPASS(fl->ifl_gen == 0);
 	if (pidx > fl->ifl_cidx)
 		MPASS(n <= fl->ifl_size - pidx + fl->ifl_cidx);
 
 	DBG_COUNTER_INC(fl_refills);
 	if (n > 8)
 		DBG_COUNTER_INC(fl_refills_large);
 	iru_init(&iru, fl->ifl_rxq, fl->ifl_id);
 	while (n--) {
 		/*
 		 * We allocate an uninitialized mbuf + cluster, mbuf is
 		 * initialized after rx.
 		 *
 		 * If the cluster is still set then we know a minimum sized packet was received
 		 */
 		bit_ffc_at(fl->ifl_rx_bitmap, frag_idx, fl->ifl_size,
 		    &frag_idx);
 		if (frag_idx < 0)
 			bit_ffc(fl->ifl_rx_bitmap, fl->ifl_size, &frag_idx);
 		MPASS(frag_idx >= 0);
 		if ((cl = sd_cl[frag_idx]) == NULL) {
 			if ((cl = m_cljget(NULL, M_NOWAIT, fl->ifl_buf_size)) == NULL)
 				break;
 
 			cb_arg.error = 0;
 			MPASS(sd_map != NULL);
 			err = bus_dmamap_load(fl->ifl_buf_tag, sd_map[frag_idx],
 			    cl, fl->ifl_buf_size, _rxq_refill_cb, &cb_arg,
 			    BUS_DMA_NOWAIT);
 			if (err != 0 || cb_arg.error) {
 				/*
 				 * !zone_pack ?
 				 */
 				if (fl->ifl_zone == zone_pack)
 					uma_zfree(fl->ifl_zone, cl);
 				break;
 			}
 
 			sd_ba[frag_idx] =  bus_addr = cb_arg.seg.ds_addr;
 			sd_cl[frag_idx] = cl;
 #if MEMORY_LOGGING
 			fl->ifl_cl_enqueued++;
 #endif
 		} else {
 			bus_addr = sd_ba[frag_idx];
 		}
 		bus_dmamap_sync(fl->ifl_buf_tag, sd_map[frag_idx],
 		    BUS_DMASYNC_PREREAD);
 
 		MPASS(sd_m[frag_idx] == NULL);
 		if ((m = m_gethdr(M_NOWAIT, MT_NOINIT)) == NULL) {
 			break;
 		}
 		sd_m[frag_idx] = m;
 		bit_set(fl->ifl_rx_bitmap, frag_idx);
 #if MEMORY_LOGGING
 		fl->ifl_m_enqueued++;
 #endif
 
 		DBG_COUNTER_INC(rx_allocs);
 		fl->ifl_rxd_idxs[i] = frag_idx;
 		fl->ifl_bus_addrs[i] = bus_addr;
 		fl->ifl_vm_addrs[i] = cl;
 		credits++;
 		i++;
 		MPASS(credits <= fl->ifl_size);
 		if (++idx == fl->ifl_size) {
 			fl->ifl_gen = 1;
 			idx = 0;
 		}
 		if (n == 0 || i == IFLIB_MAX_RX_REFRESH) {
 			iru.iru_pidx = pidx;
 			iru.iru_count = i;
 			ctx->isc_rxd_refill(ctx->ifc_softc, &iru);
 			i = 0;
 			pidx = idx;
 			fl->ifl_pidx = idx;
 			fl->ifl_credits = credits;
 		}
 	}
 
 	if (i) {
 		iru.iru_pidx = pidx;
 		iru.iru_count = i;
 		ctx->isc_rxd_refill(ctx->ifc_softc, &iru);
 		fl->ifl_pidx = idx;
 		fl->ifl_credits = credits;
 	}
 	DBG_COUNTER_INC(rxd_flush);
 	if (fl->ifl_pidx == 0)
 		pidx = fl->ifl_size - 1;
 	else
 		pidx = fl->ifl_pidx - 1;
 
 	bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map,
 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 	ctx->isc_rxd_flush(ctx->ifc_softc, fl->ifl_rxq->ifr_id, fl->ifl_id, pidx);
 	fl->ifl_fragidx = frag_idx;
 }
 
 static __inline void
 __iflib_fl_refill_lt(if_ctx_t ctx, iflib_fl_t fl, int max)
 {
 	/* we avoid allowing pidx to catch up with cidx as it confuses ixl */
 	int32_t reclaimable = fl->ifl_size - fl->ifl_credits - 1;
 #ifdef INVARIANTS
 	int32_t delta = fl->ifl_size - get_inuse(fl->ifl_size, fl->ifl_cidx, fl->ifl_pidx, fl->ifl_gen) - 1;
 #endif
 
 	MPASS(fl->ifl_credits <= fl->ifl_size);
 	MPASS(reclaimable == delta);
 
 	if (reclaimable > 0)
 		_iflib_fl_refill(ctx, fl, min(max, reclaimable));
 }
 
 uint8_t
 iflib_in_detach(if_ctx_t ctx)
 {
 	bool in_detach;
 	STATE_LOCK(ctx);
 	in_detach = !!(ctx->ifc_flags & IFC_IN_DETACH);
 	STATE_UNLOCK(ctx);
 	return (in_detach);
 }
 
 static void
 iflib_fl_bufs_free(iflib_fl_t fl)
 {
 	iflib_dma_info_t idi = fl->ifl_ifdi;
 	bus_dmamap_t sd_map;
 	uint32_t i;
 
 	for (i = 0; i < fl->ifl_size; i++) {
 		struct mbuf **sd_m = &fl->ifl_sds.ifsd_m[i];
 		caddr_t *sd_cl = &fl->ifl_sds.ifsd_cl[i];
 
 		if (*sd_cl != NULL) {
 			sd_map = fl->ifl_sds.ifsd_map[i];
 			bus_dmamap_sync(fl->ifl_buf_tag, sd_map,
 			    BUS_DMASYNC_POSTREAD);
 			bus_dmamap_unload(fl->ifl_buf_tag, sd_map);
 			if (*sd_cl != NULL)
 				uma_zfree(fl->ifl_zone, *sd_cl);
 			// XXX: Should this get moved out?
 			if (iflib_in_detach(fl->ifl_rxq->ifr_ctx))
 				bus_dmamap_destroy(fl->ifl_buf_tag, sd_map);
 			if (*sd_m != NULL) {
 				m_init(*sd_m, M_NOWAIT, MT_DATA, 0);
 				uma_zfree(zone_mbuf, *sd_m);
 			}
 		} else {
 			MPASS(*sd_cl == NULL);
 			MPASS(*sd_m == NULL);
 		}
 #if MEMORY_LOGGING
 		fl->ifl_m_dequeued++;
 		fl->ifl_cl_dequeued++;
 #endif
 		*sd_cl = NULL;
 		*sd_m = NULL;
 	}
 #ifdef INVARIANTS
 	for (i = 0; i < fl->ifl_size; i++) {
 		MPASS(fl->ifl_sds.ifsd_cl[i] == NULL);
 		MPASS(fl->ifl_sds.ifsd_m[i] == NULL);
 	}
 #endif
 	/*
 	 * Reset free list values
 	 */
 	fl->ifl_credits = fl->ifl_cidx = fl->ifl_pidx = fl->ifl_gen = fl->ifl_fragidx = 0;
 	bzero(idi->idi_vaddr, idi->idi_size);
 }
 
 /*********************************************************************
  *
  *  Initialize a receive ring and its buffers.
  *
  **********************************************************************/
 static int
 iflib_fl_setup(iflib_fl_t fl)
 {
 	iflib_rxq_t rxq = fl->ifl_rxq;
 	if_ctx_t ctx = rxq->ifr_ctx;
-	if_softc_ctx_t sctx = &ctx->ifc_softc_ctx;
 
 	bit_nclear(fl->ifl_rx_bitmap, 0, fl->ifl_size - 1);
 	/*
 	** Free current RX buffer structs and their mbufs
 	*/
 	iflib_fl_bufs_free(fl);
 	/* Now replenish the mbufs */
 	MPASS(fl->ifl_credits == 0);
-	/*
-	 * XXX don't set the max_frame_size to larger
-	 * than the hardware can handle
-	 */
-	if (sctx->isc_max_frame_size <= 2048)
-		fl->ifl_buf_size = MCLBYTES;
-	else
-		fl->ifl_buf_size = MJUMPAGESIZE;
+	fl->ifl_buf_size = ctx->ifc_rx_mbuf_sz;
 	if (fl->ifl_buf_size > ctx->ifc_max_fl_buf_size)
 		ctx->ifc_max_fl_buf_size = fl->ifl_buf_size;
 	fl->ifl_cltype = m_gettype(fl->ifl_buf_size);
 	fl->ifl_zone = m_getzone(fl->ifl_buf_size);
 
 
 	/* avoid pre-allocating zillions of clusters to an idle card
 	 * potentially speeding up attach
 	 */
 	_iflib_fl_refill(ctx, fl, min(128, fl->ifl_size));
 	MPASS(min(128, fl->ifl_size) == fl->ifl_credits);
 	if (min(128, fl->ifl_size) != fl->ifl_credits)
 		return (ENOBUFS);
 	/*
 	 * handle failure
 	 */
 	MPASS(rxq != NULL);
 	MPASS(fl->ifl_ifdi != NULL);
 	bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map,
 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 	return (0);
 }
 
 /*********************************************************************
  *
  *  Free receive ring data structures
  *
  **********************************************************************/
 static void
 iflib_rx_sds_free(iflib_rxq_t rxq)
 {
 	iflib_fl_t fl;
 	int i, j;
 
 	if (rxq->ifr_fl != NULL) {
 		for (i = 0; i < rxq->ifr_nfl; i++) {
 			fl = &rxq->ifr_fl[i];
 			if (fl->ifl_buf_tag != NULL) {
 				if (fl->ifl_sds.ifsd_map != NULL) {
 					for (j = 0; j < fl->ifl_size; j++) {
 						if (fl->ifl_sds.ifsd_map[j] ==
 						    NULL)
 							continue;
 						bus_dmamap_sync(
 						    fl->ifl_buf_tag,
 						    fl->ifl_sds.ifsd_map[j],
 						    BUS_DMASYNC_POSTREAD);
 						bus_dmamap_unload(
 						    fl->ifl_buf_tag,
 						    fl->ifl_sds.ifsd_map[j]);
 					}
 				}
 				bus_dma_tag_destroy(fl->ifl_buf_tag);
 				fl->ifl_buf_tag = NULL;
 			}
 			free(fl->ifl_sds.ifsd_m, M_IFLIB);
 			free(fl->ifl_sds.ifsd_cl, M_IFLIB);
 			free(fl->ifl_sds.ifsd_ba, M_IFLIB);
 			free(fl->ifl_sds.ifsd_map, M_IFLIB);
 			fl->ifl_sds.ifsd_m = NULL;
 			fl->ifl_sds.ifsd_cl = NULL;
 			fl->ifl_sds.ifsd_ba = NULL;
 			fl->ifl_sds.ifsd_map = NULL;
 		}
 		free(rxq->ifr_fl, M_IFLIB);
 		rxq->ifr_fl = NULL;
 		rxq->ifr_cq_gen = rxq->ifr_cq_cidx = rxq->ifr_cq_pidx = 0;
 	}
 }
 
 /*
  * MI independent logic
  *
  */
 static void
 iflib_timer(void *arg)
 {
 	iflib_txq_t txq = arg;
 	if_ctx_t ctx = txq->ift_ctx;
 	if_softc_ctx_t sctx = &ctx->ifc_softc_ctx;
 	uint64_t this_tick = ticks;
 	uint32_t reset_on = hz / 2;
 
 	if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING))
 		return;
 	/*
 	** Check on the state of the TX queue(s), this
 	** can be done without the lock because its RO
 	** and the HUNG state will be static if set.
 	*/
 	if (this_tick - txq->ift_last_timer_tick >= hz / 2) {
 		txq->ift_last_timer_tick = this_tick;
 		IFDI_TIMER(ctx, txq->ift_id);
 		if ((txq->ift_qstatus == IFLIB_QUEUE_HUNG) &&
 		    ((txq->ift_cleaned_prev == txq->ift_cleaned) ||
 		     (sctx->isc_pause_frames == 0)))
 			goto hung;
 
 		if (ifmp_ring_is_stalled(txq->ift_br))
 			txq->ift_qstatus = IFLIB_QUEUE_HUNG;
 		txq->ift_cleaned_prev = txq->ift_cleaned;
 	}
 #ifdef DEV_NETMAP
 	if (if_getcapenable(ctx->ifc_ifp) & IFCAP_NETMAP)
 		iflib_netmap_timer_adjust(ctx, txq, &reset_on);
 #endif
 	/* handle any laggards */
 	if (txq->ift_db_pending)
 		GROUPTASK_ENQUEUE(&txq->ift_task);
 
 	sctx->isc_pause_frames = 0;
 	if (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING) 
 		callout_reset_on(&txq->ift_timer, reset_on, iflib_timer, txq, txq->ift_timer.c_cpu);
 	return;
  hung:
 	device_printf(ctx->ifc_dev,  "TX(%d) desc avail = %d, pidx = %d\n",
 				  txq->ift_id, TXQ_AVAIL(txq), txq->ift_pidx);
 	STATE_LOCK(ctx);
 	if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
 	ctx->ifc_flags |= (IFC_DO_WATCHDOG|IFC_DO_RESET);
 	iflib_admin_intr_deferred(ctx);
 	STATE_UNLOCK(ctx);
 }
 
 static void
+iflib_calc_rx_mbuf_sz(if_ctx_t ctx)
+{
+	if_softc_ctx_t sctx = &ctx->ifc_softc_ctx;
+
+	/*
+	 * XXX don't set the max_frame_size to larger
+	 * than the hardware can handle
+	 */
+	if (sctx->isc_max_frame_size <= MCLBYTES)
+		ctx->ifc_rx_mbuf_sz = MCLBYTES;
+	else
+		ctx->ifc_rx_mbuf_sz = MJUMPAGESIZE;
+}
+
+uint32_t
+iflib_get_rx_mbuf_sz(if_ctx_t ctx)
+{
+	return (ctx->ifc_rx_mbuf_sz);
+}
+
+static void
 iflib_init_locked(if_ctx_t ctx)
 {
 	if_softc_ctx_t sctx = &ctx->ifc_softc_ctx;
 	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
 	if_t ifp = ctx->ifc_ifp;
 	iflib_fl_t fl;
 	iflib_txq_t txq;
 	iflib_rxq_t rxq;
 	int i, j, tx_ip_csum_flags, tx_ip6_csum_flags;
 
 
 	if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
 	IFDI_INTR_DISABLE(ctx);
 
 	tx_ip_csum_flags = scctx->isc_tx_csum_flags & (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_SCTP);
 	tx_ip6_csum_flags = scctx->isc_tx_csum_flags & (CSUM_IP6_TCP | CSUM_IP6_UDP | CSUM_IP6_SCTP);
 	/* Set hardware offload abilities */
 	if_clearhwassist(ifp);
 	if (if_getcapenable(ifp) & IFCAP_TXCSUM)
 		if_sethwassistbits(ifp, tx_ip_csum_flags, 0);
 	if (if_getcapenable(ifp) & IFCAP_TXCSUM_IPV6)
 		if_sethwassistbits(ifp,  tx_ip6_csum_flags, 0);
 	if (if_getcapenable(ifp) & IFCAP_TSO4)
 		if_sethwassistbits(ifp, CSUM_IP_TSO, 0);
 	if (if_getcapenable(ifp) & IFCAP_TSO6)
 		if_sethwassistbits(ifp, CSUM_IP6_TSO, 0);
 
 	for (i = 0, txq = ctx->ifc_txqs; i < sctx->isc_ntxqsets; i++, txq++) {
 		CALLOUT_LOCK(txq);
 		callout_stop(&txq->ift_timer);
 		CALLOUT_UNLOCK(txq);
 		iflib_netmap_txq_init(ctx, txq);
 	}
+
+	/*
+	 * Calculate a suitable Rx mbuf size prior to calling IFDI_INIT, so
+	 * that drivers can use the value when setting up the hardware receive
+	 * buffers.
+	 */
+	iflib_calc_rx_mbuf_sz(ctx);
+
 #ifdef INVARIANTS
 	i = if_getdrvflags(ifp);
 #endif
 	IFDI_INIT(ctx);
 	MPASS(if_getdrvflags(ifp) == i);
 	for (i = 0, rxq = ctx->ifc_rxqs; i < sctx->isc_nrxqsets; i++, rxq++) {
 		/* XXX this should really be done on a per-queue basis */
 		if (if_getcapenable(ifp) & IFCAP_NETMAP) {
 			MPASS(rxq->ifr_id == i);
 			iflib_netmap_rxq_init(ctx, rxq);
 			continue;
 		}
 		for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) {
 			if (iflib_fl_setup(fl)) {
 				device_printf(ctx->ifc_dev, "freelist setup failed - check cluster settings\n");
 				goto done;
 			}
 		}
 	}
 done:
 	if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE);
 	IFDI_INTR_ENABLE(ctx);
 	txq = ctx->ifc_txqs;
 	for (i = 0; i < sctx->isc_ntxqsets; i++, txq++)
 		callout_reset_on(&txq->ift_timer, hz/2, iflib_timer, txq,
 			txq->ift_timer.c_cpu);
 }
 
 static int
 iflib_media_change(if_t ifp)
 {
 	if_ctx_t ctx = if_getsoftc(ifp);
 	int err;
 
 	CTX_LOCK(ctx);
 	if ((err = IFDI_MEDIA_CHANGE(ctx)) == 0)
 		iflib_init_locked(ctx);
 	CTX_UNLOCK(ctx);
 	return (err);
 }
 
 static void
 iflib_media_status(if_t ifp, struct ifmediareq *ifmr)
 {
 	if_ctx_t ctx = if_getsoftc(ifp);
 
 	CTX_LOCK(ctx);
 	IFDI_UPDATE_ADMIN_STATUS(ctx);
 	IFDI_MEDIA_STATUS(ctx, ifmr);
 	CTX_UNLOCK(ctx);
 }
 
 void
 iflib_stop(if_ctx_t ctx)
 {
 	iflib_txq_t txq = ctx->ifc_txqs;
 	iflib_rxq_t rxq = ctx->ifc_rxqs;
 	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
 	if_shared_ctx_t sctx = ctx->ifc_sctx;
 	iflib_dma_info_t di;
 	iflib_fl_t fl;
 	int i, j;
 
 	/* Tell the stack that the interface is no longer active */
 	if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
 
 	IFDI_INTR_DISABLE(ctx);
 	DELAY(1000);
 	IFDI_STOP(ctx);
 	DELAY(1000);
 
 	iflib_debug_reset();
 	/* Wait for current tx queue users to exit to disarm watchdog timer. */
 	for (i = 0; i < scctx->isc_ntxqsets; i++, txq++) {
 		/* make sure all transmitters have completed before proceeding XXX */
 
 		CALLOUT_LOCK(txq);
 		callout_stop(&txq->ift_timer);
 		CALLOUT_UNLOCK(txq);
 
 		/* clean any enqueued buffers */
 		iflib_ifmp_purge(txq);
 		/* Free any existing tx buffers. */
 		for (j = 0; j < txq->ift_size; j++) {
 			iflib_txsd_free(ctx, txq, j);
 		}
 		txq->ift_processed = txq->ift_cleaned = txq->ift_cidx_processed = 0;
 		txq->ift_in_use = txq->ift_gen = txq->ift_cidx = txq->ift_pidx = txq->ift_no_desc_avail = 0;
 		txq->ift_closed = txq->ift_mbuf_defrag = txq->ift_mbuf_defrag_failed = 0;
 		txq->ift_no_tx_dma_setup = txq->ift_txd_encap_efbig = txq->ift_map_failed = 0;
 		txq->ift_pullups = 0;
 		ifmp_ring_reset_stats(txq->ift_br);
 		for (j = 0, di = txq->ift_ifdi; j < sctx->isc_ntxqs; j++, di++)
 			bzero((void *)di->idi_vaddr, di->idi_size);
 	}
 	for (i = 0; i < scctx->isc_nrxqsets; i++, rxq++) {
 		/* make sure all transmitters have completed before proceeding XXX */
 
 		rxq->ifr_cq_gen = rxq->ifr_cq_cidx = rxq->ifr_cq_pidx = 0;
 		for (j = 0, di = rxq->ifr_ifdi; j < sctx->isc_nrxqs; j++, di++)
 			bzero((void *)di->idi_vaddr, di->idi_size);
 		/* also resets the free lists pidx/cidx */
 		for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++)
 			iflib_fl_bufs_free(fl);
 	}
 }
 
 static inline caddr_t
 calc_next_rxd(iflib_fl_t fl, int cidx)
 {
 	qidx_t size;
 	int nrxd;
 	caddr_t start, end, cur, next;
 
 	nrxd = fl->ifl_size;
 	size = fl->ifl_rxd_size;
 	start = fl->ifl_ifdi->idi_vaddr;
 
 	if (__predict_false(size == 0))
 		return (start);
 	cur = start + size*cidx;
 	end = start + size*nrxd;
 	next = CACHE_PTR_NEXT(cur);
 	return (next < end ? next : start);
 }
 
 static inline void
 prefetch_pkts(iflib_fl_t fl, int cidx)
 {
 	int nextptr;
 	int nrxd = fl->ifl_size;
 	caddr_t next_rxd;
 
 
 	nextptr = (cidx + CACHE_PTR_INCREMENT) & (nrxd-1);
 	prefetch(&fl->ifl_sds.ifsd_m[nextptr]);
 	prefetch(&fl->ifl_sds.ifsd_cl[nextptr]);
 	next_rxd = calc_next_rxd(fl, cidx);
 	prefetch(next_rxd);
 	prefetch(fl->ifl_sds.ifsd_m[(cidx + 1) & (nrxd-1)]);
 	prefetch(fl->ifl_sds.ifsd_m[(cidx + 2) & (nrxd-1)]);
 	prefetch(fl->ifl_sds.ifsd_m[(cidx + 3) & (nrxd-1)]);
 	prefetch(fl->ifl_sds.ifsd_m[(cidx + 4) & (nrxd-1)]);
 	prefetch(fl->ifl_sds.ifsd_cl[(cidx + 1) & (nrxd-1)]);
 	prefetch(fl->ifl_sds.ifsd_cl[(cidx + 2) & (nrxd-1)]);
 	prefetch(fl->ifl_sds.ifsd_cl[(cidx + 3) & (nrxd-1)]);
 	prefetch(fl->ifl_sds.ifsd_cl[(cidx + 4) & (nrxd-1)]);
 }
 
 static void
 rxd_frag_to_sd(iflib_rxq_t rxq, if_rxd_frag_t irf, int unload, if_rxsd_t sd)
 {
 	int flid, cidx;
 	bus_dmamap_t map;
 	iflib_fl_t fl;
 	int next;
 
 	map = NULL;
 	flid = irf->irf_flid;
 	cidx = irf->irf_idx;
 	fl = &rxq->ifr_fl[flid];
 	sd->ifsd_fl = fl;
 	sd->ifsd_cidx = cidx;
 	sd->ifsd_m = &fl->ifl_sds.ifsd_m[cidx];
 	sd->ifsd_cl = &fl->ifl_sds.ifsd_cl[cidx];
 	fl->ifl_credits--;
 #if MEMORY_LOGGING
 	fl->ifl_m_dequeued++;
 #endif
 	if (rxq->ifr_ctx->ifc_flags & IFC_PREFETCH)
 		prefetch_pkts(fl, cidx);
 	next = (cidx + CACHE_PTR_INCREMENT) & (fl->ifl_size-1);
 	prefetch(&fl->ifl_sds.ifsd_map[next]);
 	map = fl->ifl_sds.ifsd_map[cidx];
 	next = (cidx + CACHE_LINE_SIZE) & (fl->ifl_size-1);
 
 	/* not valid assert if bxe really does SGE from non-contiguous elements */
 	MPASS(fl->ifl_cidx == cidx);
 	bus_dmamap_sync(fl->ifl_buf_tag, map, BUS_DMASYNC_POSTREAD);
 	if (unload)
 		bus_dmamap_unload(fl->ifl_buf_tag, map);
 	fl->ifl_cidx = (fl->ifl_cidx + 1) & (fl->ifl_size-1);
 	if (__predict_false(fl->ifl_cidx == 0))
 		fl->ifl_gen = 0;
 	bit_clear(fl->ifl_rx_bitmap, cidx);
 }
 
 static struct mbuf *
 assemble_segments(iflib_rxq_t rxq, if_rxd_info_t ri, if_rxsd_t sd)
 {
 	int i, padlen , flags;
 	struct mbuf *m, *mh, *mt;
 	caddr_t cl;
 
 	i = 0;
 	mh = NULL;
 	do {
 		rxd_frag_to_sd(rxq, &ri->iri_frags[i], TRUE, sd);
 
 		MPASS(*sd->ifsd_cl != NULL);
 		MPASS(*sd->ifsd_m != NULL);
 
 		/* Don't include zero-length frags */
 		if (ri->iri_frags[i].irf_len == 0) {
 			/* XXX we can save the cluster here, but not the mbuf */
 			m_init(*sd->ifsd_m, M_NOWAIT, MT_DATA, 0);
 			m_free(*sd->ifsd_m);
 			*sd->ifsd_m = NULL;
 			continue;
 		}
 		m = *sd->ifsd_m;
 		*sd->ifsd_m = NULL;
 		if (mh == NULL) {
 			flags = M_PKTHDR|M_EXT;
 			mh = mt = m;
 			padlen = ri->iri_pad;
 		} else {
 			flags = M_EXT;
 			mt->m_next = m;
 			mt = m;
 			/* assuming padding is only on the first fragment */
 			padlen = 0;
 		}
 		cl = *sd->ifsd_cl;
 		*sd->ifsd_cl = NULL;
 
 		/* Can these two be made one ? */
 		m_init(m, M_NOWAIT, MT_DATA, flags);
 		m_cljset(m, cl, sd->ifsd_fl->ifl_cltype);
 		/*
 		 * These must follow m_init and m_cljset
 		 */
 		m->m_data += padlen;
 		ri->iri_len -= padlen;
 		m->m_len = ri->iri_frags[i].irf_len;
 	} while (++i < ri->iri_nfrags);
 
 	return (mh);
 }
 
 /*
  * Process one software descriptor
  */
 static struct mbuf *
 iflib_rxd_pkt_get(iflib_rxq_t rxq, if_rxd_info_t ri)
 {
 	struct if_rxsd sd;
 	struct mbuf *m;
 
 	/* should I merge this back in now that the two paths are basically duplicated? */
 	if (ri->iri_nfrags == 1 &&
 	    ri->iri_frags[0].irf_len <= MIN(IFLIB_RX_COPY_THRESH, MHLEN)) {
 		rxd_frag_to_sd(rxq, &ri->iri_frags[0], FALSE, &sd);
 		m = *sd.ifsd_m;
 		*sd.ifsd_m = NULL;
 		m_init(m, M_NOWAIT, MT_DATA, M_PKTHDR);
 #ifndef __NO_STRICT_ALIGNMENT
 		if (!IP_ALIGNED(m))
 			m->m_data += 2;
 #endif
 		memcpy(m->m_data, *sd.ifsd_cl, ri->iri_len);
 		m->m_len = ri->iri_frags[0].irf_len;
        } else {
 		m = assemble_segments(rxq, ri, &sd);
 	}
 	m->m_pkthdr.len = ri->iri_len;
 	m->m_pkthdr.rcvif = ri->iri_ifp;
 	m->m_flags |= ri->iri_flags;
 	m->m_pkthdr.ether_vtag = ri->iri_vtag;
 	m->m_pkthdr.flowid = ri->iri_flowid;
 	M_HASHTYPE_SET(m, ri->iri_rsstype);
 	m->m_pkthdr.csum_flags = ri->iri_csum_flags;
 	m->m_pkthdr.csum_data = ri->iri_csum_data;
 	return (m);
 }
 
 #if defined(INET6) || defined(INET)
 static void
 iflib_get_ip_forwarding(struct lro_ctrl *lc, bool *v4, bool *v6)
 {
 	CURVNET_SET(lc->ifp->if_vnet);
 #if defined(INET6)
 	*v6 = VNET(ip6_forwarding);
 #endif
 #if defined(INET)
 	*v4 = VNET(ipforwarding);
 #endif
 	CURVNET_RESTORE();
 }
 
 /*
  * Returns true if it's possible this packet could be LROed.
  * if it returns false, it is guaranteed that tcp_lro_rx()
  * would not return zero.
  */
 static bool
 iflib_check_lro_possible(struct mbuf *m, bool v4_forwarding, bool v6_forwarding)
 {
 	struct ether_header *eh;
 	uint16_t eh_type;
 
 	eh = mtod(m, struct ether_header *);
 	eh_type = ntohs(eh->ether_type);
 	switch (eh_type) {
 #if defined(INET6)
 		case ETHERTYPE_IPV6:
 			return !v6_forwarding;
 #endif
 #if defined (INET)
 		case ETHERTYPE_IP:
 			return !v4_forwarding;
 #endif
 	}
 
 	return false;
 }
 #else
 static void
 iflib_get_ip_forwarding(struct lro_ctrl *lc __unused, bool *v4 __unused, bool *v6 __unused)
 {
 }
 #endif
 
 static bool
 iflib_rxeof(iflib_rxq_t rxq, qidx_t budget)
 {
 	if_ctx_t ctx = rxq->ifr_ctx;
 	if_shared_ctx_t sctx = ctx->ifc_sctx;
 	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
 	int avail, i;
 	qidx_t *cidxp;
 	struct if_rxd_info ri;
 	int err, budget_left, rx_bytes, rx_pkts;
 	iflib_fl_t fl;
 	struct ifnet *ifp;
 	int lro_enabled;
 	bool v4_forwarding, v6_forwarding, lro_possible;
 
 	/*
 	 * XXX early demux data packets so that if_input processing only handles
 	 * acks in interrupt context
 	 */
 	struct mbuf *m, *mh, *mt, *mf;
 
 	lro_possible = v4_forwarding = v6_forwarding = false;
 	ifp = ctx->ifc_ifp;
 	mh = mt = NULL;
 	MPASS(budget > 0);
 	rx_pkts	= rx_bytes = 0;
 	if (sctx->isc_flags & IFLIB_HAS_RXCQ)
 		cidxp = &rxq->ifr_cq_cidx;
 	else
 		cidxp = &rxq->ifr_fl[0].ifl_cidx;
 	if ((avail = iflib_rxd_avail(ctx, rxq, *cidxp, budget)) == 0) {
 		for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++)
 			__iflib_fl_refill_lt(ctx, fl, budget + 8);
 		DBG_COUNTER_INC(rx_unavail);
 		return (false);
 	}
 
 	for (budget_left = budget; budget_left > 0 && avail > 0;) {
 		if (__predict_false(!CTX_ACTIVE(ctx))) {
 			DBG_COUNTER_INC(rx_ctx_inactive);
 			break;
 		}
 		/*
 		 * Reset client set fields to their default values
 		 */
 		rxd_info_zero(&ri);
 		ri.iri_qsidx = rxq->ifr_id;
 		ri.iri_cidx = *cidxp;
 		ri.iri_ifp = ifp;
 		ri.iri_frags = rxq->ifr_frags;
 		err = ctx->isc_rxd_pkt_get(ctx->ifc_softc, &ri);
 
 		if (err)
 			goto err;
 		if (sctx->isc_flags & IFLIB_HAS_RXCQ) {
 			*cidxp = ri.iri_cidx;
 			/* Update our consumer index */
 			/* XXX NB: shurd - check if this is still safe */
 			while (rxq->ifr_cq_cidx >= scctx->isc_nrxd[0]) {
 				rxq->ifr_cq_cidx -= scctx->isc_nrxd[0];
 				rxq->ifr_cq_gen = 0;
 			}
 			/* was this only a completion queue message? */
 			if (__predict_false(ri.iri_nfrags == 0))
 				continue;
 		}
 		MPASS(ri.iri_nfrags != 0);
 		MPASS(ri.iri_len != 0);
 
 		/* will advance the cidx on the corresponding free lists */
 		m = iflib_rxd_pkt_get(rxq, &ri);
 		avail--;
 		budget_left--;
 		if (avail == 0 && budget_left)
 			avail = iflib_rxd_avail(ctx, rxq, *cidxp, budget_left);
 
 		if (__predict_false(m == NULL)) {
 			DBG_COUNTER_INC(rx_mbuf_null);
 			continue;
 		}
 		/* imm_pkt: -- cxgb */
 		if (mh == NULL)
 			mh = mt = m;
 		else {
 			mt->m_nextpkt = m;
 			mt = m;
 		}
 	}
 	/* make sure that we can refill faster than drain */
 	for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++)
 		__iflib_fl_refill_lt(ctx, fl, budget + 8);
 
 	lro_enabled = (if_getcapenable(ifp) & IFCAP_LRO);
 	if (lro_enabled)
 		iflib_get_ip_forwarding(&rxq->ifr_lc, &v4_forwarding, &v6_forwarding);
 	mt = mf = NULL;
 	while (mh != NULL) {
 		m = mh;
 		mh = mh->m_nextpkt;
 		m->m_nextpkt = NULL;
 #ifndef __NO_STRICT_ALIGNMENT
 		if (!IP_ALIGNED(m) && (m = iflib_fixup_rx(m)) == NULL)
 			continue;
 #endif
 		rx_bytes += m->m_pkthdr.len;
 		rx_pkts++;
 #if defined(INET6) || defined(INET)
 		if (lro_enabled) {
 			if (!lro_possible) {
 				lro_possible = iflib_check_lro_possible(m, v4_forwarding, v6_forwarding);
 				if (lro_possible && mf != NULL) {
 					ifp->if_input(ifp, mf);
 					DBG_COUNTER_INC(rx_if_input);
 					mt = mf = NULL;
 				}
 			}
 			if ((m->m_pkthdr.csum_flags & (CSUM_L4_CALC|CSUM_L4_VALID)) ==
 			    (CSUM_L4_CALC|CSUM_L4_VALID)) {
 				if (lro_possible && tcp_lro_rx(&rxq->ifr_lc, m, 0) == 0)
 					continue;
 			}
 		}
 #endif
 		if (lro_possible) {
 			ifp->if_input(ifp, m);
 			DBG_COUNTER_INC(rx_if_input);
 			continue;
 		}
 
 		if (mf == NULL)
 			mf = m;
 		if (mt != NULL)
 			mt->m_nextpkt = m;
 		mt = m;
 	}
 	if (mf != NULL) {
 		ifp->if_input(ifp, mf);
 		DBG_COUNTER_INC(rx_if_input);
 	}
 
 	if_inc_counter(ifp, IFCOUNTER_IBYTES, rx_bytes);
 	if_inc_counter(ifp, IFCOUNTER_IPACKETS, rx_pkts);
 
 	/*
 	 * Flush any outstanding LRO work
 	 */
 #if defined(INET6) || defined(INET)
 	tcp_lro_flush_all(&rxq->ifr_lc);
 #endif
 	if (avail)
 		return true;
 	return (iflib_rxd_avail(ctx, rxq, *cidxp, 1));
 err:
 	STATE_LOCK(ctx);
 	ctx->ifc_flags |= IFC_DO_RESET;
 	iflib_admin_intr_deferred(ctx);
 	STATE_UNLOCK(ctx);
 	return (false);
 }
 
 #define TXD_NOTIFY_COUNT(txq) (((txq)->ift_size / (txq)->ift_update_freq)-1)
 static inline qidx_t
 txq_max_db_deferred(iflib_txq_t txq, qidx_t in_use)
 {
 	qidx_t notify_count = TXD_NOTIFY_COUNT(txq);
 	qidx_t minthresh = txq->ift_size / 8;
 	if (in_use > 4*minthresh)
 		return (notify_count);
 	if (in_use > 2*minthresh)
 		return (notify_count >> 1);
 	if (in_use > minthresh)
 		return (notify_count >> 3);
 	return (0);
 }
 
 static inline qidx_t
 txq_max_rs_deferred(iflib_txq_t txq)
 {
 	qidx_t notify_count = TXD_NOTIFY_COUNT(txq);
 	qidx_t minthresh = txq->ift_size / 8;
 	if (txq->ift_in_use > 4*minthresh)
 		return (notify_count);
 	if (txq->ift_in_use > 2*minthresh)
 		return (notify_count >> 1);
 	if (txq->ift_in_use > minthresh)
 		return (notify_count >> 2);
 	return (2);
 }
 
 #define M_CSUM_FLAGS(m) ((m)->m_pkthdr.csum_flags)
 #define M_HAS_VLANTAG(m) (m->m_flags & M_VLANTAG)
 
 #define TXQ_MAX_DB_DEFERRED(txq, in_use) txq_max_db_deferred((txq), (in_use))
 #define TXQ_MAX_RS_DEFERRED(txq) txq_max_rs_deferred(txq)
 #define TXQ_MAX_DB_CONSUMED(size) (size >> 4)
 
 /* forward compatibility for cxgb */
 #define FIRST_QSET(ctx) 0
 #define NTXQSETS(ctx) ((ctx)->ifc_softc_ctx.isc_ntxqsets)
 #define NRXQSETS(ctx) ((ctx)->ifc_softc_ctx.isc_nrxqsets)
 #define QIDX(ctx, m) ((((m)->m_pkthdr.flowid & ctx->ifc_softc_ctx.isc_rss_table_mask) % NTXQSETS(ctx)) + FIRST_QSET(ctx))
 #define DESC_RECLAIMABLE(q) ((int)((q)->ift_processed - (q)->ift_cleaned - (q)->ift_ctx->ifc_softc_ctx.isc_tx_nsegments))
 
 /* XXX we should be setting this to something other than zero */
 #define RECLAIM_THRESH(ctx) ((ctx)->ifc_sctx->isc_tx_reclaim_thresh)
 #define	MAX_TX_DESC(ctx) max((ctx)->ifc_softc_ctx.isc_tx_tso_segments_max, \
     (ctx)->ifc_softc_ctx.isc_tx_nsegments)
 
 static inline bool
 iflib_txd_db_check(if_ctx_t ctx, iflib_txq_t txq, int ring, qidx_t in_use)
 {
 	qidx_t dbval, max;
 	bool rang;
 
 	rang = false;
 	max = TXQ_MAX_DB_DEFERRED(txq, in_use);
 	if (ring || txq->ift_db_pending >= max) {
 		dbval = txq->ift_npending ? txq->ift_npending : txq->ift_pidx;
 		bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,
 		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 		ctx->isc_txd_flush(ctx->ifc_softc, txq->ift_id, dbval);
 		txq->ift_db_pending = txq->ift_npending = 0;
 		rang = true;
 	}
 	return (rang);
 }
 
 #ifdef PKT_DEBUG
 static void
 print_pkt(if_pkt_info_t pi)
 {
 	printf("pi len:  %d qsidx: %d nsegs: %d ndescs: %d flags: %x pidx: %d\n",
 	       pi->ipi_len, pi->ipi_qsidx, pi->ipi_nsegs, pi->ipi_ndescs, pi->ipi_flags, pi->ipi_pidx);
 	printf("pi new_pidx: %d csum_flags: %lx tso_segsz: %d mflags: %x vtag: %d\n",
 	       pi->ipi_new_pidx, pi->ipi_csum_flags, pi->ipi_tso_segsz, pi->ipi_mflags, pi->ipi_vtag);
 	printf("pi etype: %d ehdrlen: %d ip_hlen: %d ipproto: %d\n",
 	       pi->ipi_etype, pi->ipi_ehdrlen, pi->ipi_ip_hlen, pi->ipi_ipproto);
 }
 #endif
 
 #define IS_TSO4(pi) ((pi)->ipi_csum_flags & CSUM_IP_TSO)
 #define IS_TX_OFFLOAD4(pi) ((pi)->ipi_csum_flags & (CSUM_IP_TCP | CSUM_IP_TSO))
 #define IS_TSO6(pi) ((pi)->ipi_csum_flags & CSUM_IP6_TSO)
 #define IS_TX_OFFLOAD6(pi) ((pi)->ipi_csum_flags & (CSUM_IP6_TCP | CSUM_IP6_TSO))
 
 static int
 iflib_parse_header(iflib_txq_t txq, if_pkt_info_t pi, struct mbuf **mp)
 {
 	if_shared_ctx_t sctx = txq->ift_ctx->ifc_sctx;
 	struct ether_vlan_header *eh;
 	struct mbuf *m;
 
 	m = *mp;
 	if ((sctx->isc_flags & IFLIB_NEED_SCRATCH) &&
 	    M_WRITABLE(m) == 0) {
 		if ((m = m_dup(m, M_NOWAIT)) == NULL) {
 			return (ENOMEM);
 		} else {
 			m_freem(*mp);
 			DBG_COUNTER_INC(tx_frees);
 			*mp = m;
 		}
 	}
 
 	/*
 	 * Determine where frame payload starts.
 	 * Jump over vlan headers if already present,
 	 * helpful for QinQ too.
 	 */
 	if (__predict_false(m->m_len < sizeof(*eh))) {
 		txq->ift_pullups++;
 		if (__predict_false((m = m_pullup(m, sizeof(*eh))) == NULL))
 			return (ENOMEM);
 	}
 	eh = mtod(m, struct ether_vlan_header *);
 	if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
 		pi->ipi_etype = ntohs(eh->evl_proto);
 		pi->ipi_ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
 	} else {
 		pi->ipi_etype = ntohs(eh->evl_encap_proto);
 		pi->ipi_ehdrlen = ETHER_HDR_LEN;
 	}
 
 	switch (pi->ipi_etype) {
 #ifdef INET
 	case ETHERTYPE_IP:
 	{
 		struct mbuf *n;
 		struct ip *ip = NULL;
 		struct tcphdr *th = NULL;
 		int minthlen;
 
 		minthlen = min(m->m_pkthdr.len, pi->ipi_ehdrlen + sizeof(*ip) + sizeof(*th));
 		if (__predict_false(m->m_len < minthlen)) {
 			/*
 			 * if this code bloat is causing too much of a hit
 			 * move it to a separate function and mark it noinline
 			 */
 			if (m->m_len == pi->ipi_ehdrlen) {
 				n = m->m_next;
 				MPASS(n);
 				if (n->m_len >= sizeof(*ip))  {
 					ip = (struct ip *)n->m_data;
 					if (n->m_len >= (ip->ip_hl << 2) + sizeof(*th))
 						th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2));
 				} else {
 					txq->ift_pullups++;
 					if (__predict_false((m = m_pullup(m, minthlen)) == NULL))
 						return (ENOMEM);
 					ip = (struct ip *)(m->m_data + pi->ipi_ehdrlen);
 				}
 			} else {
 				txq->ift_pullups++;
 				if (__predict_false((m = m_pullup(m, minthlen)) == NULL))
 					return (ENOMEM);
 				ip = (struct ip *)(m->m_data + pi->ipi_ehdrlen);
 				if (m->m_len >= (ip->ip_hl << 2) + sizeof(*th))
 					th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2));
 			}
 		} else {
 			ip = (struct ip *)(m->m_data + pi->ipi_ehdrlen);
 			if (m->m_len >= (ip->ip_hl << 2) + sizeof(*th))
 				th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2));
 		}
 		pi->ipi_ip_hlen = ip->ip_hl << 2;
 		pi->ipi_ipproto = ip->ip_p;
 		pi->ipi_flags |= IPI_TX_IPV4;
 
 		/* TCP checksum offload may require TCP header length */
 		if (IS_TX_OFFLOAD4(pi)) {
 			if (__predict_true(pi->ipi_ipproto == IPPROTO_TCP)) {
 				if (__predict_false(th == NULL)) {
 					txq->ift_pullups++;
 					if (__predict_false((m = m_pullup(m, (ip->ip_hl << 2) + sizeof(*th))) == NULL))
 						return (ENOMEM);
 					th = (struct tcphdr *)((caddr_t)ip + pi->ipi_ip_hlen);
 				}
 				pi->ipi_tcp_hflags = th->th_flags;
 				pi->ipi_tcp_hlen = th->th_off << 2;
 				pi->ipi_tcp_seq = th->th_seq;
 			}
 			if (IS_TSO4(pi)) {
 				if (__predict_false(ip->ip_p != IPPROTO_TCP))
 					return (ENXIO);
 				/*
 				 * TSO always requires hardware checksum offload.
 				 */
 				pi->ipi_csum_flags |= (CSUM_IP_TCP | CSUM_IP);
 				th->th_sum = in_pseudo(ip->ip_src.s_addr,
 						       ip->ip_dst.s_addr, htons(IPPROTO_TCP));
 				pi->ipi_tso_segsz = m->m_pkthdr.tso_segsz;
 				if (sctx->isc_flags & IFLIB_TSO_INIT_IP) {
 					ip->ip_sum = 0;
 					ip->ip_len = htons(pi->ipi_ip_hlen + pi->ipi_tcp_hlen + pi->ipi_tso_segsz);
 				}
 			}
 		}
 		if ((sctx->isc_flags & IFLIB_NEED_ZERO_CSUM) && (pi->ipi_csum_flags & CSUM_IP))
                        ip->ip_sum = 0;
 
 		break;
 	}
 #endif
 #ifdef INET6
 	case ETHERTYPE_IPV6:
 	{
 		struct ip6_hdr *ip6 = (struct ip6_hdr *)(m->m_data + pi->ipi_ehdrlen);
 		struct tcphdr *th;
 		pi->ipi_ip_hlen = sizeof(struct ip6_hdr);
 
 		if (__predict_false(m->m_len < pi->ipi_ehdrlen + sizeof(struct ip6_hdr))) {
 			txq->ift_pullups++;
 			if (__predict_false((m = m_pullup(m, pi->ipi_ehdrlen + sizeof(struct ip6_hdr))) == NULL))
 				return (ENOMEM);
 		}
 		th = (struct tcphdr *)((caddr_t)ip6 + pi->ipi_ip_hlen);
 
 		/* XXX-BZ this will go badly in case of ext hdrs. */
 		pi->ipi_ipproto = ip6->ip6_nxt;
 		pi->ipi_flags |= IPI_TX_IPV6;
 
 		/* TCP checksum offload may require TCP header length */
 		if (IS_TX_OFFLOAD6(pi)) {
 			if (pi->ipi_ipproto == IPPROTO_TCP) {
 				if (__predict_false(m->m_len < pi->ipi_ehdrlen + sizeof(struct ip6_hdr) + sizeof(struct tcphdr))) {
 					txq->ift_pullups++;
 					if (__predict_false((m = m_pullup(m, pi->ipi_ehdrlen + sizeof(struct ip6_hdr) + sizeof(struct tcphdr))) == NULL))
 						return (ENOMEM);
 				}
 				pi->ipi_tcp_hflags = th->th_flags;
 				pi->ipi_tcp_hlen = th->th_off << 2;
 				pi->ipi_tcp_seq = th->th_seq;
 			}
 			if (IS_TSO6(pi)) {
 				if (__predict_false(ip6->ip6_nxt != IPPROTO_TCP))
 					return (ENXIO);
 				/*
 				 * TSO always requires hardware checksum offload.
 				 */
 				pi->ipi_csum_flags |= CSUM_IP6_TCP;
 				th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
 				pi->ipi_tso_segsz = m->m_pkthdr.tso_segsz;
 			}
 		}
 		break;
 	}
 #endif
 	default:
 		pi->ipi_csum_flags &= ~CSUM_OFFLOAD;
 		pi->ipi_ip_hlen = 0;
 		break;
 	}
 	*mp = m;
 
 	return (0);
 }
 
 /*
  * If dodgy hardware rejects the scatter gather chain we've handed it
  * we'll need to remove the mbuf chain from ifsg_m[] before we can add the
  * m_defrag'd mbufs
  */
 static __noinline struct mbuf *
 iflib_remove_mbuf(iflib_txq_t txq)
 {
 	int ntxd, pidx;
 	struct mbuf *m, **ifsd_m;
 
 	ifsd_m = txq->ift_sds.ifsd_m;
 	ntxd = txq->ift_size;
 	pidx = txq->ift_pidx & (ntxd - 1);
 	ifsd_m = txq->ift_sds.ifsd_m;
 	m = ifsd_m[pidx];
 	ifsd_m[pidx] = NULL;
 	bus_dmamap_unload(txq->ift_buf_tag, txq->ift_sds.ifsd_map[pidx]);
 	if (txq->ift_sds.ifsd_tso_map != NULL)
 		bus_dmamap_unload(txq->ift_tso_buf_tag,
 		    txq->ift_sds.ifsd_tso_map[pidx]);
 #if MEMORY_LOGGING
 	txq->ift_dequeued++;
 #endif
 	return (m);
 }
 
 static inline caddr_t
 calc_next_txd(iflib_txq_t txq, int cidx, uint8_t qid)
 {
 	qidx_t size;
 	int ntxd;
 	caddr_t start, end, cur, next;
 
 	ntxd = txq->ift_size;
 	size = txq->ift_txd_size[qid];
 	start = txq->ift_ifdi[qid].idi_vaddr;
 
 	if (__predict_false(size == 0))
 		return (start);
 	cur = start + size*cidx;
 	end = start + size*ntxd;
 	next = CACHE_PTR_NEXT(cur);
 	return (next < end ? next : start);
 }
 
 /*
  * Pad an mbuf to ensure a minimum ethernet frame size.
  * min_frame_size is the frame size (less CRC) to pad the mbuf to
  */
 static __noinline int
 iflib_ether_pad(device_t dev, struct mbuf **m_head, uint16_t min_frame_size)
 {
 	/*
 	 * 18 is enough bytes to pad an ARP packet to 46 bytes, and
 	 * and ARP message is the smallest common payload I can think of
 	 */
 	static char pad[18];	/* just zeros */
 	int n;
 	struct mbuf *new_head;
 
 	if (!M_WRITABLE(*m_head)) {
 		new_head = m_dup(*m_head, M_NOWAIT);
 		if (new_head == NULL) {
 			m_freem(*m_head);
 			device_printf(dev, "cannot pad short frame, m_dup() failed");
 			DBG_COUNTER_INC(encap_pad_mbuf_fail);
 			DBG_COUNTER_INC(tx_frees);
 			return ENOMEM;
 		}
 		m_freem(*m_head);
 		*m_head = new_head;
 	}
 
 	for (n = min_frame_size - (*m_head)->m_pkthdr.len;
 	     n > 0; n -= sizeof(pad))
 		if (!m_append(*m_head, min(n, sizeof(pad)), pad))
 			break;
 
 	if (n > 0) {
 		m_freem(*m_head);
 		device_printf(dev, "cannot pad short frame\n");
 		DBG_COUNTER_INC(encap_pad_mbuf_fail);
 		DBG_COUNTER_INC(tx_frees);
 		return (ENOBUFS);
 	}
 
 	return 0;
 }
 
 static int
 iflib_encap(iflib_txq_t txq, struct mbuf **m_headp)
 {
 	if_ctx_t		ctx;
 	if_shared_ctx_t		sctx;
 	if_softc_ctx_t		scctx;
 	bus_dma_tag_t		buf_tag;
 	bus_dma_segment_t	*segs;
 	struct mbuf		*m_head, **ifsd_m;
 	void			*next_txd;
 	bus_dmamap_t		map;
 	struct if_pkt_info	pi;
 	int remap = 0;
 	int err, nsegs, ndesc, max_segs, pidx, cidx, next, ntxd;
 
 	ctx = txq->ift_ctx;
 	sctx = ctx->ifc_sctx;
 	scctx = &ctx->ifc_softc_ctx;
 	segs = txq->ift_segs;
 	ntxd = txq->ift_size;
 	m_head = *m_headp;
 	map = NULL;
 
 	/*
 	 * If we're doing TSO the next descriptor to clean may be quite far ahead
 	 */
 	cidx = txq->ift_cidx;
 	pidx = txq->ift_pidx;
 	if (ctx->ifc_flags & IFC_PREFETCH) {
 		next = (cidx + CACHE_PTR_INCREMENT) & (ntxd-1);
 		if (!(ctx->ifc_flags & IFLIB_HAS_TXCQ)) {
 			next_txd = calc_next_txd(txq, cidx, 0);
 			prefetch(next_txd);
 		}
 
 		/* prefetch the next cache line of mbuf pointers and flags */
 		prefetch(&txq->ift_sds.ifsd_m[next]);
 		prefetch(&txq->ift_sds.ifsd_map[next]);
 		next = (cidx + CACHE_LINE_SIZE) & (ntxd-1);
 	}
 	map = txq->ift_sds.ifsd_map[pidx];
 	ifsd_m = txq->ift_sds.ifsd_m;
 
 	if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
 		buf_tag = txq->ift_tso_buf_tag;
 		max_segs = scctx->isc_tx_tso_segments_max;
 		map = txq->ift_sds.ifsd_tso_map[pidx];
 		MPASS(buf_tag != NULL);
 		MPASS(max_segs > 0);
 	} else {
 		buf_tag = txq->ift_buf_tag;
 		max_segs = scctx->isc_tx_nsegments;
 		map = txq->ift_sds.ifsd_map[pidx];
 	}
 	if ((sctx->isc_flags & IFLIB_NEED_ETHER_PAD) &&
 	    __predict_false(m_head->m_pkthdr.len < scctx->isc_min_frame_size)) {
 		err = iflib_ether_pad(ctx->ifc_dev, m_headp, scctx->isc_min_frame_size);
 		if (err) {
 			DBG_COUNTER_INC(encap_txd_encap_fail);
 			return err;
 		}
 	}
 	m_head = *m_headp;
 
 	pkt_info_zero(&pi);
 	pi.ipi_mflags = (m_head->m_flags & (M_VLANTAG|M_BCAST|M_MCAST));
 	pi.ipi_pidx = pidx;
 	pi.ipi_qsidx = txq->ift_id;
 	pi.ipi_len = m_head->m_pkthdr.len;
 	pi.ipi_csum_flags = m_head->m_pkthdr.csum_flags;
 	pi.ipi_vtag = (m_head->m_flags & M_VLANTAG) ? m_head->m_pkthdr.ether_vtag : 0;
 
 	/* deliberate bitwise OR to make one condition */
 	if (__predict_true((pi.ipi_csum_flags | pi.ipi_vtag))) {
 		if (__predict_false((err = iflib_parse_header(txq, &pi, m_headp)) != 0)) {
 			DBG_COUNTER_INC(encap_txd_encap_fail);
 			return (err);
 		}
 		m_head = *m_headp;
 	}
 
 retry:
 	err = bus_dmamap_load_mbuf_sg(buf_tag, map, m_head, segs, &nsegs,
 	    BUS_DMA_NOWAIT);
 defrag:
 	if (__predict_false(err)) {
 		switch (err) {
 		case EFBIG:
 			/* try collapse once and defrag once */
 			if (remap == 0) {
 				m_head = m_collapse(*m_headp, M_NOWAIT, max_segs);
 				/* try defrag if collapsing fails */
 				if (m_head == NULL)
 					remap++;
 			}
 			if (remap == 1) {
 				txq->ift_mbuf_defrag++;
 				m_head = m_defrag(*m_headp, M_NOWAIT);
 			}
 			/*
 			 * remap should never be >1 unless bus_dmamap_load_mbuf_sg
 			 * failed to map an mbuf that was run through m_defrag
 			 */
 			MPASS(remap <= 1);
 			if (__predict_false(m_head == NULL || remap > 1))
 				goto defrag_failed;
 			remap++;
 			*m_headp = m_head;
 			goto retry;
 			break;
 		case ENOMEM:
 			txq->ift_no_tx_dma_setup++;
 			break;
 		default:
 			txq->ift_no_tx_dma_setup++;
 			m_freem(*m_headp);
 			DBG_COUNTER_INC(tx_frees);
 			*m_headp = NULL;
 			break;
 		}
 		txq->ift_map_failed++;
 		DBG_COUNTER_INC(encap_load_mbuf_fail);
 		DBG_COUNTER_INC(encap_txd_encap_fail);
 		return (err);
 	}
 	ifsd_m[pidx] = m_head;
 	/*
 	 * XXX assumes a 1 to 1 relationship between segments and
 	 *        descriptors - this does not hold true on all drivers, e.g.
 	 *        cxgb
 	 */
 	if (__predict_false(nsegs + 2 > TXQ_AVAIL(txq))) {
 		txq->ift_no_desc_avail++;
 		bus_dmamap_unload(buf_tag, map);
 		DBG_COUNTER_INC(encap_txq_avail_fail);
 		DBG_COUNTER_INC(encap_txd_encap_fail);
 		if ((txq->ift_task.gt_task.ta_flags & TASK_ENQUEUED) == 0)
 			GROUPTASK_ENQUEUE(&txq->ift_task);
 		return (ENOBUFS);
 	}
 	/*
 	 * On Intel cards we can greatly reduce the number of TX interrupts
 	 * we see by only setting report status on every Nth descriptor.
 	 * However, this also means that the driver will need to keep track
 	 * of the descriptors that RS was set on to check them for the DD bit.
 	 */
 	txq->ift_rs_pending += nsegs + 1;
 	if (txq->ift_rs_pending > TXQ_MAX_RS_DEFERRED(txq) ||
 	     iflib_no_tx_batch || (TXQ_AVAIL(txq) - nsegs) <= MAX_TX_DESC(ctx) + 2) {
 		pi.ipi_flags |= IPI_TX_INTR;
 		txq->ift_rs_pending = 0;
 	}
 
 	pi.ipi_segs = segs;
 	pi.ipi_nsegs = nsegs;
 
 	MPASS(pidx >= 0 && pidx < txq->ift_size);
 #ifdef PKT_DEBUG
 	print_pkt(&pi);
 #endif
 	if ((err = ctx->isc_txd_encap(ctx->ifc_softc, &pi)) == 0) {
 		bus_dmamap_sync(buf_tag, map, BUS_DMASYNC_PREWRITE);
 		DBG_COUNTER_INC(tx_encap);
 		MPASS(pi.ipi_new_pidx < txq->ift_size);
 
 		ndesc = pi.ipi_new_pidx - pi.ipi_pidx;
 		if (pi.ipi_new_pidx < pi.ipi_pidx) {
 			ndesc += txq->ift_size;
 			txq->ift_gen = 1;
 		}
 		/*
 		 * drivers can need as many as 
 		 * two sentinels
 		 */
 		MPASS(ndesc <= pi.ipi_nsegs + 2);
 		MPASS(pi.ipi_new_pidx != pidx);
 		MPASS(ndesc > 0);
 		txq->ift_in_use += ndesc;
 
 		/*
 		 * We update the last software descriptor again here because there may
 		 * be a sentinel and/or there may be more mbufs than segments
 		 */
 		txq->ift_pidx = pi.ipi_new_pidx;
 		txq->ift_npending += pi.ipi_ndescs;
 	} else {
 		*m_headp = m_head = iflib_remove_mbuf(txq);
 		if (err == EFBIG) {
 			txq->ift_txd_encap_efbig++;
 			if (remap < 2) {
 				remap = 1;
 				goto defrag;
 			}
 		}
 		goto defrag_failed;
 	}
 	/*
 	 * err can't possibly be non-zero here, so we don't neet to test it
 	 * to see if we need to DBG_COUNTER_INC(encap_txd_encap_fail).
 	 */
 	return (err);
 
 defrag_failed:
 	txq->ift_mbuf_defrag_failed++;
 	txq->ift_map_failed++;
 	m_freem(*m_headp);
 	DBG_COUNTER_INC(tx_frees);
 	*m_headp = NULL;
 	DBG_COUNTER_INC(encap_txd_encap_fail);
 	return (ENOMEM);
 }
 
 static void
 iflib_tx_desc_free(iflib_txq_t txq, int n)
 {
 	uint32_t qsize, cidx, mask, gen;
 	struct mbuf *m, **ifsd_m;
 	bool do_prefetch;
 
 	cidx = txq->ift_cidx;
 	gen = txq->ift_gen;
 	qsize = txq->ift_size;
 	mask = qsize-1;
 	ifsd_m = txq->ift_sds.ifsd_m;
 	do_prefetch = (txq->ift_ctx->ifc_flags & IFC_PREFETCH);
 
 	while (n-- > 0) {
 		if (do_prefetch) {
 			prefetch(ifsd_m[(cidx + 3) & mask]);
 			prefetch(ifsd_m[(cidx + 4) & mask]);
 		}
 		if ((m = ifsd_m[cidx]) != NULL) {
 			prefetch(&ifsd_m[(cidx + CACHE_PTR_INCREMENT) & mask]);
 			if (m->m_pkthdr.csum_flags & CSUM_TSO) {
 				bus_dmamap_sync(txq->ift_tso_buf_tag,
 				    txq->ift_sds.ifsd_tso_map[cidx],
 				    BUS_DMASYNC_POSTWRITE);
 				bus_dmamap_unload(txq->ift_tso_buf_tag,
 				    txq->ift_sds.ifsd_tso_map[cidx]);
 			} else {
 				bus_dmamap_sync(txq->ift_buf_tag,
 				    txq->ift_sds.ifsd_map[cidx],
 				    BUS_DMASYNC_POSTWRITE);
 				bus_dmamap_unload(txq->ift_buf_tag,
 				    txq->ift_sds.ifsd_map[cidx]);
 			}
 			/* XXX we don't support any drivers that batch packets yet */
 			MPASS(m->m_nextpkt == NULL);
 			m_freem(m);
 			ifsd_m[cidx] = NULL;
 #if MEMORY_LOGGING
 			txq->ift_dequeued++;
 #endif
 			DBG_COUNTER_INC(tx_frees);
 		}
 		if (__predict_false(++cidx == qsize)) {
 			cidx = 0;
 			gen = 0;
 		}
 	}
 	txq->ift_cidx = cidx;
 	txq->ift_gen = gen;
 }
 
 static __inline int
 iflib_completed_tx_reclaim(iflib_txq_t txq, int thresh)
 {
 	int reclaim;
 	if_ctx_t ctx = txq->ift_ctx;
 
 	KASSERT(thresh >= 0, ("invalid threshold to reclaim"));
 	MPASS(thresh /*+ MAX_TX_DESC(txq->ift_ctx) */ < txq->ift_size);
 
 	/*
 	 * Need a rate-limiting check so that this isn't called every time
 	 */
 	iflib_tx_credits_update(ctx, txq);
 	reclaim = DESC_RECLAIMABLE(txq);
 
 	if (reclaim <= thresh /* + MAX_TX_DESC(txq->ift_ctx) */) {
 #ifdef INVARIANTS
 		if (iflib_verbose_debug) {
 			printf("%s processed=%ju cleaned=%ju tx_nsegments=%d reclaim=%d thresh=%d\n", __FUNCTION__,
 			       txq->ift_processed, txq->ift_cleaned, txq->ift_ctx->ifc_softc_ctx.isc_tx_nsegments,
 			       reclaim, thresh);
 
 		}
 #endif
 		return (0);
 	}
 	iflib_tx_desc_free(txq, reclaim);
 	txq->ift_cleaned += reclaim;
 	txq->ift_in_use -= reclaim;
 
 	return (reclaim);
 }
 
 static struct mbuf **
 _ring_peek_one(struct ifmp_ring *r, int cidx, int offset, int remaining)
 {
 	int next, size;
 	struct mbuf **items;
 
 	size = r->size;
 	next = (cidx + CACHE_PTR_INCREMENT) & (size-1);
 	items = __DEVOLATILE(struct mbuf **, &r->items[0]);
 
 	prefetch(items[(cidx + offset) & (size-1)]);
 	if (remaining > 1) {
 		prefetch2cachelines(&items[next]);
 		prefetch2cachelines(items[(cidx + offset + 1) & (size-1)]);
 		prefetch2cachelines(items[(cidx + offset + 2) & (size-1)]);
 		prefetch2cachelines(items[(cidx + offset + 3) & (size-1)]);
 	}
 	return (__DEVOLATILE(struct mbuf **, &r->items[(cidx + offset) & (size-1)]));
 }
 
 static void
 iflib_txq_check_drain(iflib_txq_t txq, int budget)
 {
 
 	ifmp_ring_check_drainage(txq->ift_br, budget);
 }
 
 static uint32_t
 iflib_txq_can_drain(struct ifmp_ring *r)
 {
 	iflib_txq_t txq = r->cookie;
 	if_ctx_t ctx = txq->ift_ctx;
 
 	if (TXQ_AVAIL(txq) > MAX_TX_DESC(ctx) + 2)
 		return (1);
 	bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,
 	    BUS_DMASYNC_POSTREAD);
 	return (ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id,
 	    false));
 }
 
 static uint32_t
 iflib_txq_drain(struct ifmp_ring *r, uint32_t cidx, uint32_t pidx)
 {
 	iflib_txq_t txq = r->cookie;
 	if_ctx_t ctx = txq->ift_ctx;
 	struct ifnet *ifp = ctx->ifc_ifp;
 	struct mbuf **mp, *m;
 	int i, count, consumed, pkt_sent, bytes_sent, mcast_sent, avail;
 	int reclaimed, err, in_use_prev, desc_used;
 	bool do_prefetch, ring, rang;
 
 	if (__predict_false(!(if_getdrvflags(ifp) & IFF_DRV_RUNNING) ||
 			    !LINK_ACTIVE(ctx))) {
 		DBG_COUNTER_INC(txq_drain_notready);
 		return (0);
 	}
 	reclaimed = iflib_completed_tx_reclaim(txq, RECLAIM_THRESH(ctx));
 	rang = iflib_txd_db_check(ctx, txq, reclaimed, txq->ift_in_use);
 	avail = IDXDIFF(pidx, cidx, r->size);
 	if (__predict_false(ctx->ifc_flags & IFC_QFLUSH)) {
 		DBG_COUNTER_INC(txq_drain_flushing);
 		for (i = 0; i < avail; i++) {
 			if (__predict_true(r->items[(cidx + i) & (r->size-1)] != (void *)txq))
 				m_free(r->items[(cidx + i) & (r->size-1)]);
 			r->items[(cidx + i) & (r->size-1)] = NULL;
 		}
 		return (avail);
 	}
 
 	if (__predict_false(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_OACTIVE)) {
 		txq->ift_qstatus = IFLIB_QUEUE_IDLE;
 		CALLOUT_LOCK(txq);
 		callout_stop(&txq->ift_timer);
 		CALLOUT_UNLOCK(txq);
 		DBG_COUNTER_INC(txq_drain_oactive);
 		return (0);
 	}
 	if (reclaimed)
 		txq->ift_qstatus = IFLIB_QUEUE_IDLE;
 	consumed = mcast_sent = bytes_sent = pkt_sent = 0;
 	count = MIN(avail, TX_BATCH_SIZE);
 #ifdef INVARIANTS
 	if (iflib_verbose_debug)
 		printf("%s avail=%d ifc_flags=%x txq_avail=%d ", __FUNCTION__,
 		       avail, ctx->ifc_flags, TXQ_AVAIL(txq));
 #endif
 	do_prefetch = (ctx->ifc_flags & IFC_PREFETCH);
 	avail = TXQ_AVAIL(txq);
 	err = 0;
 	for (desc_used = i = 0; i < count && avail > MAX_TX_DESC(ctx) + 2; i++) {
 		int rem = do_prefetch ? count - i : 0;
 
 		mp = _ring_peek_one(r, cidx, i, rem);
 		MPASS(mp != NULL && *mp != NULL);
 		if (__predict_false(*mp == (struct mbuf *)txq)) {
 			consumed++;
 			reclaimed++;
 			continue;
 		}
 		in_use_prev = txq->ift_in_use;
 		err = iflib_encap(txq, mp);
 		if (__predict_false(err)) {
 			/* no room - bail out */
 			if (err == ENOBUFS)
 				break;
 			consumed++;
 			/* we can't send this packet - skip it */
 			continue;
 		}
 		consumed++;
 		pkt_sent++;
 		m = *mp;
 		DBG_COUNTER_INC(tx_sent);
 		bytes_sent += m->m_pkthdr.len;
 		mcast_sent += !!(m->m_flags & M_MCAST);
 		avail = TXQ_AVAIL(txq);
 
 		txq->ift_db_pending += (txq->ift_in_use - in_use_prev);
 		desc_used += (txq->ift_in_use - in_use_prev);
 		ETHER_BPF_MTAP(ifp, m);
 		if (__predict_false(!(ifp->if_drv_flags & IFF_DRV_RUNNING)))
 			break;
 		rang = iflib_txd_db_check(ctx, txq, false, in_use_prev);
 	}
 
 	/* deliberate use of bitwise or to avoid gratuitous short-circuit */
 	ring = rang ? false  : (iflib_min_tx_latency | err) || (TXQ_AVAIL(txq) < MAX_TX_DESC(ctx));
 	iflib_txd_db_check(ctx, txq, ring, txq->ift_in_use);
 	if_inc_counter(ifp, IFCOUNTER_OBYTES, bytes_sent);
 	if_inc_counter(ifp, IFCOUNTER_OPACKETS, pkt_sent);
 	if (mcast_sent)
 		if_inc_counter(ifp, IFCOUNTER_OMCASTS, mcast_sent);
 #ifdef INVARIANTS
 	if (iflib_verbose_debug)
 		printf("consumed=%d\n", consumed);
 #endif
 	return (consumed);
 }
 
 static uint32_t
 iflib_txq_drain_always(struct ifmp_ring *r)
 {
 	return (1);
 }
 
 static uint32_t
 iflib_txq_drain_free(struct ifmp_ring *r, uint32_t cidx, uint32_t pidx)
 {
 	int i, avail;
 	struct mbuf **mp;
 	iflib_txq_t txq;
 
 	txq = r->cookie;
 
 	txq->ift_qstatus = IFLIB_QUEUE_IDLE;
 	CALLOUT_LOCK(txq);
 	callout_stop(&txq->ift_timer);
 	CALLOUT_UNLOCK(txq);
 
 	avail = IDXDIFF(pidx, cidx, r->size);
 	for (i = 0; i < avail; i++) {
 		mp = _ring_peek_one(r, cidx, i, avail - i);
 		if (__predict_false(*mp == (struct mbuf *)txq))
 			continue;
 		m_freem(*mp);
 		DBG_COUNTER_INC(tx_frees);
 	}
 	MPASS(ifmp_ring_is_stalled(r) == 0);
 	return (avail);
 }
 
 static void
 iflib_ifmp_purge(iflib_txq_t txq)
 {
 	struct ifmp_ring *r;
 
 	r = txq->ift_br;
 	r->drain = iflib_txq_drain_free;
 	r->can_drain = iflib_txq_drain_always;
 
 	ifmp_ring_check_drainage(r, r->size);
 
 	r->drain = iflib_txq_drain;
 	r->can_drain = iflib_txq_can_drain;
 }
 
 static void
 _task_fn_tx(void *context)
 {
 	iflib_txq_t txq = context;
 	if_ctx_t ctx = txq->ift_ctx;
 #if defined(ALTQ) || defined(DEV_NETMAP)
 	if_t ifp = ctx->ifc_ifp;
 #endif
 	int abdicate = ctx->ifc_sysctl_tx_abdicate;
 
 #ifdef IFLIB_DIAGNOSTICS
 	txq->ift_cpu_exec_count[curcpu]++;
 #endif
 	if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING))
 		return;
 #ifdef DEV_NETMAP
 	if (if_getcapenable(ifp) & IFCAP_NETMAP) {
 		bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,
 		    BUS_DMASYNC_POSTREAD);
 		if (ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id, false))
 			netmap_tx_irq(ifp, txq->ift_id);
 		IFDI_TX_QUEUE_INTR_ENABLE(ctx, txq->ift_id);
 		return;
 	}
 #endif
 #ifdef ALTQ
 	if (ALTQ_IS_ENABLED(&ifp->if_snd))
 		iflib_altq_if_start(ifp);
 #endif
 	if (txq->ift_db_pending)
 		ifmp_ring_enqueue(txq->ift_br, (void **)&txq, 1, TX_BATCH_SIZE, abdicate);
 	else if (!abdicate)
 		ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE);
 	/*
 	 * When abdicating, we always need to check drainage, not just when we don't enqueue
 	 */
 	if (abdicate)
 		ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE);
 	if (ctx->ifc_flags & IFC_LEGACY)
 		IFDI_INTR_ENABLE(ctx);
 	else {
 #ifdef INVARIANTS
 		int rc =
 #endif
 			IFDI_TX_QUEUE_INTR_ENABLE(ctx, txq->ift_id);
 			KASSERT(rc != ENOTSUP, ("MSI-X support requires queue_intr_enable, but not implemented in driver"));
 	}
 }
 
 static void
 _task_fn_rx(void *context)
 {
 	iflib_rxq_t rxq = context;
 	if_ctx_t ctx = rxq->ifr_ctx;
 	bool more;
 	uint16_t budget;
 
 #ifdef IFLIB_DIAGNOSTICS
 	rxq->ifr_cpu_exec_count[curcpu]++;
 #endif
 	DBG_COUNTER_INC(task_fn_rxs);
 	if (__predict_false(!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)))
 		return;
 	more = true;
 #ifdef DEV_NETMAP
 	if (if_getcapenable(ctx->ifc_ifp) & IFCAP_NETMAP) {
 		u_int work = 0;
 		if (netmap_rx_irq(ctx->ifc_ifp, rxq->ifr_id, &work)) {
 			more = false;
 		}
 	}
 #endif
 	budget = ctx->ifc_sysctl_rx_budget;
 	if (budget == 0)
 		budget = 16;	/* XXX */
 	if (more == false || (more = iflib_rxeof(rxq, budget)) == false) {
 		if (ctx->ifc_flags & IFC_LEGACY)
 			IFDI_INTR_ENABLE(ctx);
 		else {
 #ifdef INVARIANTS
 			int rc =
 #endif
 				IFDI_RX_QUEUE_INTR_ENABLE(ctx, rxq->ifr_id);
 			KASSERT(rc != ENOTSUP, ("MSI-X support requires queue_intr_enable, but not implemented in driver"));
 			DBG_COUNTER_INC(rx_intr_enables);
 		}
 	}
 	if (__predict_false(!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)))
 		return;
 	if (more)
 		GROUPTASK_ENQUEUE(&rxq->ifr_task);
 }
 
 static void
 _task_fn_admin(void *context)
 {
 	if_ctx_t ctx = context;
 	if_softc_ctx_t sctx = &ctx->ifc_softc_ctx;
 	iflib_txq_t txq;
 	int i;
 	bool oactive, running, do_reset, do_watchdog, in_detach;
 	uint32_t reset_on = hz / 2;
 
 	STATE_LOCK(ctx);
 	running = (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING);
 	oactive = (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_OACTIVE);
 	do_reset = (ctx->ifc_flags & IFC_DO_RESET);
 	do_watchdog = (ctx->ifc_flags & IFC_DO_WATCHDOG);
 	in_detach = (ctx->ifc_flags & IFC_IN_DETACH);
 	ctx->ifc_flags &= ~(IFC_DO_RESET|IFC_DO_WATCHDOG);
 	STATE_UNLOCK(ctx);
 
 	if ((!running && !oactive) && !(ctx->ifc_sctx->isc_flags & IFLIB_ADMIN_ALWAYS_RUN))
 		return;
 	if (in_detach)
 		return;
 
 	CTX_LOCK(ctx);
 	for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) {
 		CALLOUT_LOCK(txq);
 		callout_stop(&txq->ift_timer);
 		CALLOUT_UNLOCK(txq);
 	}
 	if (do_watchdog) {
 		ctx->ifc_watchdog_events++;
 		IFDI_WATCHDOG_RESET(ctx);
 	}
 	IFDI_UPDATE_ADMIN_STATUS(ctx);
 	for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) {
 #ifdef DEV_NETMAP
 		reset_on = hz / 2;
 		if (if_getcapenable(ctx->ifc_ifp) & IFCAP_NETMAP)
 			iflib_netmap_timer_adjust(ctx, txq, &reset_on);
 #endif
 		callout_reset_on(&txq->ift_timer, reset_on, iflib_timer, txq, txq->ift_timer.c_cpu);
 	}
 	IFDI_LINK_INTR_ENABLE(ctx);
 	if (do_reset)
 		iflib_if_init_locked(ctx);
 	CTX_UNLOCK(ctx);
 
 	if (LINK_ACTIVE(ctx) == 0)
 		return;
 	for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++)
 		iflib_txq_check_drain(txq, IFLIB_RESTART_BUDGET);
 }
 
 
 static void
 _task_fn_iov(void *context)
 {
 	if_ctx_t ctx = context;
 
 	if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING) &&
 	    !(ctx->ifc_sctx->isc_flags & IFLIB_ADMIN_ALWAYS_RUN))
 		return;
 
 	CTX_LOCK(ctx);
 	IFDI_VFLR_HANDLE(ctx);
 	CTX_UNLOCK(ctx);
 }
 
 static int
 iflib_sysctl_int_delay(SYSCTL_HANDLER_ARGS)
 {
 	int err;
 	if_int_delay_info_t info;
 	if_ctx_t ctx;
 
 	info = (if_int_delay_info_t)arg1;
 	ctx = info->iidi_ctx;
 	info->iidi_req = req;
 	info->iidi_oidp = oidp;
 	CTX_LOCK(ctx);
 	err = IFDI_SYSCTL_INT_DELAY(ctx, info);
 	CTX_UNLOCK(ctx);
 	return (err);
 }
 
 /*********************************************************************
  *
  *  IFNET FUNCTIONS
  *
  **********************************************************************/
 
 static void
 iflib_if_init_locked(if_ctx_t ctx)
 {
 	iflib_stop(ctx);
 	iflib_init_locked(ctx);
 }
 
 
 static void
 iflib_if_init(void *arg)
 {
 	if_ctx_t ctx = arg;
 
 	CTX_LOCK(ctx);
 	iflib_if_init_locked(ctx);
 	CTX_UNLOCK(ctx);
 }
 
 static int
 iflib_if_transmit(if_t ifp, struct mbuf *m)
 {
 	if_ctx_t	ctx = if_getsoftc(ifp);
 
 	iflib_txq_t txq;
 	int err, qidx;
 	int abdicate = ctx->ifc_sysctl_tx_abdicate;
 
 	if (__predict_false((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || !LINK_ACTIVE(ctx))) {
 		DBG_COUNTER_INC(tx_frees);
 		m_freem(m);
 		return (ENOBUFS);
 	}
 
 	MPASS(m->m_nextpkt == NULL);
 	/* ALTQ-enabled interfaces always use queue 0. */
 	qidx = 0;
 	if ((NTXQSETS(ctx) > 1) && M_HASHTYPE_GET(m) && !ALTQ_IS_ENABLED(&ifp->if_snd))
 		qidx = QIDX(ctx, m);
 	/*
 	 * XXX calculate buf_ring based on flowid (divvy up bits?)
 	 */
 	txq = &ctx->ifc_txqs[qidx];
 
 #ifdef DRIVER_BACKPRESSURE
 	if (txq->ift_closed) {
 		while (m != NULL) {
 			next = m->m_nextpkt;
 			m->m_nextpkt = NULL;
 			m_freem(m);
 			DBG_COUNTER_INC(tx_frees);
 			m = next;
 		}
 		return (ENOBUFS);
 	}
 #endif
 #ifdef notyet
 	qidx = count = 0;
 	mp = marr;
 	next = m;
 	do {
 		count++;
 		next = next->m_nextpkt;
 	} while (next != NULL);
 
 	if (count > nitems(marr))
 		if ((mp = malloc(count*sizeof(struct mbuf *), M_IFLIB, M_NOWAIT)) == NULL) {
 			/* XXX check nextpkt */
 			m_freem(m);
 			/* XXX simplify for now */
 			DBG_COUNTER_INC(tx_frees);
 			return (ENOBUFS);
 		}
 	for (next = m, i = 0; next != NULL; i++) {
 		mp[i] = next;
 		next = next->m_nextpkt;
 		mp[i]->m_nextpkt = NULL;
 	}
 #endif
 	DBG_COUNTER_INC(tx_seen);
 	err = ifmp_ring_enqueue(txq->ift_br, (void **)&m, 1, TX_BATCH_SIZE, abdicate);
 
 	if (abdicate)
 		GROUPTASK_ENQUEUE(&txq->ift_task);
  	if (err) {
 		if (!abdicate)
 			GROUPTASK_ENQUEUE(&txq->ift_task);
 		/* support forthcoming later */
 #ifdef DRIVER_BACKPRESSURE
 		txq->ift_closed = TRUE;
 #endif
 		ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE);
 		m_freem(m);
 		DBG_COUNTER_INC(tx_frees);
 	}
 
 	return (err);
 }
 
 #ifdef ALTQ
 /*
  * The overall approach to integrating iflib with ALTQ is to continue to use
  * the iflib mp_ring machinery between the ALTQ queue(s) and the hardware
  * ring.  Technically, when using ALTQ, queueing to an intermediate mp_ring
  * is redundant/unnecessary, but doing so minimizes the amount of
  * ALTQ-specific code required in iflib.  It is assumed that the overhead of
  * redundantly queueing to an intermediate mp_ring is swamped by the
  * performance limitations inherent in using ALTQ.
  *
  * When ALTQ support is compiled in, all iflib drivers will use a transmit
  * routine, iflib_altq_if_transmit(), that checks if ALTQ is enabled for the
  * given interface.  If ALTQ is enabled for an interface, then all
  * transmitted packets for that interface will be submitted to the ALTQ
  * subsystem via IFQ_ENQUEUE().  We don't use the legacy if_transmit()
  * implementation because it uses IFQ_HANDOFF(), which will duplicatively
  * update stats that the iflib machinery handles, and which is sensitve to
  * the disused IFF_DRV_OACTIVE flag.  Additionally, iflib_altq_if_start()
  * will be installed as the start routine for use by ALTQ facilities that
  * need to trigger queue drains on a scheduled basis.
  *
  */
 static void
 iflib_altq_if_start(if_t ifp)
 {
 	struct ifaltq *ifq = &ifp->if_snd;
 	struct mbuf *m;
 	
 	IFQ_LOCK(ifq);
 	IFQ_DEQUEUE_NOLOCK(ifq, m);
 	while (m != NULL) {
 		iflib_if_transmit(ifp, m);
 		IFQ_DEQUEUE_NOLOCK(ifq, m);
 	}
 	IFQ_UNLOCK(ifq);
 }
 
 static int
 iflib_altq_if_transmit(if_t ifp, struct mbuf *m)
 {
 	int err;
 
 	if (ALTQ_IS_ENABLED(&ifp->if_snd)) {
 		IFQ_ENQUEUE(&ifp->if_snd, m, err);
 		if (err == 0)
 			iflib_altq_if_start(ifp);
 	} else
 		err = iflib_if_transmit(ifp, m);
 
 	return (err);
 }
 #endif /* ALTQ */
 
 static void
 iflib_if_qflush(if_t ifp)
 {
 	if_ctx_t ctx = if_getsoftc(ifp);
 	iflib_txq_t txq = ctx->ifc_txqs;
 	int i;
 
 	STATE_LOCK(ctx);
 	ctx->ifc_flags |= IFC_QFLUSH;
 	STATE_UNLOCK(ctx);
 	for (i = 0; i < NTXQSETS(ctx); i++, txq++)
 		while (!(ifmp_ring_is_idle(txq->ift_br) || ifmp_ring_is_stalled(txq->ift_br)))
 			iflib_txq_check_drain(txq, 0);
 	STATE_LOCK(ctx);
 	ctx->ifc_flags &= ~IFC_QFLUSH;
 	STATE_UNLOCK(ctx);
 
 	/*
 	 * When ALTQ is enabled, this will also take care of purging the
 	 * ALTQ queue(s).
 	 */
 	if_qflush(ifp);
 }
 
 
 #define IFCAP_FLAGS (IFCAP_HWCSUM_IPV6 | IFCAP_HWCSUM | IFCAP_LRO | \
 		     IFCAP_TSO | IFCAP_VLAN_HWTAGGING | IFCAP_HWSTATS | \
 		     IFCAP_VLAN_MTU | IFCAP_VLAN_HWFILTER | \
 		     IFCAP_VLAN_HWTSO | IFCAP_VLAN_HWCSUM)
 
 static int
 iflib_if_ioctl(if_t ifp, u_long command, caddr_t data)
 {
 	if_ctx_t ctx = if_getsoftc(ifp);
 	struct ifreq	*ifr = (struct ifreq *)data;
 #if defined(INET) || defined(INET6)
 	struct ifaddr	*ifa = (struct ifaddr *)data;
 #endif
 	bool		avoid_reset = FALSE;
 	int		err = 0, reinit = 0, bits;
 
 	switch (command) {
 	case SIOCSIFADDR:
 #ifdef INET
 		if (ifa->ifa_addr->sa_family == AF_INET)
 			avoid_reset = TRUE;
 #endif
 #ifdef INET6
 		if (ifa->ifa_addr->sa_family == AF_INET6)
 			avoid_reset = TRUE;
 #endif
 		/*
 		** Calling init results in link renegotiation,
 		** so we avoid doing it when possible.
 		*/
 		if (avoid_reset) {
 			if_setflagbits(ifp, IFF_UP,0);
 			if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING))
 				reinit = 1;
 #ifdef INET
 			if (!(if_getflags(ifp) & IFF_NOARP))
 				arp_ifinit(ifp, ifa);
 #endif
 		} else
 			err = ether_ioctl(ifp, command, data);
 		break;
 	case SIOCSIFMTU:
 		CTX_LOCK(ctx);
 		if (ifr->ifr_mtu == if_getmtu(ifp)) {
 			CTX_UNLOCK(ctx);
 			break;
 		}
 		bits = if_getdrvflags(ifp);
 		/* stop the driver and free any clusters before proceeding */
 		iflib_stop(ctx);
 
 		if ((err = IFDI_MTU_SET(ctx, ifr->ifr_mtu)) == 0) {
 			STATE_LOCK(ctx);
 			if (ifr->ifr_mtu > ctx->ifc_max_fl_buf_size)
 				ctx->ifc_flags |= IFC_MULTISEG;
 			else
 				ctx->ifc_flags &= ~IFC_MULTISEG;
 			STATE_UNLOCK(ctx);
 			err = if_setmtu(ifp, ifr->ifr_mtu);
 		}
 		iflib_init_locked(ctx);
 		STATE_LOCK(ctx);
 		if_setdrvflags(ifp, bits);
 		STATE_UNLOCK(ctx);
 		CTX_UNLOCK(ctx);
 		break;
 	case SIOCSIFFLAGS:
 		CTX_LOCK(ctx);
 		if (if_getflags(ifp) & IFF_UP) {
 			if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
 				if ((if_getflags(ifp) ^ ctx->ifc_if_flags) &
 				    (IFF_PROMISC | IFF_ALLMULTI)) {
 					err = IFDI_PROMISC_SET(ctx, if_getflags(ifp));
 				}
 			} else
 				reinit = 1;
 		} else if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
 			iflib_stop(ctx);
 		}
 		ctx->ifc_if_flags = if_getflags(ifp);
 		CTX_UNLOCK(ctx);
 		break;
 	case SIOCADDMULTI:
 	case SIOCDELMULTI:
 		if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
 			CTX_LOCK(ctx);
 			IFDI_INTR_DISABLE(ctx);
 			IFDI_MULTI_SET(ctx);
 			IFDI_INTR_ENABLE(ctx);
 			CTX_UNLOCK(ctx);
 		}
 		break;
 	case SIOCSIFMEDIA:
 		CTX_LOCK(ctx);
 		IFDI_MEDIA_SET(ctx);
 		CTX_UNLOCK(ctx);
 		/* falls thru */
 	case SIOCGIFMEDIA:
 	case SIOCGIFXMEDIA:
 		err = ifmedia_ioctl(ifp, ifr, &ctx->ifc_media, command);
 		break;
 	case SIOCGI2C:
 	{
 		struct ifi2creq i2c;
 
 		err = copyin(ifr_data_get_ptr(ifr), &i2c, sizeof(i2c));
 		if (err != 0)
 			break;
 		if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) {
 			err = EINVAL;
 			break;
 		}
 		if (i2c.len > sizeof(i2c.data)) {
 			err = EINVAL;
 			break;
 		}
 
 		if ((err = IFDI_I2C_REQ(ctx, &i2c)) == 0)
 			err = copyout(&i2c, ifr_data_get_ptr(ifr),
 			    sizeof(i2c));
 		break;
 	}
 	case SIOCSIFCAP:
 	{
 		int mask, setmask, oldmask;
 
 		oldmask = if_getcapenable(ifp);
 		mask = ifr->ifr_reqcap ^ oldmask;
 		mask &= ctx->ifc_softc_ctx.isc_capabilities;
 		setmask = 0;
 #ifdef TCP_OFFLOAD
 		setmask |= mask & (IFCAP_TOE4|IFCAP_TOE6);
 #endif
 		setmask |= (mask & IFCAP_FLAGS);
 		setmask |= (mask & IFCAP_WOL);
 
 		/*
 		 * If any RX csum has changed, change all the ones that
 		 * are supported by the driver.
 		 */
 		if (setmask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) {
 			setmask |= ctx->ifc_softc_ctx.isc_capabilities &
 			    (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6);
 		}
 
 		/*
 		 * want to ensure that traffic has stopped before we change any of the flags
 		 */
 		if (setmask) {
 			CTX_LOCK(ctx);
 			bits = if_getdrvflags(ifp);
 			if (bits & IFF_DRV_RUNNING && setmask & ~IFCAP_WOL)
 				iflib_stop(ctx);
 			STATE_LOCK(ctx);
 			if_togglecapenable(ifp, setmask);
 			STATE_UNLOCK(ctx);
 			if (bits & IFF_DRV_RUNNING && setmask & ~IFCAP_WOL)
 				iflib_init_locked(ctx);
 			STATE_LOCK(ctx);
 			if_setdrvflags(ifp, bits);
 			STATE_UNLOCK(ctx);
 			CTX_UNLOCK(ctx);
 		}
 		if_vlancap(ifp);
 		break;
 	}
 	case SIOCGPRIVATE_0:
 	case SIOCSDRVSPEC:
 	case SIOCGDRVSPEC:
 		CTX_LOCK(ctx);
 		err = IFDI_PRIV_IOCTL(ctx, command, data);
 		CTX_UNLOCK(ctx);
 		break;
 	default:
 		err = ether_ioctl(ifp, command, data);
 		break;
 	}
 	if (reinit)
 		iflib_if_init(ctx);
 	return (err);
 }
 
 static uint64_t
 iflib_if_get_counter(if_t ifp, ift_counter cnt)
 {
 	if_ctx_t ctx = if_getsoftc(ifp);
 
 	return (IFDI_GET_COUNTER(ctx, cnt));
 }
 
 /*********************************************************************
  *
  *  OTHER FUNCTIONS EXPORTED TO THE STACK
  *
  **********************************************************************/
 
 static void
 iflib_vlan_register(void *arg, if_t ifp, uint16_t vtag)
 {
 	if_ctx_t ctx = if_getsoftc(ifp);
 
 	if ((void *)ctx != arg)
 		return;
 
 	if ((vtag == 0) || (vtag > 4095))
 		return;
 
 	CTX_LOCK(ctx);
 	IFDI_VLAN_REGISTER(ctx, vtag);
 	/* Re-init to load the changes */
 	if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER)
 		iflib_if_init_locked(ctx);
 	CTX_UNLOCK(ctx);
 }
 
 static void
 iflib_vlan_unregister(void *arg, if_t ifp, uint16_t vtag)
 {
 	if_ctx_t ctx = if_getsoftc(ifp);
 
 	if ((void *)ctx != arg)
 		return;
 
 	if ((vtag == 0) || (vtag > 4095))
 		return;
 
 	CTX_LOCK(ctx);
 	IFDI_VLAN_UNREGISTER(ctx, vtag);
 	/* Re-init to load the changes */
 	if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER)
 		iflib_if_init_locked(ctx);
 	CTX_UNLOCK(ctx);
 }
 
 static void
 iflib_led_func(void *arg, int onoff)
 {
 	if_ctx_t ctx = arg;
 
 	CTX_LOCK(ctx);
 	IFDI_LED_FUNC(ctx, onoff);
 	CTX_UNLOCK(ctx);
 }
 
 /*********************************************************************
  *
  *  BUS FUNCTION DEFINITIONS
  *
  **********************************************************************/
 
 int
 iflib_device_probe(device_t dev)
 {
 	pci_vendor_info_t *ent;
 
 	uint16_t	pci_vendor_id, pci_device_id;
 	uint16_t	pci_subvendor_id, pci_subdevice_id;
 	uint16_t	pci_rev_id;
 	if_shared_ctx_t sctx;
 
 	if ((sctx = DEVICE_REGISTER(dev)) == NULL || sctx->isc_magic != IFLIB_MAGIC)
 		return (ENOTSUP);
 
 	pci_vendor_id = pci_get_vendor(dev);
 	pci_device_id = pci_get_device(dev);
 	pci_subvendor_id = pci_get_subvendor(dev);
 	pci_subdevice_id = pci_get_subdevice(dev);
 	pci_rev_id = pci_get_revid(dev);
 	if (sctx->isc_parse_devinfo != NULL)
 		sctx->isc_parse_devinfo(&pci_device_id, &pci_subvendor_id, &pci_subdevice_id, &pci_rev_id);
 
 	ent = sctx->isc_vendor_info;
 	while (ent->pvi_vendor_id != 0) {
 		if (pci_vendor_id != ent->pvi_vendor_id) {
 			ent++;
 			continue;
 		}
 		if ((pci_device_id == ent->pvi_device_id) &&
 		    ((pci_subvendor_id == ent->pvi_subvendor_id) ||
 		     (ent->pvi_subvendor_id == 0)) &&
 		    ((pci_subdevice_id == ent->pvi_subdevice_id) ||
 		     (ent->pvi_subdevice_id == 0)) &&
 		    ((pci_rev_id == ent->pvi_rev_id) ||
 		     (ent->pvi_rev_id == 0))) {
 
 			device_set_desc_copy(dev, ent->pvi_name);
 			/* this needs to be changed to zero if the bus probing code
 			 * ever stops re-probing on best match because the sctx
 			 * may have its values over written by register calls
 			 * in subsequent probes
 			 */
 			return (BUS_PROBE_DEFAULT);
 		}
 		ent++;
 	}
 	return (ENXIO);
 }
 
 static void
 iflib_reset_qvalues(if_ctx_t ctx)
 {
 	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
 	if_shared_ctx_t sctx = ctx->ifc_sctx;
 	device_t dev = ctx->ifc_dev;
 	int i;
 
 	scctx->isc_txrx_budget_bytes_max = IFLIB_MAX_TX_BYTES;
 	scctx->isc_tx_qdepth = IFLIB_DEFAULT_TX_QDEPTH;
 	/*
 	 * XXX sanity check that ntxd & nrxd are a power of 2
 	 */
 	if (ctx->ifc_sysctl_ntxqs != 0)
 		scctx->isc_ntxqsets = ctx->ifc_sysctl_ntxqs;
 	if (ctx->ifc_sysctl_nrxqs != 0)
 		scctx->isc_nrxqsets = ctx->ifc_sysctl_nrxqs;
 
 	for (i = 0; i < sctx->isc_ntxqs; i++) {
 		if (ctx->ifc_sysctl_ntxds[i] != 0)
 			scctx->isc_ntxd[i] = ctx->ifc_sysctl_ntxds[i];
 		else
 			scctx->isc_ntxd[i] = sctx->isc_ntxd_default[i];
 	}
 
 	for (i = 0; i < sctx->isc_nrxqs; i++) {
 		if (ctx->ifc_sysctl_nrxds[i] != 0)
 			scctx->isc_nrxd[i] = ctx->ifc_sysctl_nrxds[i];
 		else
 			scctx->isc_nrxd[i] = sctx->isc_nrxd_default[i];
 	}
 
 	for (i = 0; i < sctx->isc_nrxqs; i++) {
 		if (scctx->isc_nrxd[i] < sctx->isc_nrxd_min[i]) {
 			device_printf(dev, "nrxd%d: %d less than nrxd_min %d - resetting to min\n",
 				      i, scctx->isc_nrxd[i], sctx->isc_nrxd_min[i]);
 			scctx->isc_nrxd[i] = sctx->isc_nrxd_min[i];
 		}
 		if (scctx->isc_nrxd[i] > sctx->isc_nrxd_max[i]) {
 			device_printf(dev, "nrxd%d: %d greater than nrxd_max %d - resetting to max\n",
 				      i, scctx->isc_nrxd[i], sctx->isc_nrxd_max[i]);
 			scctx->isc_nrxd[i] = sctx->isc_nrxd_max[i];
 		}
 	}
 
 	for (i = 0; i < sctx->isc_ntxqs; i++) {
 		if (scctx->isc_ntxd[i] < sctx->isc_ntxd_min[i]) {
 			device_printf(dev, "ntxd%d: %d less than ntxd_min %d - resetting to min\n",
 				      i, scctx->isc_ntxd[i], sctx->isc_ntxd_min[i]);
 			scctx->isc_ntxd[i] = sctx->isc_ntxd_min[i];
 		}
 		if (scctx->isc_ntxd[i] > sctx->isc_ntxd_max[i]) {
 			device_printf(dev, "ntxd%d: %d greater than ntxd_max %d - resetting to max\n",
 				      i, scctx->isc_ntxd[i], sctx->isc_ntxd_max[i]);
 			scctx->isc_ntxd[i] = sctx->isc_ntxd_max[i];
 		}
 	}
 }
 
 int
 iflib_device_register(device_t dev, void *sc, if_shared_ctx_t sctx, if_ctx_t *ctxp)
 {
 	int err, rid, msix;
 	if_ctx_t ctx;
 	if_t ifp;
 	if_softc_ctx_t scctx;
 	int i;
 	uint16_t main_txq;
 	uint16_t main_rxq;
 
 
 	ctx = malloc(sizeof(* ctx), M_IFLIB, M_WAITOK|M_ZERO);
 
 	if (sc == NULL) {
 		sc = malloc(sctx->isc_driver->size, M_IFLIB, M_WAITOK|M_ZERO);
 		device_set_softc(dev, ctx);
 		ctx->ifc_flags |= IFC_SC_ALLOCATED;
 	}
 
 	ctx->ifc_sctx = sctx;
 	ctx->ifc_dev = dev;
 	ctx->ifc_softc = sc;
 
 	if ((err = iflib_register(ctx)) != 0) {
 		device_printf(dev, "iflib_register failed %d\n", err);
 		goto fail_ctx_free;
 	}
 	iflib_add_device_sysctl_pre(ctx);
 
 	scctx = &ctx->ifc_softc_ctx;
 	ifp = ctx->ifc_ifp;
 
 	iflib_reset_qvalues(ctx);
 	CTX_LOCK(ctx);
 	if ((err = IFDI_ATTACH_PRE(ctx)) != 0) {
 		device_printf(dev, "IFDI_ATTACH_PRE failed %d\n", err);
 		goto fail_unlock;
 	}
 	_iflib_pre_assert(scctx);
 	ctx->ifc_txrx = *scctx->isc_txrx;
 
 #ifdef INVARIANTS
 	MPASS(scctx->isc_capabilities);
 	if (scctx->isc_capabilities & IFCAP_TXCSUM)
 		MPASS(scctx->isc_tx_csum_flags);
 #endif
 
 	if_setcapabilities(ifp, scctx->isc_capabilities | IFCAP_HWSTATS);
 	if_setcapenable(ifp, scctx->isc_capenable | IFCAP_HWSTATS);
 
 	if (scctx->isc_ntxqsets == 0 || (scctx->isc_ntxqsets_max && scctx->isc_ntxqsets_max < scctx->isc_ntxqsets))
 		scctx->isc_ntxqsets = scctx->isc_ntxqsets_max;
 	if (scctx->isc_nrxqsets == 0 || (scctx->isc_nrxqsets_max && scctx->isc_nrxqsets_max < scctx->isc_nrxqsets))
 		scctx->isc_nrxqsets = scctx->isc_nrxqsets_max;
 
 	main_txq = (sctx->isc_flags & IFLIB_HAS_TXCQ) ? 1 : 0;
 	main_rxq = (sctx->isc_flags & IFLIB_HAS_RXCQ) ? 1 : 0;
 
 	/* XXX change for per-queue sizes */
 	device_printf(dev, "Using %d tx descriptors and %d rx descriptors\n",
 	    scctx->isc_ntxd[main_txq], scctx->isc_nrxd[main_rxq]);
 	for (i = 0; i < sctx->isc_nrxqs; i++) {
 		if (!powerof2(scctx->isc_nrxd[i])) {
 			/* round down instead? */
 			device_printf(dev, "# rx descriptors must be a power of 2\n");
 			err = EINVAL;
 			goto fail_iflib_detach;
 		}
 	}
 	for (i = 0; i < sctx->isc_ntxqs; i++) {
 		if (!powerof2(scctx->isc_ntxd[i])) {
 			device_printf(dev,
 			    "# tx descriptors must be a power of 2");
 			err = EINVAL;
 			goto fail_iflib_detach;
 		}
 	}
 
 	if (scctx->isc_tx_nsegments > scctx->isc_ntxd[main_txq] /
 	    MAX_SINGLE_PACKET_FRACTION)
 		scctx->isc_tx_nsegments = max(1, scctx->isc_ntxd[main_txq] /
 		    MAX_SINGLE_PACKET_FRACTION);
 	if (scctx->isc_tx_tso_segments_max > scctx->isc_ntxd[main_txq] /
 	    MAX_SINGLE_PACKET_FRACTION)
 		scctx->isc_tx_tso_segments_max = max(1,
 		    scctx->isc_ntxd[main_txq] / MAX_SINGLE_PACKET_FRACTION);
 
 	/* TSO parameters - dig these out of the data sheet - simply correspond to tag setup */
 	if (if_getcapabilities(ifp) & IFCAP_TSO) {
 		/*
 		 * The stack can't handle a TSO size larger than IP_MAXPACKET,
 		 * but some MACs do.
 		 */
 		if_sethwtsomax(ifp, min(scctx->isc_tx_tso_size_max,
 		    IP_MAXPACKET));
 		/*
 		 * Take maximum number of m_pullup(9)'s in iflib_parse_header()
 		 * into account.  In the worst case, each of these calls will
 		 * add another mbuf and, thus, the requirement for another DMA
 		 * segment.  So for best performance, it doesn't make sense to
 		 * advertize a maximum of TSO segments that typically will
 		 * require defragmentation in iflib_encap().
 		 */
 		if_sethwtsomaxsegcount(ifp, scctx->isc_tx_tso_segments_max - 3);
 		if_sethwtsomaxsegsize(ifp, scctx->isc_tx_tso_segsize_max);
 	}
 	if (scctx->isc_rss_table_size == 0)
 		scctx->isc_rss_table_size = 64;
 	scctx->isc_rss_table_mask = scctx->isc_rss_table_size-1;
 
 	GROUPTASK_INIT(&ctx->ifc_admin_task, 0, _task_fn_admin, ctx);
 	/* XXX format name */
 	taskqgroup_attach(qgroup_if_config_tqg, &ctx->ifc_admin_task, ctx,
 	    NULL, NULL, "admin");
 
 	/* Set up cpu set.  If it fails, use the set of all CPUs. */
 	if (bus_get_cpus(dev, INTR_CPUS, sizeof(ctx->ifc_cpus), &ctx->ifc_cpus) != 0) {
 		device_printf(dev, "Unable to fetch CPU list\n");
 		CPU_COPY(&all_cpus, &ctx->ifc_cpus);
 	}
 	MPASS(CPU_COUNT(&ctx->ifc_cpus) > 0);
 
 	/*
 	** Now set up MSI or MSI-X, should return us the number of supported
 	** vectors (will be 1 for a legacy interrupt and MSI).
 	*/
 	if (sctx->isc_flags & IFLIB_SKIP_MSIX) {
 		msix = scctx->isc_vectors;
 	} else if (scctx->isc_msix_bar != 0)
 	       /*
 		* The simple fact that isc_msix_bar is not 0 does not mean we
 		* we have a good value there that is known to work.
 		*/
 		msix = iflib_msix_init(ctx);
 	else {
 		scctx->isc_vectors = 1;
 		scctx->isc_ntxqsets = 1;
 		scctx->isc_nrxqsets = 1;
 		scctx->isc_intr = IFLIB_INTR_LEGACY;
 		msix = 0;
 	}
 	/* Get memory for the station queues */
 	if ((err = iflib_queues_alloc(ctx))) {
 		device_printf(dev, "Unable to allocate queue memory\n");
 		goto fail_intr_free;
 	}
 
 	if ((err = iflib_qset_structures_setup(ctx)))
 		goto fail_queues;
 
 	/*
 	 * Group taskqueues aren't properly set up until SMP is started,
 	 * so we disable interrupts until we can handle them post
 	 * SI_SUB_SMP.
 	 *
 	 * XXX: disabling interrupts doesn't actually work, at least for
 	 * the non-MSI case.  When they occur before SI_SUB_SMP completes,
 	 * we do null handling and depend on this not causing too large an
 	 * interrupt storm.
 	 */
 	IFDI_INTR_DISABLE(ctx);
 	if (msix > 1 && (err = IFDI_MSIX_INTR_ASSIGN(ctx, msix)) != 0) {
 		device_printf(dev, "IFDI_MSIX_INTR_ASSIGN failed %d\n", err);
 		goto fail_queues;
 	}
 	if (msix <= 1) {
 		rid = 0;
 		if (scctx->isc_intr == IFLIB_INTR_MSI) {
 			MPASS(msix == 1);
 			rid = 1;
 		}
 		if ((err = iflib_legacy_setup(ctx, ctx->isc_legacy_intr, ctx->ifc_softc, &rid, "irq0")) != 0) {
 			device_printf(dev, "iflib_legacy_setup failed %d\n", err);
 			goto fail_queues;
 		}
 	}
 
 	ether_ifattach(ctx->ifc_ifp, ctx->ifc_mac);
 
 	if ((err = IFDI_ATTACH_POST(ctx)) != 0) {
 		device_printf(dev, "IFDI_ATTACH_POST failed %d\n", err);
 		goto fail_detach;
 	}
 
 	/*
 	 * Tell the upper layer(s) if IFCAP_VLAN_MTU is supported.
 	 * This must appear after the call to ether_ifattach() because
 	 * ether_ifattach() sets if_hdrlen to the default value.
 	 */
 	if (if_getcapabilities(ifp) & IFCAP_VLAN_MTU)
 		if_setifheaderlen(ifp, sizeof(struct ether_vlan_header));
 
 	if ((err = iflib_netmap_attach(ctx))) {
 		device_printf(ctx->ifc_dev, "netmap attach failed: %d\n", err);
 		goto fail_detach;
 	}
 	*ctxp = ctx;
 
 	NETDUMP_SET(ctx->ifc_ifp, iflib);
 
 	if_setgetcounterfn(ctx->ifc_ifp, iflib_if_get_counter);
 	iflib_add_device_sysctl_post(ctx);
 	ctx->ifc_flags |= IFC_INIT_DONE;
 	CTX_UNLOCK(ctx);
 	return (0);
 
 fail_detach:
 	ether_ifdetach(ctx->ifc_ifp);
 fail_intr_free:
 	iflib_free_intr_mem(ctx);
 fail_queues:
 	iflib_tx_structures_free(ctx);
 	iflib_rx_structures_free(ctx);
 fail_iflib_detach:
 	IFDI_DETACH(ctx);
 fail_unlock:
 	CTX_UNLOCK(ctx);
 fail_ctx_free:
         if (ctx->ifc_flags & IFC_SC_ALLOCATED)
                 free(ctx->ifc_softc, M_IFLIB);
         free(ctx, M_IFLIB);
 	return (err);
 }
 
 int
 iflib_pseudo_register(device_t dev, if_shared_ctx_t sctx, if_ctx_t *ctxp,
 					  struct iflib_cloneattach_ctx *clctx)
 {
 	int err;
 	if_ctx_t ctx;
 	if_t ifp;
 	if_softc_ctx_t scctx;
 	int i;
 	void *sc;
 	uint16_t main_txq;
 	uint16_t main_rxq;
 
 	ctx = malloc(sizeof(*ctx), M_IFLIB, M_WAITOK|M_ZERO);
 	sc = malloc(sctx->isc_driver->size, M_IFLIB, M_WAITOK|M_ZERO);
 	ctx->ifc_flags |= IFC_SC_ALLOCATED;
 	if (sctx->isc_flags & (IFLIB_PSEUDO|IFLIB_VIRTUAL))
 		ctx->ifc_flags |= IFC_PSEUDO;
 
 	ctx->ifc_sctx = sctx;
 	ctx->ifc_softc = sc;
 	ctx->ifc_dev = dev;
 
 	if ((err = iflib_register(ctx)) != 0) {
 		device_printf(dev, "%s: iflib_register failed %d\n", __func__, err);
 		goto fail_ctx_free;
 	}
 	iflib_add_device_sysctl_pre(ctx);
 
 	scctx = &ctx->ifc_softc_ctx;
 	ifp = ctx->ifc_ifp;
 
 	/*
 	 * XXX sanity check that ntxd & nrxd are a power of 2
 	 */
 	iflib_reset_qvalues(ctx);
 
 	if ((err = IFDI_ATTACH_PRE(ctx)) != 0) {
 		device_printf(dev, "IFDI_ATTACH_PRE failed %d\n", err);
 		goto fail_ctx_free;
 	}
 	if (sctx->isc_flags & IFLIB_GEN_MAC)
 		iflib_gen_mac(ctx);
 	if ((err = IFDI_CLONEATTACH(ctx, clctx->cc_ifc, clctx->cc_name,
 								clctx->cc_params)) != 0) {
 		device_printf(dev, "IFDI_CLONEATTACH failed %d\n", err);
 		goto fail_ctx_free;
 	}
 	ifmedia_add(&ctx->ifc_media, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
 	ifmedia_add(&ctx->ifc_media, IFM_ETHER | IFM_AUTO, 0, NULL);
 	ifmedia_set(&ctx->ifc_media, IFM_ETHER | IFM_AUTO);
 
 #ifdef INVARIANTS
 	MPASS(scctx->isc_capabilities);
 	if (scctx->isc_capabilities & IFCAP_TXCSUM)
 		MPASS(scctx->isc_tx_csum_flags);
 #endif
 
 	if_setcapabilities(ifp, scctx->isc_capabilities | IFCAP_HWSTATS | IFCAP_LINKSTATE);
 	if_setcapenable(ifp, scctx->isc_capenable | IFCAP_HWSTATS | IFCAP_LINKSTATE);
 
 	ifp->if_flags |= IFF_NOGROUP;
 	if (sctx->isc_flags & IFLIB_PSEUDO) {
 		ether_ifattach(ctx->ifc_ifp, ctx->ifc_mac);
 
 		if ((err = IFDI_ATTACH_POST(ctx)) != 0) {
 			device_printf(dev, "IFDI_ATTACH_POST failed %d\n", err);
 			goto fail_detach;
 		}
 		*ctxp = ctx;
 
 		/*
 		 * Tell the upper layer(s) if IFCAP_VLAN_MTU is supported.
 		 * This must appear after the call to ether_ifattach() because
 		 * ether_ifattach() sets if_hdrlen to the default value.
 		 */
 		if (if_getcapabilities(ifp) & IFCAP_VLAN_MTU)
 			if_setifheaderlen(ifp,
 			    sizeof(struct ether_vlan_header));
 
 		if_setgetcounterfn(ctx->ifc_ifp, iflib_if_get_counter);
 		iflib_add_device_sysctl_post(ctx);
 		ctx->ifc_flags |= IFC_INIT_DONE;
 		return (0);
 	}
 	_iflib_pre_assert(scctx);
 	ctx->ifc_txrx = *scctx->isc_txrx;
 
 	if (scctx->isc_ntxqsets == 0 || (scctx->isc_ntxqsets_max && scctx->isc_ntxqsets_max < scctx->isc_ntxqsets))
 		scctx->isc_ntxqsets = scctx->isc_ntxqsets_max;
 	if (scctx->isc_nrxqsets == 0 || (scctx->isc_nrxqsets_max && scctx->isc_nrxqsets_max < scctx->isc_nrxqsets))
 		scctx->isc_nrxqsets = scctx->isc_nrxqsets_max;
 
 	main_txq = (sctx->isc_flags & IFLIB_HAS_TXCQ) ? 1 : 0;
 	main_rxq = (sctx->isc_flags & IFLIB_HAS_RXCQ) ? 1 : 0;
 
 	/* XXX change for per-queue sizes */
 	device_printf(dev, "Using %d tx descriptors and %d rx descriptors\n",
 	    scctx->isc_ntxd[main_txq], scctx->isc_nrxd[main_rxq]);
 	for (i = 0; i < sctx->isc_nrxqs; i++) {
 		if (!powerof2(scctx->isc_nrxd[i])) {
 			/* round down instead? */
 			device_printf(dev, "# rx descriptors must be a power of 2\n");
 			err = EINVAL;
 			goto fail_iflib_detach;
 		}
 	}
 	for (i = 0; i < sctx->isc_ntxqs; i++) {
 		if (!powerof2(scctx->isc_ntxd[i])) {
 			device_printf(dev,
 			    "# tx descriptors must be a power of 2");
 			err = EINVAL;
 			goto fail_iflib_detach;
 		}
 	}
 
 	if (scctx->isc_tx_nsegments > scctx->isc_ntxd[main_txq] /
 	    MAX_SINGLE_PACKET_FRACTION)
 		scctx->isc_tx_nsegments = max(1, scctx->isc_ntxd[main_txq] /
 		    MAX_SINGLE_PACKET_FRACTION);
 	if (scctx->isc_tx_tso_segments_max > scctx->isc_ntxd[main_txq] /
 	    MAX_SINGLE_PACKET_FRACTION)
 		scctx->isc_tx_tso_segments_max = max(1,
 		    scctx->isc_ntxd[main_txq] / MAX_SINGLE_PACKET_FRACTION);
 
 	/* TSO parameters - dig these out of the data sheet - simply correspond to tag setup */
 	if (if_getcapabilities(ifp) & IFCAP_TSO) {
 		/*
 		 * The stack can't handle a TSO size larger than IP_MAXPACKET,
 		 * but some MACs do.
 		 */
 		if_sethwtsomax(ifp, min(scctx->isc_tx_tso_size_max,
 		    IP_MAXPACKET));
 		/*
 		 * Take maximum number of m_pullup(9)'s in iflib_parse_header()
 		 * into account.  In the worst case, each of these calls will
 		 * add another mbuf and, thus, the requirement for another DMA
 		 * segment.  So for best performance, it doesn't make sense to
 		 * advertize a maximum of TSO segments that typically will
 		 * require defragmentation in iflib_encap().
 		 */
 		if_sethwtsomaxsegcount(ifp, scctx->isc_tx_tso_segments_max - 3);
 		if_sethwtsomaxsegsize(ifp, scctx->isc_tx_tso_segsize_max);
 	}
 	if (scctx->isc_rss_table_size == 0)
 		scctx->isc_rss_table_size = 64;
 	scctx->isc_rss_table_mask = scctx->isc_rss_table_size-1;
 
 	GROUPTASK_INIT(&ctx->ifc_admin_task, 0, _task_fn_admin, ctx);
 	/* XXX format name */
 	taskqgroup_attach(qgroup_if_config_tqg, &ctx->ifc_admin_task, ctx,
 	    NULL, NULL, "admin");
 
 	/* XXX --- can support > 1 -- but keep it simple for now */
 	scctx->isc_intr = IFLIB_INTR_LEGACY;
 
 	/* Get memory for the station queues */
 	if ((err = iflib_queues_alloc(ctx))) {
 		device_printf(dev, "Unable to allocate queue memory\n");
 		goto fail_iflib_detach;
 	}
 
 	if ((err = iflib_qset_structures_setup(ctx))) {
 		device_printf(dev, "qset structure setup failed %d\n", err);
 		goto fail_queues;
 	}
 
 	/*
 	 * XXX What if anything do we want to do about interrupts?
 	 */
 	ether_ifattach(ctx->ifc_ifp, ctx->ifc_mac);
 	if ((err = IFDI_ATTACH_POST(ctx)) != 0) {
 		device_printf(dev, "IFDI_ATTACH_POST failed %d\n", err);
 		goto fail_detach;
 	}
 
 	/*
 	 * Tell the upper layer(s) if IFCAP_VLAN_MTU is supported.
 	 * This must appear after the call to ether_ifattach() because
 	 * ether_ifattach() sets if_hdrlen to the default value.
 	 */
 	if (if_getcapabilities(ifp) & IFCAP_VLAN_MTU)
 		if_setifheaderlen(ifp, sizeof(struct ether_vlan_header));
 
 	/* XXX handle more than one queue */
 	for (i = 0; i < scctx->isc_nrxqsets; i++)
 		IFDI_RX_CLSET(ctx, 0, i, ctx->ifc_rxqs[i].ifr_fl[0].ifl_sds.ifsd_cl);
 
 	*ctxp = ctx;
 
 	if_setgetcounterfn(ctx->ifc_ifp, iflib_if_get_counter);
 	iflib_add_device_sysctl_post(ctx);
 	ctx->ifc_flags |= IFC_INIT_DONE;
 	return (0);
 fail_detach:
 	ether_ifdetach(ctx->ifc_ifp);
 fail_queues:
 	iflib_tx_structures_free(ctx);
 	iflib_rx_structures_free(ctx);
 fail_iflib_detach:
 	IFDI_DETACH(ctx);
 fail_ctx_free:
 	free(ctx->ifc_softc, M_IFLIB);
 	free(ctx, M_IFLIB);
 	return (err);
 }
 
 int
 iflib_pseudo_deregister(if_ctx_t ctx)
 {
 	if_t ifp = ctx->ifc_ifp;
 	iflib_txq_t txq;
 	iflib_rxq_t rxq;
 	int i, j;
 	struct taskqgroup *tqg;
 	iflib_fl_t fl;
 
 	/* Unregister VLAN events */
 	if (ctx->ifc_vlan_attach_event != NULL)
 		EVENTHANDLER_DEREGISTER(vlan_config, ctx->ifc_vlan_attach_event);
 	if (ctx->ifc_vlan_detach_event != NULL)
 		EVENTHANDLER_DEREGISTER(vlan_unconfig, ctx->ifc_vlan_detach_event);
 
 	ether_ifdetach(ifp);
 	/* ether_ifdetach calls if_qflush - lock must be destroy afterwards*/
 	CTX_LOCK_DESTROY(ctx);
 	/* XXX drain any dependent tasks */
 	tqg = qgroup_if_io_tqg;
 	for (txq = ctx->ifc_txqs, i = 0; i < NTXQSETS(ctx); i++, txq++) {
 		callout_drain(&txq->ift_timer);
 		if (txq->ift_task.gt_uniq != NULL)
 			taskqgroup_detach(tqg, &txq->ift_task);
 	}
 	for (i = 0, rxq = ctx->ifc_rxqs; i < NRXQSETS(ctx); i++, rxq++) {
 		if (rxq->ifr_task.gt_uniq != NULL)
 			taskqgroup_detach(tqg, &rxq->ifr_task);
 
 		for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++)
 			free(fl->ifl_rx_bitmap, M_IFLIB);
 	}
 	tqg = qgroup_if_config_tqg;
 	if (ctx->ifc_admin_task.gt_uniq != NULL)
 		taskqgroup_detach(tqg, &ctx->ifc_admin_task);
 	if (ctx->ifc_vflr_task.gt_uniq != NULL)
 		taskqgroup_detach(tqg, &ctx->ifc_vflr_task);
 
 	if_free(ifp);
 
 	iflib_tx_structures_free(ctx);
 	iflib_rx_structures_free(ctx);
 	if (ctx->ifc_flags & IFC_SC_ALLOCATED)
 		free(ctx->ifc_softc, M_IFLIB);
 	free(ctx, M_IFLIB);
 	return (0);
 }
 
 int
 iflib_device_attach(device_t dev)
 {
 	if_ctx_t ctx;
 	if_shared_ctx_t sctx;
 
 	if ((sctx = DEVICE_REGISTER(dev)) == NULL || sctx->isc_magic != IFLIB_MAGIC)
 		return (ENOTSUP);
 
 	pci_enable_busmaster(dev);
 
 	return (iflib_device_register(dev, NULL, sctx, &ctx));
 }
 
 int
 iflib_device_deregister(if_ctx_t ctx)
 {
 	if_t ifp = ctx->ifc_ifp;
 	iflib_txq_t txq;
 	iflib_rxq_t rxq;
 	device_t dev = ctx->ifc_dev;
 	int i, j;
 	struct taskqgroup *tqg;
 	iflib_fl_t fl;
 
 	/* Make sure VLANS are not using driver */
 	if (if_vlantrunkinuse(ifp)) {
 		device_printf(dev, "Vlan in use, detach first\n");
 		return (EBUSY);
 	}
 #ifdef PCI_IOV
 	if (!CTX_IS_VF(ctx) && pci_iov_detach(dev) != 0) {
 		device_printf(dev, "SR-IOV in use; detach first.\n");
 		return (EBUSY);
 	}
 #endif
 
 	STATE_LOCK(ctx);
 	ctx->ifc_flags |= IFC_IN_DETACH;
 	STATE_UNLOCK(ctx);
 
 	CTX_LOCK(ctx);
 	iflib_stop(ctx);
 	CTX_UNLOCK(ctx);
 
 	/* Unregister VLAN events */
 	if (ctx->ifc_vlan_attach_event != NULL)
 		EVENTHANDLER_DEREGISTER(vlan_config, ctx->ifc_vlan_attach_event);
 	if (ctx->ifc_vlan_detach_event != NULL)
 		EVENTHANDLER_DEREGISTER(vlan_unconfig, ctx->ifc_vlan_detach_event);
 
 	iflib_netmap_detach(ifp);
 	ether_ifdetach(ifp);
 	if (ctx->ifc_led_dev != NULL)
 		led_destroy(ctx->ifc_led_dev);
 	/* XXX drain any dependent tasks */
 	tqg = qgroup_if_io_tqg;
 	for (txq = ctx->ifc_txqs, i = 0; i < NTXQSETS(ctx); i++, txq++) {
 		callout_drain(&txq->ift_timer);
 		if (txq->ift_task.gt_uniq != NULL)
 			taskqgroup_detach(tqg, &txq->ift_task);
 	}
 	for (i = 0, rxq = ctx->ifc_rxqs; i < NRXQSETS(ctx); i++, rxq++) {
 		if (rxq->ifr_task.gt_uniq != NULL)
 			taskqgroup_detach(tqg, &rxq->ifr_task);
 
 		for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++)
 			free(fl->ifl_rx_bitmap, M_IFLIB);
 	}
 	tqg = qgroup_if_config_tqg;
 	if (ctx->ifc_admin_task.gt_uniq != NULL)
 		taskqgroup_detach(tqg, &ctx->ifc_admin_task);
 	if (ctx->ifc_vflr_task.gt_uniq != NULL)
 		taskqgroup_detach(tqg, &ctx->ifc_vflr_task);
 	CTX_LOCK(ctx);
 	IFDI_DETACH(ctx);
 	CTX_UNLOCK(ctx);
 
 	/* ether_ifdetach calls if_qflush - lock must be destroy afterwards*/
 	CTX_LOCK_DESTROY(ctx);
 	device_set_softc(ctx->ifc_dev, NULL);
 	iflib_free_intr_mem(ctx);
 
 	bus_generic_detach(dev);
 	if_free(ifp);
 
 	iflib_tx_structures_free(ctx);
 	iflib_rx_structures_free(ctx);
 	if (ctx->ifc_flags & IFC_SC_ALLOCATED)
 		free(ctx->ifc_softc, M_IFLIB);
 	STATE_LOCK_DESTROY(ctx);
 	free(ctx, M_IFLIB);
 	return (0);
 }
 
 static void
 iflib_free_intr_mem(if_ctx_t ctx)
 {
 
 	if (ctx->ifc_softc_ctx.isc_intr != IFLIB_INTR_MSIX) {
 		iflib_irq_free(ctx, &ctx->ifc_legacy_irq);
 	}
 	if (ctx->ifc_softc_ctx.isc_intr != IFLIB_INTR_LEGACY) {
 		pci_release_msi(ctx->ifc_dev);
 	}
 	if (ctx->ifc_msix_mem != NULL) {
 		bus_release_resource(ctx->ifc_dev, SYS_RES_MEMORY,
 		    rman_get_rid(ctx->ifc_msix_mem), ctx->ifc_msix_mem);
 		ctx->ifc_msix_mem = NULL;
 	}
 }
 
 int
 iflib_device_detach(device_t dev)
 {
 	if_ctx_t ctx = device_get_softc(dev);
 
 	return (iflib_device_deregister(ctx));
 }
 
 int
 iflib_device_suspend(device_t dev)
 {
 	if_ctx_t ctx = device_get_softc(dev);
 
 	CTX_LOCK(ctx);
 	IFDI_SUSPEND(ctx);
 	CTX_UNLOCK(ctx);
 
 	return bus_generic_suspend(dev);
 }
 int
 iflib_device_shutdown(device_t dev)
 {
 	if_ctx_t ctx = device_get_softc(dev);
 
 	CTX_LOCK(ctx);
 	IFDI_SHUTDOWN(ctx);
 	CTX_UNLOCK(ctx);
 
 	return bus_generic_suspend(dev);
 }
 
 
 int
 iflib_device_resume(device_t dev)
 {
 	if_ctx_t ctx = device_get_softc(dev);
 	iflib_txq_t txq = ctx->ifc_txqs;
 
 	CTX_LOCK(ctx);
 	IFDI_RESUME(ctx);
 	iflib_if_init_locked(ctx);
 	CTX_UNLOCK(ctx);
 	for (int i = 0; i < NTXQSETS(ctx); i++, txq++)
 		iflib_txq_check_drain(txq, IFLIB_RESTART_BUDGET);
 
 	return (bus_generic_resume(dev));
 }
 
 int
 iflib_device_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *params)
 {
 	int error;
 	if_ctx_t ctx = device_get_softc(dev);
 
 	CTX_LOCK(ctx);
 	error = IFDI_IOV_INIT(ctx, num_vfs, params);
 	CTX_UNLOCK(ctx);
 
 	return (error);
 }
 
 void
 iflib_device_iov_uninit(device_t dev)
 {
 	if_ctx_t ctx = device_get_softc(dev);
 
 	CTX_LOCK(ctx);
 	IFDI_IOV_UNINIT(ctx);
 	CTX_UNLOCK(ctx);
 }
 
 int
 iflib_device_iov_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *params)
 {
 	int error;
 	if_ctx_t ctx = device_get_softc(dev);
 
 	CTX_LOCK(ctx);
 	error = IFDI_IOV_VF_ADD(ctx, vfnum, params);
 	CTX_UNLOCK(ctx);
 
 	return (error);
 }
 
 /*********************************************************************
  *
  *  MODULE FUNCTION DEFINITIONS
  *
  **********************************************************************/
 
 /*
  * - Start a fast taskqueue thread for each core
  * - Start a taskqueue for control operations
  */
 static int
 iflib_module_init(void)
 {
 	return (0);
 }
 
 static int
 iflib_module_event_handler(module_t mod, int what, void *arg)
 {
 	int err;
 
 	switch (what) {
 	case MOD_LOAD:
 		if ((err = iflib_module_init()) != 0)
 			return (err);
 		break;
 	case MOD_UNLOAD:
 		return (EBUSY);
 	default:
 		return (EOPNOTSUPP);
 	}
 
 	return (0);
 }
 
 /*********************************************************************
  *
  *  PUBLIC FUNCTION DEFINITIONS
  *     ordered as in iflib.h
  *
  **********************************************************************/
 
 
 static void
 _iflib_assert(if_shared_ctx_t sctx)
 {
 	MPASS(sctx->isc_tx_maxsize);
 	MPASS(sctx->isc_tx_maxsegsize);
 
 	MPASS(sctx->isc_rx_maxsize);
 	MPASS(sctx->isc_rx_nsegments);
 	MPASS(sctx->isc_rx_maxsegsize);
 
 	MPASS(sctx->isc_nrxd_min[0]);
 	MPASS(sctx->isc_nrxd_max[0]);
 	MPASS(sctx->isc_nrxd_default[0]);
 	MPASS(sctx->isc_ntxd_min[0]);
 	MPASS(sctx->isc_ntxd_max[0]);
 	MPASS(sctx->isc_ntxd_default[0]);
 }
 
 static void
 _iflib_pre_assert(if_softc_ctx_t scctx)
 {
 
 	MPASS(scctx->isc_txrx->ift_txd_encap);
 	MPASS(scctx->isc_txrx->ift_txd_flush);
 	MPASS(scctx->isc_txrx->ift_txd_credits_update);
 	MPASS(scctx->isc_txrx->ift_rxd_available);
 	MPASS(scctx->isc_txrx->ift_rxd_pkt_get);
 	MPASS(scctx->isc_txrx->ift_rxd_refill);
 	MPASS(scctx->isc_txrx->ift_rxd_flush);
 }
 
 static int
 iflib_register(if_ctx_t ctx)
 {
 	if_shared_ctx_t sctx = ctx->ifc_sctx;
 	driver_t *driver = sctx->isc_driver;
 	device_t dev = ctx->ifc_dev;
 	if_t ifp;
 
 	_iflib_assert(sctx);
 
 	CTX_LOCK_INIT(ctx);
 	STATE_LOCK_INIT(ctx, device_get_nameunit(ctx->ifc_dev));
 	ifp = ctx->ifc_ifp = if_alloc(IFT_ETHER);
 	if (ifp == NULL) {
 		device_printf(dev, "can not allocate ifnet structure\n");
 		return (ENOMEM);
 	}
 
 	/*
 	 * Initialize our context's device specific methods
 	 */
 	kobj_init((kobj_t) ctx, (kobj_class_t) driver);
 	kobj_class_compile((kobj_class_t) driver);
 	driver->refs++;
 
 	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
 	if_setsoftc(ifp, ctx);
 	if_setdev(ifp, dev);
 	if_setinitfn(ifp, iflib_if_init);
 	if_setioctlfn(ifp, iflib_if_ioctl);
 #ifdef ALTQ
 	if_setstartfn(ifp, iflib_altq_if_start);
 	if_settransmitfn(ifp, iflib_altq_if_transmit);
 	if_setsendqready(ifp);
 #else
 	if_settransmitfn(ifp, iflib_if_transmit);
 #endif
 	if_setqflushfn(ifp, iflib_if_qflush);
 	if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST);
 
 	ctx->ifc_vlan_attach_event =
 		EVENTHANDLER_REGISTER(vlan_config, iflib_vlan_register, ctx,
 							  EVENTHANDLER_PRI_FIRST);
 	ctx->ifc_vlan_detach_event =
 		EVENTHANDLER_REGISTER(vlan_unconfig, iflib_vlan_unregister, ctx,
 							  EVENTHANDLER_PRI_FIRST);
 
 	ifmedia_init(&ctx->ifc_media, IFM_IMASK,
 					 iflib_media_change, iflib_media_status);
 
 	return (0);
 }
 
 
 static int
 iflib_queues_alloc(if_ctx_t ctx)
 {
 	if_shared_ctx_t sctx = ctx->ifc_sctx;
 	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
 	device_t dev = ctx->ifc_dev;
 	int nrxqsets = scctx->isc_nrxqsets;
 	int ntxqsets = scctx->isc_ntxqsets;
 	iflib_txq_t txq;
 	iflib_rxq_t rxq;
 	iflib_fl_t fl = NULL;
 	int i, j, cpu, err, txconf, rxconf;
 	iflib_dma_info_t ifdip;
 	uint32_t *rxqsizes = scctx->isc_rxqsizes;
 	uint32_t *txqsizes = scctx->isc_txqsizes;
 	uint8_t nrxqs = sctx->isc_nrxqs;
 	uint8_t ntxqs = sctx->isc_ntxqs;
 	int nfree_lists = sctx->isc_nfl ? sctx->isc_nfl : 1;
 	caddr_t *vaddrs;
 	uint64_t *paddrs;
 
 	KASSERT(ntxqs > 0, ("number of queues per qset must be at least 1"));
 	KASSERT(nrxqs > 0, ("number of queues per qset must be at least 1"));
 
 	/* Allocate the TX ring struct memory */
 	if (!(ctx->ifc_txqs =
 	    (iflib_txq_t) malloc(sizeof(struct iflib_txq) *
 	    ntxqsets, M_IFLIB, M_NOWAIT | M_ZERO))) {
 		device_printf(dev, "Unable to allocate TX ring memory\n");
 		err = ENOMEM;
 		goto fail;
 	}
 
 	/* Now allocate the RX */
 	if (!(ctx->ifc_rxqs =
 	    (iflib_rxq_t) malloc(sizeof(struct iflib_rxq) *
 	    nrxqsets, M_IFLIB, M_NOWAIT | M_ZERO))) {
 		device_printf(dev, "Unable to allocate RX ring memory\n");
 		err = ENOMEM;
 		goto rx_fail;
 	}
 
 	txq = ctx->ifc_txqs;
 	rxq = ctx->ifc_rxqs;
 
 	/*
 	 * XXX handle allocation failure
 	 */
 	for (txconf = i = 0, cpu = CPU_FIRST(); i < ntxqsets; i++, txconf++, txq++, cpu = CPU_NEXT(cpu)) {
 		/* Set up some basics */
 
 		if ((ifdip = malloc(sizeof(struct iflib_dma_info) * ntxqs,
 		    M_IFLIB, M_NOWAIT | M_ZERO)) == NULL) {
 			device_printf(dev,
 			    "Unable to allocate TX DMA info memory\n");
 			err = ENOMEM;
 			goto err_tx_desc;
 		}
 		txq->ift_ifdi = ifdip;
 		for (j = 0; j < ntxqs; j++, ifdip++) {
 			if (iflib_dma_alloc(ctx, txqsizes[j], ifdip, 0)) {
 				device_printf(dev,
 				    "Unable to allocate TX descriptors\n");
 				err = ENOMEM;
 				goto err_tx_desc;
 			}
 			txq->ift_txd_size[j] = scctx->isc_txd_size[j];
 			bzero((void *)ifdip->idi_vaddr, txqsizes[j]);
 		}
 		txq->ift_ctx = ctx;
 		txq->ift_id = i;
 		if (sctx->isc_flags & IFLIB_HAS_TXCQ) {
 			txq->ift_br_offset = 1;
 		} else {
 			txq->ift_br_offset = 0;
 		}
 		/* XXX fix this */
 		txq->ift_timer.c_cpu = cpu;
 
 		if (iflib_txsd_alloc(txq)) {
 			device_printf(dev, "Critical Failure setting up TX buffers\n");
 			err = ENOMEM;
 			goto err_tx_desc;
 		}
 
 		/* Initialize the TX lock */
 		snprintf(txq->ift_mtx_name, MTX_NAME_LEN, "%s:tx(%d):callout",
 		    device_get_nameunit(dev), txq->ift_id);
 		mtx_init(&txq->ift_mtx, txq->ift_mtx_name, NULL, MTX_DEF);
 		callout_init_mtx(&txq->ift_timer, &txq->ift_mtx, 0);
 
 		snprintf(txq->ift_db_mtx_name, MTX_NAME_LEN, "%s:tx(%d):db",
 			 device_get_nameunit(dev), txq->ift_id);
 
 		err = ifmp_ring_alloc(&txq->ift_br, 2048, txq, iflib_txq_drain,
 				      iflib_txq_can_drain, M_IFLIB, M_WAITOK);
 		if (err) {
 			/* XXX free any allocated rings */
 			device_printf(dev, "Unable to allocate buf_ring\n");
 			goto err_tx_desc;
 		}
 	}
 
 	for (rxconf = i = 0; i < nrxqsets; i++, rxconf++, rxq++) {
 		/* Set up some basics */
 
 		if ((ifdip = malloc(sizeof(struct iflib_dma_info) * nrxqs,
 		   M_IFLIB, M_NOWAIT | M_ZERO)) == NULL) {
 			device_printf(dev,
 			    "Unable to allocate RX DMA info memory\n");
 			err = ENOMEM;
 			goto err_tx_desc;
 		}
 
 		rxq->ifr_ifdi = ifdip;
 		/* XXX this needs to be changed if #rx queues != #tx queues */
 		rxq->ifr_ntxqirq = 1;
 		rxq->ifr_txqid[0] = i;
 		for (j = 0; j < nrxqs; j++, ifdip++) {
 			if (iflib_dma_alloc(ctx, rxqsizes[j], ifdip, 0)) {
 				device_printf(dev,
 				    "Unable to allocate RX descriptors\n");
 				err = ENOMEM;
 				goto err_tx_desc;
 			}
 			bzero((void *)ifdip->idi_vaddr, rxqsizes[j]);
 		}
 		rxq->ifr_ctx = ctx;
 		rxq->ifr_id = i;
 		if (sctx->isc_flags & IFLIB_HAS_RXCQ) {
 			rxq->ifr_fl_offset = 1;
 		} else {
 			rxq->ifr_fl_offset = 0;
 		}
 		rxq->ifr_nfl = nfree_lists;
 		if (!(fl =
 			  (iflib_fl_t) malloc(sizeof(struct iflib_fl) * nfree_lists, M_IFLIB, M_NOWAIT | M_ZERO))) {
 			device_printf(dev, "Unable to allocate free list memory\n");
 			err = ENOMEM;
 			goto err_tx_desc;
 		}
 		rxq->ifr_fl = fl;
 		for (j = 0; j < nfree_lists; j++) {
 			fl[j].ifl_rxq = rxq;
 			fl[j].ifl_id = j;
 			fl[j].ifl_ifdi = &rxq->ifr_ifdi[j + rxq->ifr_fl_offset];
 			fl[j].ifl_rxd_size = scctx->isc_rxd_size[j];
 		}
 		/* Allocate receive buffers for the ring */
 		if (iflib_rxsd_alloc(rxq)) {
 			device_printf(dev,
 			    "Critical Failure setting up receive buffers\n");
 			err = ENOMEM;
 			goto err_rx_desc;
 		}
 
 		for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) 
 			fl->ifl_rx_bitmap = bit_alloc(fl->ifl_size, M_IFLIB,
 			    M_WAITOK);
 	}
 
 	/* TXQs */
 	vaddrs = malloc(sizeof(caddr_t)*ntxqsets*ntxqs, M_IFLIB, M_WAITOK);
 	paddrs = malloc(sizeof(uint64_t)*ntxqsets*ntxqs, M_IFLIB, M_WAITOK);
 	for (i = 0; i < ntxqsets; i++) {
 		iflib_dma_info_t di = ctx->ifc_txqs[i].ift_ifdi;
 
 		for (j = 0; j < ntxqs; j++, di++) {
 			vaddrs[i*ntxqs + j] = di->idi_vaddr;
 			paddrs[i*ntxqs + j] = di->idi_paddr;
 		}
 	}
 	if ((err = IFDI_TX_QUEUES_ALLOC(ctx, vaddrs, paddrs, ntxqs, ntxqsets)) != 0) {
 		device_printf(ctx->ifc_dev,
 		    "Unable to allocate device TX queue\n");
 		iflib_tx_structures_free(ctx);
 		free(vaddrs, M_IFLIB);
 		free(paddrs, M_IFLIB);
 		goto err_rx_desc;
 	}
 	free(vaddrs, M_IFLIB);
 	free(paddrs, M_IFLIB);
 
 	/* RXQs */
 	vaddrs = malloc(sizeof(caddr_t)*nrxqsets*nrxqs, M_IFLIB, M_WAITOK);
 	paddrs = malloc(sizeof(uint64_t)*nrxqsets*nrxqs, M_IFLIB, M_WAITOK);
 	for (i = 0; i < nrxqsets; i++) {
 		iflib_dma_info_t di = ctx->ifc_rxqs[i].ifr_ifdi;
 
 		for (j = 0; j < nrxqs; j++, di++) {
 			vaddrs[i*nrxqs + j] = di->idi_vaddr;
 			paddrs[i*nrxqs + j] = di->idi_paddr;
 		}
 	}
 	if ((err = IFDI_RX_QUEUES_ALLOC(ctx, vaddrs, paddrs, nrxqs, nrxqsets)) != 0) {
 		device_printf(ctx->ifc_dev,
 		    "Unable to allocate device RX queue\n");
 		iflib_tx_structures_free(ctx);
 		free(vaddrs, M_IFLIB);
 		free(paddrs, M_IFLIB);
 		goto err_rx_desc;
 	}
 	free(vaddrs, M_IFLIB);
 	free(paddrs, M_IFLIB);
 
 	return (0);
 
 /* XXX handle allocation failure changes */
 err_rx_desc:
 err_tx_desc:
 rx_fail:
 	if (ctx->ifc_rxqs != NULL)
 		free(ctx->ifc_rxqs, M_IFLIB);
 	ctx->ifc_rxqs = NULL;
 	if (ctx->ifc_txqs != NULL)
 		free(ctx->ifc_txqs, M_IFLIB);
 	ctx->ifc_txqs = NULL;
 fail:
 	return (err);
 }
 
 static int
 iflib_tx_structures_setup(if_ctx_t ctx)
 {
 	iflib_txq_t txq = ctx->ifc_txqs;
 	int i;
 
 	for (i = 0; i < NTXQSETS(ctx); i++, txq++)
 		iflib_txq_setup(txq);
 
 	return (0);
 }
 
 static void
 iflib_tx_structures_free(if_ctx_t ctx)
 {
 	iflib_txq_t txq = ctx->ifc_txqs;
 	if_shared_ctx_t sctx = ctx->ifc_sctx;
 	int i, j;
 
 	for (i = 0; i < NTXQSETS(ctx); i++, txq++) {
 		iflib_txq_destroy(txq);
 		for (j = 0; j < sctx->isc_ntxqs; j++)
 			iflib_dma_free(&txq->ift_ifdi[j]);
 	}
 	free(ctx->ifc_txqs, M_IFLIB);
 	ctx->ifc_txqs = NULL;
 	IFDI_QUEUES_FREE(ctx);
 }
 
 /*********************************************************************
  *
  *  Initialize all receive rings.
  *
  **********************************************************************/
 static int
 iflib_rx_structures_setup(if_ctx_t ctx)
 {
 	iflib_rxq_t rxq = ctx->ifc_rxqs;
 	int q;
 #if defined(INET6) || defined(INET)
 	int i, err;
 #endif
 
 	for (q = 0; q < ctx->ifc_softc_ctx.isc_nrxqsets; q++, rxq++) {
 #if defined(INET6) || defined(INET)
 		tcp_lro_free(&rxq->ifr_lc);
 		if ((err = tcp_lro_init_args(&rxq->ifr_lc, ctx->ifc_ifp,
 		    TCP_LRO_ENTRIES, min(1024,
 		    ctx->ifc_softc_ctx.isc_nrxd[rxq->ifr_fl_offset]))) != 0) {
 			device_printf(ctx->ifc_dev, "LRO Initialization failed!\n");
 			goto fail;
 		}
 		rxq->ifr_lro_enabled = TRUE;
 #endif
 		IFDI_RXQ_SETUP(ctx, rxq->ifr_id);
 	}
 	return (0);
 #if defined(INET6) || defined(INET)
 fail:
 	/*
 	 * Free RX software descriptors allocated so far, we will only handle
 	 * the rings that completed, the failing case will have
 	 * cleaned up for itself. 'q' failed, so its the terminus.
 	 */
 	rxq = ctx->ifc_rxqs;
 	for (i = 0; i < q; ++i, rxq++) {
 		iflib_rx_sds_free(rxq);
 		rxq->ifr_cq_gen = rxq->ifr_cq_cidx = rxq->ifr_cq_pidx = 0;
 	}
 	return (err);
 #endif
 }
 
 /*********************************************************************
  *
  *  Free all receive rings.
  *
  **********************************************************************/
 static void
 iflib_rx_structures_free(if_ctx_t ctx)
 {
 	iflib_rxq_t rxq = ctx->ifc_rxqs;
 
 	for (int i = 0; i < ctx->ifc_softc_ctx.isc_nrxqsets; i++, rxq++) {
 		iflib_rx_sds_free(rxq);
 	}
 	free(ctx->ifc_rxqs, M_IFLIB);
 	ctx->ifc_rxqs = NULL;
 }
 
 static int
 iflib_qset_structures_setup(if_ctx_t ctx)
 {
 	int err;
 
 	/*
 	 * It is expected that the caller takes care of freeing queues if this
 	 * fails.
 	 */
 	if ((err = iflib_tx_structures_setup(ctx)) != 0) {
 		device_printf(ctx->ifc_dev, "iflib_tx_structures_setup failed: %d\n", err);
 		return (err);
 	}
 
 	if ((err = iflib_rx_structures_setup(ctx)) != 0)
 		device_printf(ctx->ifc_dev, "iflib_rx_structures_setup failed: %d\n", err);
 
 	return (err);
 }
 
 int
 iflib_irq_alloc(if_ctx_t ctx, if_irq_t irq, int rid,
 		driver_filter_t filter, void *filter_arg, driver_intr_t handler, void *arg, const char *name)
 {
 
 	return (_iflib_irq_alloc(ctx, irq, rid, filter, handler, arg, name));
 }
 
 #ifdef SMP
 static int
 find_nth(if_ctx_t ctx, int qid)
 {
 	cpuset_t cpus;
 	int i, cpuid, eqid, count;
 
 	CPU_COPY(&ctx->ifc_cpus, &cpus);
 	count = CPU_COUNT(&cpus);
 	eqid = qid % count;
 	/* clear up to the qid'th bit */
 	for (i = 0; i < eqid; i++) {
 		cpuid = CPU_FFS(&cpus);
 		MPASS(cpuid != 0);
 		CPU_CLR(cpuid-1, &cpus);
 	}
 	cpuid = CPU_FFS(&cpus);
 	MPASS(cpuid != 0);
 	return (cpuid-1);
 }
 
 #ifdef SCHED_ULE
 extern struct cpu_group *cpu_top;              /* CPU topology */
 
 static int
 find_child_with_core(int cpu, struct cpu_group *grp)
 {
 	int i;
 
 	if (grp->cg_children == 0)
 		return -1;
 
 	MPASS(grp->cg_child);
 	for (i = 0; i < grp->cg_children; i++) {
 		if (CPU_ISSET(cpu, &grp->cg_child[i].cg_mask))
 			return i;
 	}
 
 	return -1;
 }
 
 /*
  * Find the nth "close" core to the specified core
  * "close" is defined as the deepest level that shares
  * at least an L2 cache.  With threads, this will be
  * threads on the same core.  If the sahred cache is L3
  * or higher, simply returns the same core.
  */
 static int
 find_close_core(int cpu, int core_offset)
 {
 	struct cpu_group *grp;
 	int i;
 	int fcpu;
 	cpuset_t cs;
 
 	grp = cpu_top;
 	if (grp == NULL)
 		return cpu;
 	i = 0;
 	while ((i = find_child_with_core(cpu, grp)) != -1) {
 		/* If the child only has one cpu, don't descend */
 		if (grp->cg_child[i].cg_count <= 1)
 			break;
 		grp = &grp->cg_child[i];
 	}
 
 	/* If they don't share at least an L2 cache, use the same CPU */
 	if (grp->cg_level > CG_SHARE_L2 || grp->cg_level == CG_SHARE_NONE)
 		return cpu;
 
 	/* Now pick one */
 	CPU_COPY(&grp->cg_mask, &cs);
 
 	/* Add the selected CPU offset to core offset. */
 	for (i = 0; (fcpu = CPU_FFS(&cs)) != 0; i++) {
 		if (fcpu - 1 == cpu)
 			break;
 		CPU_CLR(fcpu - 1, &cs);
 	}
 	MPASS(fcpu);
 
 	core_offset += i;
 
 	CPU_COPY(&grp->cg_mask, &cs);
 	for (i = core_offset % grp->cg_count; i > 0; i--) {
 		MPASS(CPU_FFS(&cs));
 		CPU_CLR(CPU_FFS(&cs) - 1, &cs);
 	}
 	MPASS(CPU_FFS(&cs));
 	return CPU_FFS(&cs) - 1;
 }
 #else
 static int
 find_close_core(int cpu, int core_offset __unused)
 {
 	return cpu;
 }
 #endif
 
 static int
 get_core_offset(if_ctx_t ctx, iflib_intr_type_t type, int qid)
 {
 	switch (type) {
 	case IFLIB_INTR_TX:
 		/* TX queues get cores which share at least an L2 cache with the corresponding RX queue */
 		/* XXX handle multiple RX threads per core and more than two core per L2 group */
 		return qid / CPU_COUNT(&ctx->ifc_cpus) + 1;
 	case IFLIB_INTR_RX:
 	case IFLIB_INTR_RXTX:
 		/* RX queues get the specified core */
 		return qid / CPU_COUNT(&ctx->ifc_cpus);
 	default:
 		return -1;
 	}
 }
 #else
 #define get_core_offset(ctx, type, qid)	CPU_FIRST()
 #define find_close_core(cpuid, tid)	CPU_FIRST()
 #define find_nth(ctx, gid)		CPU_FIRST()
 #endif
 
 /* Just to avoid copy/paste */
 static inline int
 iflib_irq_set_affinity(if_ctx_t ctx, if_irq_t irq, iflib_intr_type_t type,
     int qid, struct grouptask *gtask, struct taskqgroup *tqg, void *uniq,
     const char *name)
 {
 	device_t dev;
 	int err, cpuid, tid;
 
 	dev = ctx->ifc_dev;
 	cpuid = find_nth(ctx, qid);
 	tid = get_core_offset(ctx, type, qid);
 	MPASS(tid >= 0);
 	cpuid = find_close_core(cpuid, tid);
 	err = taskqgroup_attach_cpu(tqg, gtask, uniq, cpuid, dev, irq->ii_res,
 	    name);
 	if (err) {
 		device_printf(dev, "taskqgroup_attach_cpu failed %d\n", err);
 		return (err);
 	}
 #ifdef notyet
 	if (cpuid > ctx->ifc_cpuid_highest)
 		ctx->ifc_cpuid_highest = cpuid;
 #endif
 	return 0;
 }
 
 int
 iflib_irq_alloc_generic(if_ctx_t ctx, if_irq_t irq, int rid,
 			iflib_intr_type_t type, driver_filter_t *filter,
 			void *filter_arg, int qid, const char *name)
 {
 	device_t dev;
 	struct grouptask *gtask;
 	struct taskqgroup *tqg;
 	iflib_filter_info_t info;
 	gtask_fn_t *fn;
 	int tqrid, err;
 	driver_filter_t *intr_fast;
 	void *q;
 
 	info = &ctx->ifc_filter_info;
 	tqrid = rid;
 
 	switch (type) {
 	/* XXX merge tx/rx for netmap? */
 	case IFLIB_INTR_TX:
 		q = &ctx->ifc_txqs[qid];
 		info = &ctx->ifc_txqs[qid].ift_filter_info;
 		gtask = &ctx->ifc_txqs[qid].ift_task;
 		tqg = qgroup_if_io_tqg;
 		fn = _task_fn_tx;
 		intr_fast = iflib_fast_intr;
 		GROUPTASK_INIT(gtask, 0, fn, q);
 		ctx->ifc_flags |= IFC_NETMAP_TX_IRQ;
 		break;
 	case IFLIB_INTR_RX:
 		q = &ctx->ifc_rxqs[qid];
 		info = &ctx->ifc_rxqs[qid].ifr_filter_info;
 		gtask = &ctx->ifc_rxqs[qid].ifr_task;
 		tqg = qgroup_if_io_tqg;
 		fn = _task_fn_rx;
 		intr_fast = iflib_fast_intr;
 		GROUPTASK_INIT(gtask, 0, fn, q);
 		break;
 	case IFLIB_INTR_RXTX:
 		q = &ctx->ifc_rxqs[qid];
 		info = &ctx->ifc_rxqs[qid].ifr_filter_info;
 		gtask = &ctx->ifc_rxqs[qid].ifr_task;
 		tqg = qgroup_if_io_tqg;
 		fn = _task_fn_rx;
 		intr_fast = iflib_fast_intr_rxtx;
 		GROUPTASK_INIT(gtask, 0, fn, q);
 		break;
 	case IFLIB_INTR_ADMIN:
 		q = ctx;
 		tqrid = -1;
 		info = &ctx->ifc_filter_info;
 		gtask = &ctx->ifc_admin_task;
 		tqg = qgroup_if_config_tqg;
 		fn = _task_fn_admin;
 		intr_fast = iflib_fast_intr_ctx;
 		break;
 	default:
 		panic("unknown net intr type");
 	}
 
 	info->ifi_filter = filter;
 	info->ifi_filter_arg = filter_arg;
 	info->ifi_task = gtask;
 	info->ifi_ctx = q;
 
 	dev = ctx->ifc_dev;
 	err = _iflib_irq_alloc(ctx, irq, rid, intr_fast, NULL, info,  name);
 	if (err != 0) {
 		device_printf(dev, "_iflib_irq_alloc failed %d\n", err);
 		return (err);
 	}
 	if (type == IFLIB_INTR_ADMIN)
 		return (0);
 
 	if (tqrid != -1) {
 		err = iflib_irq_set_affinity(ctx, irq, type, qid, gtask, tqg,
 		    q, name);
 		if (err)
 			return (err);
 	} else {
 		taskqgroup_attach(tqg, gtask, q, dev, irq->ii_res, name);
 	}
 
 	return (0);
 }
 
 void
 iflib_softirq_alloc_generic(if_ctx_t ctx, if_irq_t irq, iflib_intr_type_t type, void *arg, int qid, const char *name)
 {
 	struct grouptask *gtask;
 	struct taskqgroup *tqg;
 	gtask_fn_t *fn;
 	void *q;
 	int err;
 
 	switch (type) {
 	case IFLIB_INTR_TX:
 		q = &ctx->ifc_txqs[qid];
 		gtask = &ctx->ifc_txqs[qid].ift_task;
 		tqg = qgroup_if_io_tqg;
 		fn = _task_fn_tx;
 		break;
 	case IFLIB_INTR_RX:
 		q = &ctx->ifc_rxqs[qid];
 		gtask = &ctx->ifc_rxqs[qid].ifr_task;
 		tqg = qgroup_if_io_tqg;
 		fn = _task_fn_rx;
 		break;
 	case IFLIB_INTR_IOV:
 		q = ctx;
 		gtask = &ctx->ifc_vflr_task;
 		tqg = qgroup_if_config_tqg;
 		fn = _task_fn_iov;
 		break;
 	default:
 		panic("unknown net intr type");
 	}
 	GROUPTASK_INIT(gtask, 0, fn, q);
 	if (irq != NULL) {
 		err = iflib_irq_set_affinity(ctx, irq, type, qid, gtask, tqg,
 		    q, name);
 		if (err)
 			taskqgroup_attach(tqg, gtask, q, ctx->ifc_dev,
 			    irq->ii_res, name);
 	} else {
 		taskqgroup_attach(tqg, gtask, q, NULL, NULL, name);
 	}
 }
 
 void
 iflib_irq_free(if_ctx_t ctx, if_irq_t irq)
 {
 
 	if (irq->ii_tag)
 		bus_teardown_intr(ctx->ifc_dev, irq->ii_res, irq->ii_tag);
 
 	if (irq->ii_res)
 		bus_release_resource(ctx->ifc_dev, SYS_RES_IRQ,
 		    rman_get_rid(irq->ii_res), irq->ii_res);
 }
 
 static int
 iflib_legacy_setup(if_ctx_t ctx, driver_filter_t filter, void *filter_arg, int *rid, const char *name)
 {
 	iflib_txq_t txq = ctx->ifc_txqs;
 	iflib_rxq_t rxq = ctx->ifc_rxqs;
 	if_irq_t irq = &ctx->ifc_legacy_irq;
 	iflib_filter_info_t info;
 	device_t dev;
 	struct grouptask *gtask;
 	struct resource *res;
 	struct taskqgroup *tqg;
 	gtask_fn_t *fn;
 	int tqrid;
 	void *q;
 	int err;
 
 	q = &ctx->ifc_rxqs[0];
 	info = &rxq[0].ifr_filter_info;
 	gtask = &rxq[0].ifr_task;
 	tqg = qgroup_if_io_tqg;
 	tqrid = irq->ii_rid = *rid;
 	fn = _task_fn_rx;
 
 	ctx->ifc_flags |= IFC_LEGACY;
 	info->ifi_filter = filter;
 	info->ifi_filter_arg = filter_arg;
 	info->ifi_task = gtask;
 	info->ifi_ctx = ctx;
 
 	dev = ctx->ifc_dev;
 	/* We allocate a single interrupt resource */
 	if ((err = _iflib_irq_alloc(ctx, irq, tqrid, iflib_fast_intr_ctx, NULL, info, name)) != 0)
 		return (err);
 	GROUPTASK_INIT(gtask, 0, fn, q);
 	res = irq->ii_res;
 	taskqgroup_attach(tqg, gtask, q, dev, res, name);
 
 	GROUPTASK_INIT(&txq->ift_task, 0, _task_fn_tx, txq);
 	taskqgroup_attach(qgroup_if_io_tqg, &txq->ift_task, txq, dev, res,
 	    "tx");
 	return (0);
 }
 
 void
 iflib_led_create(if_ctx_t ctx)
 {
 
 	ctx->ifc_led_dev = led_create(iflib_led_func, ctx,
 	    device_get_nameunit(ctx->ifc_dev));
 }
 
 void
 iflib_tx_intr_deferred(if_ctx_t ctx, int txqid)
 {
 
 	GROUPTASK_ENQUEUE(&ctx->ifc_txqs[txqid].ift_task);
 }
 
 void
 iflib_rx_intr_deferred(if_ctx_t ctx, int rxqid)
 {
 
 	GROUPTASK_ENQUEUE(&ctx->ifc_rxqs[rxqid].ifr_task);
 }
 
 void
 iflib_admin_intr_deferred(if_ctx_t ctx)
 {
 #ifdef INVARIANTS
 	struct grouptask *gtask;
 
 	gtask = &ctx->ifc_admin_task;
 	MPASS(gtask != NULL && gtask->gt_taskqueue != NULL);
 #endif
 
 	GROUPTASK_ENQUEUE(&ctx->ifc_admin_task);
 }
 
 void
 iflib_iov_intr_deferred(if_ctx_t ctx)
 {
 
 	GROUPTASK_ENQUEUE(&ctx->ifc_vflr_task);
 }
 
 void
 iflib_io_tqg_attach(struct grouptask *gt, void *uniq, int cpu, char *name)
 {
 
 	taskqgroup_attach_cpu(qgroup_if_io_tqg, gt, uniq, cpu, NULL, NULL,
 	    name);
 }
 
 void
 iflib_config_gtask_init(void *ctx, struct grouptask *gtask, gtask_fn_t *fn,
 	const char *name)
 {
 
 	GROUPTASK_INIT(gtask, 0, fn, ctx);
 	taskqgroup_attach(qgroup_if_config_tqg, gtask, gtask, NULL, NULL,
 	    name);
 }
 
 void
 iflib_config_gtask_deinit(struct grouptask *gtask)
 {
 
 	taskqgroup_detach(qgroup_if_config_tqg, gtask);	
 }
 
 void
 iflib_link_state_change(if_ctx_t ctx, int link_state, uint64_t baudrate)
 {
 	if_t ifp = ctx->ifc_ifp;
 	iflib_txq_t txq = ctx->ifc_txqs;
 
 	if_setbaudrate(ifp, baudrate);
 	if (baudrate >= IF_Gbps(10)) {
 		STATE_LOCK(ctx);
 		ctx->ifc_flags |= IFC_PREFETCH;
 		STATE_UNLOCK(ctx);
 	}
 	/* If link down, disable watchdog */
 	if ((ctx->ifc_link_state == LINK_STATE_UP) && (link_state == LINK_STATE_DOWN)) {
 		for (int i = 0; i < ctx->ifc_softc_ctx.isc_ntxqsets; i++, txq++)
 			txq->ift_qstatus = IFLIB_QUEUE_IDLE;
 	}
 	ctx->ifc_link_state = link_state;
 	if_link_state_change(ifp, link_state);
 }
 
 static int
 iflib_tx_credits_update(if_ctx_t ctx, iflib_txq_t txq)
 {
 	int credits;
 #ifdef INVARIANTS
 	int credits_pre = txq->ift_cidx_processed;
 #endif
 
 	if (ctx->isc_txd_credits_update == NULL)
 		return (0);
 
 	bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,
 	    BUS_DMASYNC_POSTREAD);
 	if ((credits = ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id, true)) == 0)
 		return (0);
 
 	txq->ift_processed += credits;
 	txq->ift_cidx_processed += credits;
 
 	MPASS(credits_pre + credits == txq->ift_cidx_processed);
 	if (txq->ift_cidx_processed >= txq->ift_size)
 		txq->ift_cidx_processed -= txq->ift_size;
 	return (credits);
 }
 
 static int
 iflib_rxd_avail(if_ctx_t ctx, iflib_rxq_t rxq, qidx_t cidx, qidx_t budget)
 {
 	iflib_fl_t fl;
 	u_int i;
 
 	for (i = 0, fl = &rxq->ifr_fl[0]; i < rxq->ifr_nfl; i++, fl++)
 		bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map,
 		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 	return (ctx->isc_rxd_available(ctx->ifc_softc, rxq->ifr_id, cidx,
 	    budget));
 }
 
 void
 iflib_add_int_delay_sysctl(if_ctx_t ctx, const char *name,
 	const char *description, if_int_delay_info_t info,
 	int offset, int value)
 {
 	info->iidi_ctx = ctx;
 	info->iidi_offset = offset;
 	info->iidi_value = value;
 	SYSCTL_ADD_PROC(device_get_sysctl_ctx(ctx->ifc_dev),
 	    SYSCTL_CHILDREN(device_get_sysctl_tree(ctx->ifc_dev)),
 	    OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW,
 	    info, 0, iflib_sysctl_int_delay, "I", description);
 }
 
 struct sx *
 iflib_ctx_lock_get(if_ctx_t ctx)
 {
 
 	return (&ctx->ifc_ctx_sx);
 }
 
 static int
 iflib_msix_init(if_ctx_t ctx)
 {
 	device_t dev = ctx->ifc_dev;
 	if_shared_ctx_t sctx = ctx->ifc_sctx;
 	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
 	int vectors, queues, rx_queues, tx_queues, queuemsgs, msgs;
 	int iflib_num_tx_queues, iflib_num_rx_queues;
 	int err, admincnt, bar;
 
 	iflib_num_tx_queues = ctx->ifc_sysctl_ntxqs;
 	iflib_num_rx_queues = ctx->ifc_sysctl_nrxqs;
 
 	if (bootverbose)
 		device_printf(dev, "msix_init qsets capped at %d\n",
 		    imax(scctx->isc_ntxqsets, scctx->isc_nrxqsets));
 
 	bar = ctx->ifc_softc_ctx.isc_msix_bar;
 	admincnt = sctx->isc_admin_intrcnt;
 	/* Override by tuneable */
 	if (scctx->isc_disable_msix)
 		goto msi;
 
 	/* First try MSI-X */
 	if ((msgs = pci_msix_count(dev)) == 0) {
 		if (bootverbose)
 			device_printf(dev, "MSI-X not supported or disabled\n");
 		goto msi;
 	}
 	/*
 	 * bar == -1 => "trust me I know what I'm doing"
 	 * Some drivers are for hardware that is so shoddily
 	 * documented that no one knows which bars are which
 	 * so the developer has to map all bars. This hack
 	 * allows shoddy garbage to use MSI-X in this framework.
 	 */
 	if (bar != -1) {
 		ctx->ifc_msix_mem = bus_alloc_resource_any(dev,
 	            SYS_RES_MEMORY, &bar, RF_ACTIVE);
 		if (ctx->ifc_msix_mem == NULL) {
 			device_printf(dev, "Unable to map MSI-X table\n");
 			goto msi;
 		}
 	}
 #if IFLIB_DEBUG
 	/* use only 1 qset in debug mode */
 	queuemsgs = min(msgs - admincnt, 1);
 #else
 	queuemsgs = msgs - admincnt;
 #endif
 #ifdef RSS
 	queues = imin(queuemsgs, rss_getnumbuckets());
 #else
 	queues = queuemsgs;
 #endif
 	queues = imin(CPU_COUNT(&ctx->ifc_cpus), queues);
 	if (bootverbose)
 		device_printf(dev,
 		    "intr CPUs: %d queue msgs: %d admincnt: %d\n",
 		    CPU_COUNT(&ctx->ifc_cpus), queuemsgs, admincnt);
 #ifdef  RSS
 	/* If we're doing RSS, clamp at the number of RSS buckets */
 	if (queues > rss_getnumbuckets())
 		queues = rss_getnumbuckets();
 #endif
 	if (iflib_num_rx_queues > 0 && iflib_num_rx_queues < queuemsgs - admincnt)
 		rx_queues = iflib_num_rx_queues;
 	else
 		rx_queues = queues;
 
 	if (rx_queues > scctx->isc_nrxqsets)
 		rx_queues = scctx->isc_nrxqsets;
 
 	/*
 	 * We want this to be all logical CPUs by default
 	 */
 	if (iflib_num_tx_queues > 0 && iflib_num_tx_queues < queues)
 		tx_queues = iflib_num_tx_queues;
 	else
 		tx_queues = mp_ncpus;
 
 	if (tx_queues > scctx->isc_ntxqsets)
 		tx_queues = scctx->isc_ntxqsets;
 
 	if (ctx->ifc_sysctl_qs_eq_override == 0) {
 #ifdef INVARIANTS
 		if (tx_queues != rx_queues)
 			device_printf(dev,
 			    "queue equality override not set, capping rx_queues at %d and tx_queues at %d\n",
 			    min(rx_queues, tx_queues), min(rx_queues, tx_queues));
 #endif
 		tx_queues = min(rx_queues, tx_queues);
 		rx_queues = min(rx_queues, tx_queues);
 	}
 
 	device_printf(dev, "Using %d rx queues %d tx queues\n",
 	    rx_queues, tx_queues);
 
 	vectors = rx_queues + admincnt;
 	if ((err = pci_alloc_msix(dev, &vectors)) == 0) {
 		device_printf(dev, "Using MSI-X interrupts with %d vectors\n",
 		    vectors);
 		scctx->isc_vectors = vectors;
 		scctx->isc_nrxqsets = rx_queues;
 		scctx->isc_ntxqsets = tx_queues;
 		scctx->isc_intr = IFLIB_INTR_MSIX;
 
 		return (vectors);
 	} else {
 		device_printf(dev,
 		    "failed to allocate %d MSI-X vectors, err: %d - using MSI\n",
 		    vectors, err);
 		bus_release_resource(dev, SYS_RES_MEMORY, bar,
 		    ctx->ifc_msix_mem);
 		ctx->ifc_msix_mem = NULL;
 	}
 msi:
 	vectors = pci_msi_count(dev);
 	scctx->isc_nrxqsets = 1;
 	scctx->isc_ntxqsets = 1;
 	scctx->isc_vectors = vectors;
 	if (vectors == 1 && pci_alloc_msi(dev, &vectors) == 0) {
 		device_printf(dev,"Using an MSI interrupt\n");
 		scctx->isc_intr = IFLIB_INTR_MSI;
 	} else {
 		scctx->isc_vectors = 1;
 		device_printf(dev,"Using a Legacy interrupt\n");
 		scctx->isc_intr = IFLIB_INTR_LEGACY;
 	}
 
 	return (vectors);
 }
 
 static const char *ring_states[] = { "IDLE", "BUSY", "STALLED", "ABDICATED" };
 
 static int
 mp_ring_state_handler(SYSCTL_HANDLER_ARGS)
 {
 	int rc;
 	uint16_t *state = ((uint16_t *)oidp->oid_arg1);
 	struct sbuf *sb;
 	const char *ring_state = "UNKNOWN";
 
 	/* XXX needed ? */
 	rc = sysctl_wire_old_buffer(req, 0);
 	MPASS(rc == 0);
 	if (rc != 0)
 		return (rc);
 	sb = sbuf_new_for_sysctl(NULL, NULL, 80, req);
 	MPASS(sb != NULL);
 	if (sb == NULL)
 		return (ENOMEM);
 	if (state[3] <= 3)
 		ring_state = ring_states[state[3]];
 
 	sbuf_printf(sb, "pidx_head: %04hd pidx_tail: %04hd cidx: %04hd state: %s",
 		    state[0], state[1], state[2], ring_state);
 	rc = sbuf_finish(sb);
 	sbuf_delete(sb);
         return(rc);
 }
 
 enum iflib_ndesc_handler {
 	IFLIB_NTXD_HANDLER,
 	IFLIB_NRXD_HANDLER,
 };
 
 static int
 mp_ndesc_handler(SYSCTL_HANDLER_ARGS)
 {
 	if_ctx_t ctx = (void *)arg1;
 	enum iflib_ndesc_handler type = arg2;
 	char buf[256] = {0};
 	qidx_t *ndesc;
 	char *p, *next;
 	int nqs, rc, i;
 
 	MPASS(type == IFLIB_NTXD_HANDLER || type == IFLIB_NRXD_HANDLER);
 
 	nqs = 8;
 	switch(type) {
 	case IFLIB_NTXD_HANDLER:
 		ndesc = ctx->ifc_sysctl_ntxds;
 		if (ctx->ifc_sctx)
 			nqs = ctx->ifc_sctx->isc_ntxqs;
 		break;
 	case IFLIB_NRXD_HANDLER:
 		ndesc = ctx->ifc_sysctl_nrxds;
 		if (ctx->ifc_sctx)
 			nqs = ctx->ifc_sctx->isc_nrxqs;
 		break;
 	default:
 			panic("unhandled type");
 	}
 	if (nqs == 0)
 		nqs = 8;
 
 	for (i=0; i<8; i++) {
 		if (i >= nqs)
 			break;
 		if (i)
 			strcat(buf, ",");
 		sprintf(strchr(buf, 0), "%d", ndesc[i]);
 	}
 
 	rc = sysctl_handle_string(oidp, buf, sizeof(buf), req);
 	if (rc || req->newptr == NULL)
 		return rc;
 
 	for (i = 0, next = buf, p = strsep(&next, " ,"); i < 8 && p;
 	    i++, p = strsep(&next, " ,")) {
 		ndesc[i] = strtoul(p, NULL, 10);
 	}
 
 	return(rc);
 }
 
 #define NAME_BUFLEN 32
 static void
 iflib_add_device_sysctl_pre(if_ctx_t ctx)
 {
         device_t dev = iflib_get_dev(ctx);
 	struct sysctl_oid_list *child, *oid_list;
 	struct sysctl_ctx_list *ctx_list;
 	struct sysctl_oid *node;
 
 	ctx_list = device_get_sysctl_ctx(dev);
 	child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev));
 	ctx->ifc_sysctl_node = node = SYSCTL_ADD_NODE(ctx_list, child, OID_AUTO, "iflib",
 						      CTLFLAG_RD, NULL, "IFLIB fields");
 	oid_list = SYSCTL_CHILDREN(node);
 
 	SYSCTL_ADD_STRING(ctx_list, oid_list, OID_AUTO, "driver_version",
 		       CTLFLAG_RD, ctx->ifc_sctx->isc_driver_version, 0,
 		       "driver version");
 
 	SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "override_ntxqs",
 		       CTLFLAG_RWTUN, &ctx->ifc_sysctl_ntxqs, 0,
 			"# of txqs to use, 0 => use default #");
 	SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "override_nrxqs",
 		       CTLFLAG_RWTUN, &ctx->ifc_sysctl_nrxqs, 0,
 			"# of rxqs to use, 0 => use default #");
 	SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "override_qs_enable",
 		       CTLFLAG_RWTUN, &ctx->ifc_sysctl_qs_eq_override, 0,
                        "permit #txq != #rxq");
 	SYSCTL_ADD_INT(ctx_list, oid_list, OID_AUTO, "disable_msix",
                       CTLFLAG_RWTUN, &ctx->ifc_softc_ctx.isc_disable_msix, 0,
                       "disable MSI-X (default 0)");
 	SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "rx_budget",
 		       CTLFLAG_RWTUN, &ctx->ifc_sysctl_rx_budget, 0,
                        "set the rx budget");
 	SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "tx_abdicate",
 		       CTLFLAG_RWTUN, &ctx->ifc_sysctl_tx_abdicate, 0,
 		       "cause tx to abdicate instead of running to completion");
 
 	/* XXX change for per-queue sizes */
 	SYSCTL_ADD_PROC(ctx_list, oid_list, OID_AUTO, "override_ntxds",
 		       CTLTYPE_STRING|CTLFLAG_RWTUN, ctx, IFLIB_NTXD_HANDLER,
                        mp_ndesc_handler, "A",
                        "list of # of tx descriptors to use, 0 = use default #");
 	SYSCTL_ADD_PROC(ctx_list, oid_list, OID_AUTO, "override_nrxds",
 		       CTLTYPE_STRING|CTLFLAG_RWTUN, ctx, IFLIB_NRXD_HANDLER,
                        mp_ndesc_handler, "A",
                        "list of # of rx descriptors to use, 0 = use default #");
 }
 
 static void
 iflib_add_device_sysctl_post(if_ctx_t ctx)
 {
 	if_shared_ctx_t sctx = ctx->ifc_sctx;
 	if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
         device_t dev = iflib_get_dev(ctx);
 	struct sysctl_oid_list *child;
 	struct sysctl_ctx_list *ctx_list;
 	iflib_fl_t fl;
 	iflib_txq_t txq;
 	iflib_rxq_t rxq;
 	int i, j;
 	char namebuf[NAME_BUFLEN];
 	char *qfmt;
 	struct sysctl_oid *queue_node, *fl_node, *node;
 	struct sysctl_oid_list *queue_list, *fl_list;
 	ctx_list = device_get_sysctl_ctx(dev);
 
 	node = ctx->ifc_sysctl_node;
 	child = SYSCTL_CHILDREN(node);
 
 	if (scctx->isc_ntxqsets > 100)
 		qfmt = "txq%03d";
 	else if (scctx->isc_ntxqsets > 10)
 		qfmt = "txq%02d";
 	else
 		qfmt = "txq%d";
 	for (i = 0, txq = ctx->ifc_txqs; i < scctx->isc_ntxqsets; i++, txq++) {
 		snprintf(namebuf, NAME_BUFLEN, qfmt, i);
 		queue_node = SYSCTL_ADD_NODE(ctx_list, child, OID_AUTO, namebuf,
 					     CTLFLAG_RD, NULL, "Queue Name");
 		queue_list = SYSCTL_CHILDREN(queue_node);
 #if MEMORY_LOGGING
 		SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "txq_dequeued",
 				CTLFLAG_RD,
 				&txq->ift_dequeued, "total mbufs freed");
 		SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "txq_enqueued",
 				CTLFLAG_RD,
 				&txq->ift_enqueued, "total mbufs enqueued");
 #endif
 		SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "mbuf_defrag",
 				   CTLFLAG_RD,
 				   &txq->ift_mbuf_defrag, "# of times m_defrag was called");
 		SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "m_pullups",
 				   CTLFLAG_RD,
 				   &txq->ift_pullups, "# of times m_pullup was called");
 		SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "mbuf_defrag_failed",
 				   CTLFLAG_RD,
 				   &txq->ift_mbuf_defrag_failed, "# of times m_defrag failed");
 		SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "no_desc_avail",
 				   CTLFLAG_RD,
 				   &txq->ift_no_desc_avail, "# of times no descriptors were available");
 		SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "tx_map_failed",
 				   CTLFLAG_RD,
 				   &txq->ift_map_failed, "# of times dma map failed");
 		SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "txd_encap_efbig",
 				   CTLFLAG_RD,
 				   &txq->ift_txd_encap_efbig, "# of times txd_encap returned EFBIG");
 		SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "no_tx_dma_setup",
 				   CTLFLAG_RD,
 				   &txq->ift_no_tx_dma_setup, "# of times map failed for other than EFBIG");
 		SYSCTL_ADD_U16(ctx_list, queue_list, OID_AUTO, "txq_pidx",
 				   CTLFLAG_RD,
 				   &txq->ift_pidx, 1, "Producer Index");
 		SYSCTL_ADD_U16(ctx_list, queue_list, OID_AUTO, "txq_cidx",
 				   CTLFLAG_RD,
 				   &txq->ift_cidx, 1, "Consumer Index");
 		SYSCTL_ADD_U16(ctx_list, queue_list, OID_AUTO, "txq_cidx_processed",
 				   CTLFLAG_RD,
 				   &txq->ift_cidx_processed, 1, "Consumer Index seen by credit update");
 		SYSCTL_ADD_U16(ctx_list, queue_list, OID_AUTO, "txq_in_use",
 				   CTLFLAG_RD,
 				   &txq->ift_in_use, 1, "descriptors in use");
 		SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "txq_processed",
 				   CTLFLAG_RD,
 				   &txq->ift_processed, "descriptors procesed for clean");
 		SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "txq_cleaned",
 				   CTLFLAG_RD,
 				   &txq->ift_cleaned, "total cleaned");
 		SYSCTL_ADD_PROC(ctx_list, queue_list, OID_AUTO, "ring_state",
 				CTLTYPE_STRING | CTLFLAG_RD, __DEVOLATILE(uint64_t *, &txq->ift_br->state),
 				0, mp_ring_state_handler, "A", "soft ring state");
 		SYSCTL_ADD_COUNTER_U64(ctx_list, queue_list, OID_AUTO, "r_enqueues",
 				       CTLFLAG_RD, &txq->ift_br->enqueues,
 				       "# of enqueues to the mp_ring for this queue");
 		SYSCTL_ADD_COUNTER_U64(ctx_list, queue_list, OID_AUTO, "r_drops",
 				       CTLFLAG_RD, &txq->ift_br->drops,
 				       "# of drops in the mp_ring for this queue");
 		SYSCTL_ADD_COUNTER_U64(ctx_list, queue_list, OID_AUTO, "r_starts",
 				       CTLFLAG_RD, &txq->ift_br->starts,
 				       "# of normal consumer starts in the mp_ring for this queue");
 		SYSCTL_ADD_COUNTER_U64(ctx_list, queue_list, OID_AUTO, "r_stalls",
 				       CTLFLAG_RD, &txq->ift_br->stalls,
 					       "# of consumer stalls in the mp_ring for this queue");
 		SYSCTL_ADD_COUNTER_U64(ctx_list, queue_list, OID_AUTO, "r_restarts",
 			       CTLFLAG_RD, &txq->ift_br->restarts,
 				       "# of consumer restarts in the mp_ring for this queue");
 		SYSCTL_ADD_COUNTER_U64(ctx_list, queue_list, OID_AUTO, "r_abdications",
 				       CTLFLAG_RD, &txq->ift_br->abdications,
 				       "# of consumer abdications in the mp_ring for this queue");
 	}
 
 	if (scctx->isc_nrxqsets > 100)
 		qfmt = "rxq%03d";
 	else if (scctx->isc_nrxqsets > 10)
 		qfmt = "rxq%02d";
 	else
 		qfmt = "rxq%d";
 	for (i = 0, rxq = ctx->ifc_rxqs; i < scctx->isc_nrxqsets; i++, rxq++) {
 		snprintf(namebuf, NAME_BUFLEN, qfmt, i);
 		queue_node = SYSCTL_ADD_NODE(ctx_list, child, OID_AUTO, namebuf,
 					     CTLFLAG_RD, NULL, "Queue Name");
 		queue_list = SYSCTL_CHILDREN(queue_node);
 		if (sctx->isc_flags & IFLIB_HAS_RXCQ) {
 			SYSCTL_ADD_U16(ctx_list, queue_list, OID_AUTO, "rxq_cq_pidx",
 				       CTLFLAG_RD,
 				       &rxq->ifr_cq_pidx, 1, "Producer Index");
 			SYSCTL_ADD_U16(ctx_list, queue_list, OID_AUTO, "rxq_cq_cidx",
 				       CTLFLAG_RD,
 				       &rxq->ifr_cq_cidx, 1, "Consumer Index");
 		}
 
 		for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) {
 			snprintf(namebuf, NAME_BUFLEN, "rxq_fl%d", j);
 			fl_node = SYSCTL_ADD_NODE(ctx_list, queue_list, OID_AUTO, namebuf,
 						     CTLFLAG_RD, NULL, "freelist Name");
 			fl_list = SYSCTL_CHILDREN(fl_node);
 			SYSCTL_ADD_U16(ctx_list, fl_list, OID_AUTO, "pidx",
 				       CTLFLAG_RD,
 				       &fl->ifl_pidx, 1, "Producer Index");
 			SYSCTL_ADD_U16(ctx_list, fl_list, OID_AUTO, "cidx",
 				       CTLFLAG_RD,
 				       &fl->ifl_cidx, 1, "Consumer Index");
 			SYSCTL_ADD_U16(ctx_list, fl_list, OID_AUTO, "credits",
 				       CTLFLAG_RD,
 				       &fl->ifl_credits, 1, "credits available");
 #if MEMORY_LOGGING
 			SYSCTL_ADD_QUAD(ctx_list, fl_list, OID_AUTO, "fl_m_enqueued",
 					CTLFLAG_RD,
 					&fl->ifl_m_enqueued, "mbufs allocated");
 			SYSCTL_ADD_QUAD(ctx_list, fl_list, OID_AUTO, "fl_m_dequeued",
 					CTLFLAG_RD,
 					&fl->ifl_m_dequeued, "mbufs freed");
 			SYSCTL_ADD_QUAD(ctx_list, fl_list, OID_AUTO, "fl_cl_enqueued",
 					CTLFLAG_RD,
 					&fl->ifl_cl_enqueued, "clusters allocated");
 			SYSCTL_ADD_QUAD(ctx_list, fl_list, OID_AUTO, "fl_cl_dequeued",
 					CTLFLAG_RD,
 					&fl->ifl_cl_dequeued, "clusters freed");
 #endif
 
 		}
 	}
 
 }
 
 void
 iflib_request_reset(if_ctx_t ctx)
 {
 
 	STATE_LOCK(ctx);
 	ctx->ifc_flags |= IFC_DO_RESET;
 	STATE_UNLOCK(ctx);
 }
 
 #ifndef __NO_STRICT_ALIGNMENT
 static struct mbuf *
 iflib_fixup_rx(struct mbuf *m)
 {
 	struct mbuf *n;
 
 	if (m->m_len <= (MCLBYTES - ETHER_HDR_LEN)) {
 		bcopy(m->m_data, m->m_data + ETHER_HDR_LEN, m->m_len);
 		m->m_data += ETHER_HDR_LEN;
 		n = m;
 	} else {
 		MGETHDR(n, M_NOWAIT, MT_DATA);
 		if (n == NULL) {
 			m_freem(m);
 			return (NULL);
 		}
 		bcopy(m->m_data, n->m_data, ETHER_HDR_LEN);
 		m->m_data += ETHER_HDR_LEN;
 		m->m_len -= ETHER_HDR_LEN;
 		n->m_len = ETHER_HDR_LEN;
 		M_MOVE_PKTHDR(n, m);
 		n->m_next = m;
 	}
 	return (n);
 }
 #endif
 
 #ifdef NETDUMP
 static void
 iflib_netdump_init(struct ifnet *ifp, int *nrxr, int *ncl, int *clsize)
 {
 	if_ctx_t ctx;
 
 	ctx = if_getsoftc(ifp);
 	CTX_LOCK(ctx);
 	*nrxr = NRXQSETS(ctx);
 	*ncl = ctx->ifc_rxqs[0].ifr_fl->ifl_size;
 	*clsize = ctx->ifc_rxqs[0].ifr_fl->ifl_buf_size;
 	CTX_UNLOCK(ctx);
 }
 
 static void
 iflib_netdump_event(struct ifnet *ifp, enum netdump_ev event)
 {
 	if_ctx_t ctx;
 	if_softc_ctx_t scctx;
 	iflib_fl_t fl;
 	iflib_rxq_t rxq;
 	int i, j;
 
 	ctx = if_getsoftc(ifp);
 	scctx = &ctx->ifc_softc_ctx;
 
 	switch (event) {
 	case NETDUMP_START:
 		for (i = 0; i < scctx->isc_nrxqsets; i++) {
 			rxq = &ctx->ifc_rxqs[i];
 			for (j = 0; j < rxq->ifr_nfl; j++) {
 				fl = rxq->ifr_fl;
 				fl->ifl_zone = m_getzone(fl->ifl_buf_size);
 			}
 		}
 		iflib_no_tx_batch = 1;
 		break;
 	default:
 		break;
 	}
 }
 
 static int
 iflib_netdump_transmit(struct ifnet *ifp, struct mbuf *m)
 {
 	if_ctx_t ctx;
 	iflib_txq_t txq;
 	int error;
 
 	ctx = if_getsoftc(ifp);
 	if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
 	    IFF_DRV_RUNNING)
 		return (EBUSY);
 
 	txq = &ctx->ifc_txqs[0];
 	error = iflib_encap(txq, &m);
 	if (error == 0)
 		(void)iflib_txd_db_check(ctx, txq, true, txq->ift_in_use);
 	return (error);
 }
 
 static int
 iflib_netdump_poll(struct ifnet *ifp, int count)
 {
 	if_ctx_t ctx;
 	if_softc_ctx_t scctx;
 	iflib_txq_t txq;
 	int i;
 
 	ctx = if_getsoftc(ifp);
 	scctx = &ctx->ifc_softc_ctx;
 
 	if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
 	    IFF_DRV_RUNNING)
 		return (EBUSY);
 
 	txq = &ctx->ifc_txqs[0];
 	(void)iflib_completed_tx_reclaim(txq, RECLAIM_THRESH(ctx));
 
 	for (i = 0; i < scctx->isc_nrxqsets; i++)
 		(void)iflib_rxeof(&ctx->ifc_rxqs[i], 16 /* XXX */);
 	return (0);
 }
 #endif /* NETDUMP */
Index: head/sys/net/iflib.h
===================================================================
--- head/sys/net/iflib.h	(revision 345304)
+++ head/sys/net/iflib.h	(revision 345305)
@@ -1,455 +1,457 @@
 /*-
  * Copyright (c) 2014-2017, Matthew Macy (mmacy@mattmacy.io)
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
  *
  *  1. Redistributions of source code must retain the above copyright notice,
  *     this list of conditions and the following disclaimer.
  *
  *  2. Neither the name of Matthew Macy nor the names of its
  *     contributors may be used to endorse or promote products derived from
  *     this software without specific prior written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  *
  * $FreeBSD$
  */
 #ifndef __IFLIB_H_
 #define __IFLIB_H_
 
 #include <sys/kobj.h>
 #include <sys/bus.h>
 #include <sys/cpuset.h>
 #include <machine/bus.h>
 #include <sys/nv.h>
 #include <sys/gtaskqueue.h>
 
 struct if_clone;
 
 /*
  * The value type for indexing, limits max descriptors
  * to 65535 can be conditionally redefined to uint32_t
  * in the future if the need arises.
  */
 typedef uint16_t qidx_t;
 #define QIDX_INVALID 0xFFFF
 
 struct iflib_ctx;
 typedef struct iflib_ctx *if_ctx_t;
 struct if_shared_ctx;
 typedef struct if_shared_ctx *if_shared_ctx_t;
 struct if_int_delay_info;
 typedef struct if_int_delay_info  *if_int_delay_info_t;
 struct if_pseudo;
 typedef struct if_pseudo *if_pseudo_t;
 
 /*
  * File organization:
  *  - public structures
  *  - iflib accessors
  *  - iflib utility functions
  *  - iflib core functions
  */
 
 typedef struct if_rxd_frag {
 	uint8_t irf_flid;
 	qidx_t irf_idx;
 	uint16_t irf_len;
 } *if_rxd_frag_t;
 
 /* bnxt supports 64 with hardware LRO enabled */
 #define IFLIB_MAX_RX_SEGS		64
 
 typedef struct if_rxd_info {
 	/* set by iflib */
 	uint16_t iri_qsidx;		/* qset index */
 	uint16_t iri_vtag;		/* vlan tag - if flag set */
 	/* XXX redundant with the new irf_len field */
 	uint16_t iri_len;		/* packet length */
 	qidx_t iri_cidx;		/* consumer index of cq */
 	struct ifnet *iri_ifp;		/* some drivers >1 interface per softc */
 
 	/* updated by driver */
 	if_rxd_frag_t iri_frags;
 	uint32_t iri_flowid;		/* RSS hash for packet */
 	uint32_t iri_csum_flags;	/* m_pkthdr csum flags */
 
 	uint32_t iri_csum_data;		/* m_pkthdr csum data */
 	uint8_t iri_flags;		/* mbuf flags for packet */
 	uint8_t	 iri_nfrags;		/* number of fragments in packet */
 	uint8_t	 iri_rsstype;		/* RSS hash type */
 	uint8_t	 iri_pad;		/* any padding in the received data */
 } *if_rxd_info_t;
 
 typedef struct if_rxd_update {
 	uint64_t	*iru_paddrs;
 	caddr_t		*iru_vaddrs;
 	qidx_t		*iru_idxs;
 	qidx_t		iru_pidx;
 	uint16_t	iru_qsidx;
 	uint16_t	iru_count;
 	uint16_t	iru_buf_size;
 	uint8_t		iru_flidx;
 } *if_rxd_update_t;
 
 #define IPI_TX_INTR	0x1		/* send an interrupt when this packet is sent */
 #define IPI_TX_IPV4	0x2		/* ethertype IPv4 */
 #define IPI_TX_IPV6	0x4		/* ethertype IPv6 */
 
 typedef struct if_pkt_info {
 	bus_dma_segment_t	*ipi_segs;	/* physical addresses */
 	uint32_t		ipi_len;	/* packet length */
 	uint16_t		ipi_qsidx;	/* queue set index */
 	qidx_t			ipi_nsegs;	/* number of segments */
 
 	qidx_t			ipi_ndescs;	/* number of descriptors used by encap */
 	uint16_t		ipi_flags;	/* iflib per-packet flags */
 	qidx_t			ipi_pidx;	/* start pidx for encap */
 	qidx_t			ipi_new_pidx;	/* next available pidx post-encap */
 	/* offload handling */
 	uint8_t			ipi_ehdrlen;	/* ether header length */
 	uint8_t			ipi_ip_hlen;	/* ip header length */
 	uint8_t			ipi_tcp_hlen;	/* tcp header length */
 	uint8_t			ipi_ipproto;	/* ip protocol */
 
 	uint32_t		ipi_csum_flags;	/* packet checksum flags */
 	uint16_t		ipi_tso_segsz;	/* tso segment size */
 	uint16_t		ipi_vtag;	/* VLAN tag */
 	uint16_t		ipi_etype;	/* ether header type */
 	uint8_t			ipi_tcp_hflags;	/* tcp header flags */
 	uint8_t			ipi_mflags;	/* packet mbuf flags */
 
 	uint32_t		ipi_tcp_seq;	/* tcp seqno */
 	uint32_t		ipi_tcp_sum;	/* tcp csum */
 } *if_pkt_info_t;
 
 typedef struct if_irq {
 	struct resource  *ii_res;
 	int               ii_rid;
 	void             *ii_tag;
 } *if_irq_t;
 
 struct if_int_delay_info {
 	if_ctx_t iidi_ctx;	/* Back-pointer to the iflib ctx (softc) */
 	int iidi_offset;			/* Register offset to read/write */
 	int iidi_value;			/* Current value in usecs */
 	struct sysctl_oid *iidi_oidp;
 	struct sysctl_req *iidi_req;
 };
 
 typedef enum {
 	IFLIB_INTR_LEGACY,
 	IFLIB_INTR_MSI,
 	IFLIB_INTR_MSIX
 } iflib_intr_mode_t;
 
 /*
  * This really belongs in pciio.h or some place more general
  * but this is the only consumer for now.
  */
 typedef struct pci_vendor_info {
 	uint32_t	pvi_vendor_id;
 	uint32_t	pvi_device_id;
 	uint32_t	pvi_subvendor_id;
 	uint32_t	pvi_subdevice_id;
 	uint32_t	pvi_rev_id;
 	uint32_t	pvi_class_mask;
 	caddr_t		pvi_name;
 } pci_vendor_info_t;
 
 #define PVID(vendor, devid, name) {vendor, devid, 0, 0, 0, 0, name}
 #define PVID_OEM(vendor, devid, svid, sdevid, revid, name) {vendor, devid, svid, sdevid, revid, 0, name}
 #define PVID_END {0, 0, 0, 0, 0, 0, NULL}
 
 #define IFLIB_PNP_DESCR "U32:vendor;U32:device;U32:subvendor;U32:subdevice;" \
     "U32:revision;U32:class;D:#"
 #define IFLIB_PNP_INFO(b, u, t) \
     MODULE_PNP_INFO(IFLIB_PNP_DESCR, b, u, t, nitems(t) - 1)
 
 typedef struct if_txrx {
 	int (*ift_txd_encap) (void *, if_pkt_info_t);
 	void (*ift_txd_flush) (void *, uint16_t, qidx_t pidx);
 	int (*ift_txd_credits_update) (void *, uint16_t qsidx, bool clear);
 
 	int (*ift_rxd_available) (void *, uint16_t qsidx, qidx_t pidx, qidx_t budget);
 	int (*ift_rxd_pkt_get) (void *, if_rxd_info_t ri);
 	void (*ift_rxd_refill) (void * , if_rxd_update_t iru);
 	void (*ift_rxd_flush) (void *, uint16_t qsidx, uint8_t flidx, qidx_t pidx);
 	int (*ift_legacy_intr) (void *);
 } *if_txrx_t;
 
 typedef struct if_softc_ctx {
 	int isc_vectors;
 	int isc_nrxqsets;
 	int isc_ntxqsets;
 	uint8_t isc_min_tx_latency; /* disable doorbell update batching */
 	uint8_t isc_rx_mvec_enable; /* generate mvecs on rx */
 	uint32_t isc_txrx_budget_bytes_max;
 	int isc_msix_bar;		/* can be model specific - initialize in attach_pre */
 	int isc_tx_nsegments;		/* can be model specific - initialize in attach_pre */
 	int isc_ntxd[8];
 	int isc_nrxd[8];
 
 	uint32_t isc_txqsizes[8];
 	uint32_t isc_rxqsizes[8];
 	/* is there such thing as a descriptor that is more than 248 bytes ? */
 	uint8_t isc_txd_size[8];
 	uint8_t isc_rxd_size[8];
 
 	int isc_tx_tso_segments_max;
 	int isc_tx_tso_size_max;
 	int isc_tx_tso_segsize_max;
 	int isc_tx_csum_flags;
 	int isc_capabilities;
 	int isc_capenable;
 	int isc_rss_table_size;
 	int isc_rss_table_mask;
 	int isc_nrxqsets_max;
 	int isc_ntxqsets_max;
 	uint32_t isc_tx_qdepth;
 
 	iflib_intr_mode_t isc_intr;
 	uint16_t isc_max_frame_size; /* set at init time by driver */
 	uint16_t isc_min_frame_size; /* set at init time by driver, only used if
 					IFLIB_NEED_ETHER_PAD is set. */
 	uint32_t isc_pause_frames;   /* set by driver for iflib_timer to detect */
 	pci_vendor_info_t isc_vendor_info;	/* set by iflib prior to attach_pre */
 	int isc_disable_msix;
 	if_txrx_t isc_txrx;
 } *if_softc_ctx_t;
 
 /*
  * Initialization values for device
  */
 struct if_shared_ctx {
 	unsigned isc_magic;
 	driver_t *isc_driver;
 	bus_size_t isc_q_align;
 	bus_size_t isc_tx_maxsize;
 	bus_size_t isc_tx_maxsegsize;
 	bus_size_t isc_tso_maxsize;
 	bus_size_t isc_tso_maxsegsize;
 	bus_size_t isc_rx_maxsize;
 	bus_size_t isc_rx_maxsegsize;
 	int isc_rx_nsegments;
 	int isc_admin_intrcnt;		/* # of admin/link interrupts */
 
 	/* fields necessary for probe */
 	pci_vendor_info_t *isc_vendor_info;
 	char *isc_driver_version;
 	/* optional function to transform the read values to match the table*/
 	void (*isc_parse_devinfo) (uint16_t *device_id, uint16_t *subvendor_id,
 				   uint16_t *subdevice_id, uint16_t *rev_id);
 	int isc_nrxd_min[8];
 	int isc_nrxd_default[8];
 	int isc_nrxd_max[8];
 	int isc_ntxd_min[8];
 	int isc_ntxd_default[8];
 	int isc_ntxd_max[8];
 
 	/* actively used during operation */
 	int isc_nfl __aligned(CACHE_LINE_SIZE);
 	int isc_ntxqs;			/* # of tx queues per tx qset - usually 1 */
 	int isc_nrxqs;			/* # of rx queues per rx qset - intel 1, chelsio 2, broadcom 3 */
 	int isc_rx_process_limit;
 	int isc_tx_reclaim_thresh;
 	int isc_flags;
 	const char *isc_name;
 };
 
 typedef struct iflib_dma_info {
 	bus_addr_t		idi_paddr;
 	caddr_t			idi_vaddr;
 	bus_dma_tag_t		idi_tag;
 	bus_dmamap_t		idi_map;
 	uint32_t		idi_size;
 } *iflib_dma_info_t;
 
 #define IFLIB_MAGIC 0xCAFEF00D
 
 typedef enum {
 	IFLIB_INTR_RX,
 	IFLIB_INTR_TX,
 	IFLIB_INTR_RXTX,
 	IFLIB_INTR_ADMIN,
 	IFLIB_INTR_IOV,
 } iflib_intr_type_t;
 
 #ifndef ETH_ADDR_LEN
 #define ETH_ADDR_LEN 6
 #endif
 
 
 /*
  * Interface has a separate command queue for RX
  */
 #define IFLIB_HAS_RXCQ		0x01
 /*
  * Driver has already allocated vectors
  */
 #define IFLIB_SKIP_MSIX		0x02
 /*
  * Interface is a virtual function
  */
 #define IFLIB_IS_VF		0x04
 /*
  * Interface has a separate command queue for TX
  */
 #define IFLIB_HAS_TXCQ		0x08
 /*
  * Interface does checksum in place
  */
 #define IFLIB_NEED_SCRATCH	0x10
 /*
  * Interface doesn't expect in_pseudo for th_sum
  */
 #define IFLIB_TSO_INIT_IP	0x20
 /*
  * Interface doesn't align IP header
  */
 #define IFLIB_DO_RX_FIXUP	0x40
 /*
  * Driver needs csum zeroed for offloading
  */
 #define IFLIB_NEED_ZERO_CSUM	0x80
 /*
  * Driver needs frames padded to some minimum length
  */
 #define IFLIB_NEED_ETHER_PAD	0x100
 /*
  * Packets can be freed immediately after encap
  */
 #define IFLIB_TXD_ENCAP_PIO	0x00200
 /*
  * Use RX completion handler
  */
 #define IFLIB_RX_COMPLETION	0x00400
 /*
  * Skip refilling cluster free lists
  */
 #define IFLIB_SKIP_CLREFILL	0x00800
 /*
  * Don't reset on hang
  */
 #define IFLIB_NO_HANG_RESET	0x01000
 /*
  * Don't need/want most of the niceties of
  * queue management
  */
 #define IFLIB_PSEUDO	0x02000
 /*
  * No DMA support needed / wanted
  */
 #define IFLIB_VIRTUAL	0x04000
 /*
  * autogenerate a MAC address
  */
 #define IFLIB_GEN_MAC	0x08000
 /*
  * Interface needs admin task to ignore interface up/down status
  */
 #define IFLIB_ADMIN_ALWAYS_RUN	0x10000
 
 
 /*
  * field accessors
  */
 void *iflib_get_softc(if_ctx_t ctx);
 
 device_t iflib_get_dev(if_ctx_t ctx);
 
 if_t iflib_get_ifp(if_ctx_t ctx);
 
 struct ifmedia *iflib_get_media(if_ctx_t ctx);
 
 if_softc_ctx_t iflib_get_softc_ctx(if_ctx_t ctx);
 if_shared_ctx_t iflib_get_sctx(if_ctx_t ctx);
 
 void iflib_set_mac(if_ctx_t ctx, uint8_t mac[ETHER_ADDR_LEN]);
 void iflib_request_reset(if_ctx_t ctx);
 uint8_t iflib_in_detach(if_ctx_t ctx);
 
+uint32_t iflib_get_rx_mbuf_sz(if_ctx_t ctx);
+
 /*
  * If the driver can plug cleanly in to newbus use these
  */
 int iflib_device_probe(device_t);
 int iflib_device_attach(device_t);
 int iflib_device_detach(device_t);
 int iflib_device_suspend(device_t);
 int iflib_device_resume(device_t);
 int iflib_device_shutdown(device_t);
 
 
 int iflib_device_iov_init(device_t, uint16_t, const nvlist_t *);
 void iflib_device_iov_uninit(device_t);
 int iflib_device_iov_add_vf(device_t, uint16_t, const nvlist_t *);
 
 /*
  * If the driver can't plug cleanly in to newbus
  * use these
  */
 int iflib_device_register(device_t dev, void *softc, if_shared_ctx_t sctx, if_ctx_t *ctxp);
 int iflib_device_deregister(if_ctx_t);
 
 
 
 int iflib_irq_alloc(if_ctx_t, if_irq_t, int, driver_filter_t, void *filter_arg, driver_intr_t, void *arg, const char *name);
 int iflib_irq_alloc_generic(if_ctx_t ctx, if_irq_t irq, int rid,
 			    iflib_intr_type_t type, driver_filter_t *filter,
 			    void *filter_arg, int qid, const char *name);
 void iflib_softirq_alloc_generic(if_ctx_t ctx, if_irq_t irq, iflib_intr_type_t type,  void *arg, int qid, const char *name);
 
 void iflib_irq_free(if_ctx_t ctx, if_irq_t irq);
 
 void iflib_io_tqg_attach(struct grouptask *gt, void *uniq, int cpu, char *name);
 
 void iflib_config_gtask_init(void *ctx, struct grouptask *gtask,
 			     gtask_fn_t *fn, const char *name);
 
 void iflib_config_gtask_deinit(struct grouptask *gtask);
 
 
 
 void iflib_tx_intr_deferred(if_ctx_t ctx, int txqid);
 void iflib_rx_intr_deferred(if_ctx_t ctx, int rxqid);
 void iflib_admin_intr_deferred(if_ctx_t ctx);
 void iflib_iov_intr_deferred(if_ctx_t ctx);
 
 
 void iflib_link_state_change(if_ctx_t ctx, int linkstate, uint64_t baudrate);
 
 int iflib_dma_alloc(if_ctx_t ctx, int size, iflib_dma_info_t dma, int mapflags);
 int iflib_dma_alloc_align(if_ctx_t ctx, int size, int align, iflib_dma_info_t dma, int mapflags);
 void iflib_dma_free(iflib_dma_info_t dma);
 
 int iflib_dma_alloc_multi(if_ctx_t ctx, int *sizes, iflib_dma_info_t *dmalist, int mapflags, int count);
 
 void iflib_dma_free_multi(iflib_dma_info_t *dmalist, int count);
 
 
 struct sx *iflib_ctx_lock_get(if_ctx_t);
 struct mtx *iflib_qset_lock_get(if_ctx_t, uint16_t);
 
 void iflib_led_create(if_ctx_t ctx);
 
 void iflib_add_int_delay_sysctl(if_ctx_t, const char *, const char *,
 								if_int_delay_info_t, int, int);
 
 /*
  * Pseudo device support
  */
 if_pseudo_t iflib_clone_register(if_shared_ctx_t);
 void iflib_clone_deregister(if_pseudo_t);
 #endif /*  __IFLIB_H_ */
Index: head/sys/sys/param.h
===================================================================
--- head/sys/sys/param.h	(revision 345304)
+++ head/sys/sys/param.h	(revision 345305)
@@ -1,367 +1,367 @@
 /*-
  * SPDX-License-Identifier: BSD-3-Clause
  *
  * Copyright (c) 1982, 1986, 1989, 1993
  *	The Regents of the University of California.  All rights reserved.
  * (c) UNIX System Laboratories, Inc.
  * All or some portions of this file are derived from material licensed
  * to the University of California by American Telephone and Telegraph
  * Co. or Unix System Laboratories, Inc. and are reproduced herein with
  * the permission of UNIX System Laboratories, Inc.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  * 3. Neither the name of the University nor the names of its contributors
  *    may be used to endorse or promote products derived from this software
  *    without specific prior written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
  *	@(#)param.h	8.3 (Berkeley) 4/4/95
  * $FreeBSD$
  */
 
 #ifndef _SYS_PARAM_H_
 #define _SYS_PARAM_H_
 
 #include <sys/_null.h>
 
 #define	BSD	199506		/* System version (year & month). */
 #define BSD4_3	1
 #define BSD4_4	1
 
 /*
  * __FreeBSD_version numbers are documented in the Porter's Handbook.
  * If you bump the version for any reason, you should update the documentation
  * there.
  * Currently this lives here in the doc/ repository:
  *
  *	head/en_US.ISO8859-1/books/porters-handbook/versions/chapter.xml
  *
  * scheme is:  <major><two digit minor>Rxx
  *		'R' is in the range 0 to 4 if this is a release branch or
  *		X.0-CURRENT before releng/X.0 is created, otherwise 'R' is
  *		in the range 5 to 9.
  */
 #undef __FreeBSD_version
-#define __FreeBSD_version 1300016	/* Master, propagated to newvers */
+#define __FreeBSD_version 1300017	/* Master, propagated to newvers */
 
 /*
  * __FreeBSD_kernel__ indicates that this system uses the kernel of FreeBSD,
  * which by definition is always true on FreeBSD. This macro is also defined
  * on other systems that use the kernel of FreeBSD, such as GNU/kFreeBSD.
  *
  * It is tempting to use this macro in userland code when we want to enable
  * kernel-specific routines, and in fact it's fine to do this in code that
  * is part of FreeBSD itself.  However, be aware that as presence of this
  * macro is still not widespread (e.g. older FreeBSD versions, 3rd party
  * compilers, etc), it is STRONGLY DISCOURAGED to check for this macro in
  * external applications without also checking for __FreeBSD__ as an
  * alternative.
  */
 #undef __FreeBSD_kernel__
 #define __FreeBSD_kernel__
 
 #if defined(_KERNEL) || defined(IN_RTLD)
 #define	P_OSREL_SIGWAIT			700000
 #define	P_OSREL_SIGSEGV			700004
 #define	P_OSREL_MAP_ANON		800104
 #define	P_OSREL_MAP_FSTRICT		1100036
 #define	P_OSREL_SHUTDOWN_ENOTCONN	1100077
 #define	P_OSREL_MAP_GUARD		1200035
 #define	P_OSREL_WRFSBASE		1200041
 #define	P_OSREL_CK_CYLGRP		1200046
 #define	P_OSREL_VMTOTAL64		1200054
 #define	P_OSREL_CK_SUPERBLOCK		1300000
 #define	P_OSREL_CK_INODE		1300005
 
 #define	P_OSREL_MAJOR(x)		((x) / 100000)
 #endif
 
 #ifndef LOCORE
 #include <sys/types.h>
 #endif
 
 /*
  * Machine-independent constants (some used in following include files).
  * Redefined constants are from POSIX 1003.1 limits file.
  *
  * MAXCOMLEN should be >= sizeof(ac_comm) (see <acct.h>)
  */
 #include <sys/syslimits.h>
 
 #define	MAXCOMLEN	19		/* max command name remembered */
 #define	MAXINTERP	PATH_MAX	/* max interpreter file name length */
 #define	MAXLOGNAME	33		/* max login name length (incl. NUL) */
 #define	MAXUPRC		CHILD_MAX	/* max simultaneous processes */
 #define	NCARGS		ARG_MAX		/* max bytes for an exec function */
 #define	NGROUPS		(NGROUPS_MAX+1)	/* max number groups */
 #define	NOFILE		OPEN_MAX	/* max open files per process */
 #define	NOGROUP		65535		/* marker for empty group set member */
 #define MAXHOSTNAMELEN	256		/* max hostname size */
 #define SPECNAMELEN	255		/* max length of devicename */
 
 /* More types and definitions used throughout the kernel. */
 #ifdef _KERNEL
 #include <sys/cdefs.h>
 #include <sys/errno.h>
 #ifndef LOCORE
 #include <sys/time.h>
 #include <sys/priority.h>
 #endif
 
 #ifndef FALSE
 #define	FALSE	0
 #endif
 #ifndef TRUE
 #define	TRUE	1
 #endif
 #endif
 
 #ifndef _KERNEL
 /* Signals. */
 #include <sys/signal.h>
 #endif
 
 /* Machine type dependent parameters. */
 #include <machine/param.h>
 #ifndef _KERNEL
 #include <sys/limits.h>
 #endif
 
 #ifndef DEV_BSHIFT
 #define	DEV_BSHIFT	9		/* log2(DEV_BSIZE) */
 #endif
 #define	DEV_BSIZE	(1<<DEV_BSHIFT)
 
 #ifndef BLKDEV_IOSIZE
 #define BLKDEV_IOSIZE  PAGE_SIZE	/* default block device I/O size */
 #endif
 #ifndef DFLTPHYS
 #define DFLTPHYS	(64 * 1024)	/* default max raw I/O transfer size */
 #endif
 #ifndef MAXPHYS
 #define MAXPHYS		(128 * 1024)	/* max raw I/O transfer size */
 #endif
 #ifndef MAXDUMPPGS
 #define MAXDUMPPGS	(DFLTPHYS/PAGE_SIZE)
 #endif
 
 /*
  * Constants related to network buffer management.
  * MCLBYTES must be no larger than PAGE_SIZE.
  */
 #ifndef	MSIZE
 #define	MSIZE		256		/* size of an mbuf */
 #endif
 
 #ifndef	MCLSHIFT
 #define MCLSHIFT	11		/* convert bytes to mbuf clusters */
 #endif	/* MCLSHIFT */
 
 #define MCLBYTES	(1 << MCLSHIFT)	/* size of an mbuf cluster */
 
 #if PAGE_SIZE < 2048
 #define	MJUMPAGESIZE	MCLBYTES
 #elif PAGE_SIZE <= 8192
 #define	MJUMPAGESIZE	PAGE_SIZE
 #else
 #define	MJUMPAGESIZE	(8 * 1024)
 #endif
 
 #define	MJUM9BYTES	(9 * 1024)	/* jumbo cluster 9k */
 #define	MJUM16BYTES	(16 * 1024)	/* jumbo cluster 16k */
 
 /*
  * Some macros for units conversion
  */
 
 /* clicks to bytes */
 #ifndef ctob
 #define ctob(x)	((x)<<PAGE_SHIFT)
 #endif
 
 /* bytes to clicks */
 #ifndef btoc
 #define btoc(x)	(((vm_offset_t)(x)+PAGE_MASK)>>PAGE_SHIFT)
 #endif
 
 /*
  * btodb() is messy and perhaps slow because `bytes' may be an off_t.  We
  * want to shift an unsigned type to avoid sign extension and we don't
  * want to widen `bytes' unnecessarily.  Assume that the result fits in
  * a daddr_t.
  */
 #ifndef btodb
 #define btodb(bytes)	 		/* calculates (bytes / DEV_BSIZE) */ \
 	(sizeof (bytes) > sizeof(long) \
 	 ? (daddr_t)((unsigned long long)(bytes) >> DEV_BSHIFT) \
 	 : (daddr_t)((unsigned long)(bytes) >> DEV_BSHIFT))
 #endif
 
 #ifndef dbtob
 #define dbtob(db)			/* calculates (db * DEV_BSIZE) */ \
 	((off_t)(db) << DEV_BSHIFT)
 #endif
 
 #define	PRIMASK	0x0ff
 #define	PCATCH	0x100		/* OR'd with pri for tsleep to check signals */
 #define	PDROP	0x200	/* OR'd with pri to stop re-entry of interlock mutex */
 
 #define	NZERO	0		/* default "nice" */
 
 #define	NBBY	8		/* number of bits in a byte */
 #define	NBPW	sizeof(int)	/* number of bytes per word (integer) */
 
 #define	CMASK	022		/* default file mask: S_IWGRP|S_IWOTH */
 
 #define	NODEV	(dev_t)(-1)	/* non-existent device */
 
 /*
  * File system parameters and macros.
  *
  * MAXBSIZE -	Filesystems are made out of blocks of at most MAXBSIZE bytes
  *		per block.  MAXBSIZE may be made larger without effecting
  *		any existing filesystems as long as it does not exceed MAXPHYS,
  *		and may be made smaller at the risk of not being able to use
  *		filesystems which require a block size exceeding MAXBSIZE.
  *
  * MAXBCACHEBUF - Maximum size of a buffer in the buffer cache.  This must
  *		be >= MAXBSIZE and can be set differently for different
  *		architectures by defining it in <machine/param.h>.
  *		Making this larger allows NFS to do larger reads/writes.
  *
  * BKVASIZE -	Nominal buffer space per buffer, in bytes.  BKVASIZE is the
  *		minimum KVM memory reservation the kernel is willing to make.
  *		Filesystems can of course request smaller chunks.  Actual
  *		backing memory uses a chunk size of a page (PAGE_SIZE).
  *		The default value here can be overridden on a per-architecture
  *		basis by defining it in <machine/param.h>.
  *
  *		If you make BKVASIZE too small you risk seriously fragmenting
  *		the buffer KVM map which may slow things down a bit.  If you
  *		make it too big the kernel will not be able to optimally use
  *		the KVM memory reserved for the buffer cache and will wind
  *		up with too-few buffers.
  *
  *		The default is 16384, roughly 2x the block size used by a
  *		normal UFS filesystem.
  */
 #define MAXBSIZE	65536	/* must be power of 2 */
 #ifndef	MAXBCACHEBUF
 #define	MAXBCACHEBUF	MAXBSIZE /* must be a power of 2 >= MAXBSIZE */
 #endif
 #ifndef	BKVASIZE
 #define BKVASIZE	16384	/* must be power of 2 */
 #endif
 #define BKVAMASK	(BKVASIZE-1)
 
 /*
  * MAXPATHLEN defines the longest permissible path length after expanding
  * symbolic links. It is used to allocate a temporary buffer from the buffer
  * pool in which to do the name expansion, hence should be a power of two,
  * and must be less than or equal to MAXBSIZE.  MAXSYMLINKS defines the
  * maximum number of symbolic links that may be expanded in a path name.
  * It should be set high enough to allow all legitimate uses, but halt
  * infinite loops reasonably quickly.
  */
 #define	MAXPATHLEN	PATH_MAX
 #define MAXSYMLINKS	32
 
 /* Bit map related macros. */
 #define	setbit(a,i)	(((unsigned char *)(a))[(i)/NBBY] |= 1<<((i)%NBBY))
 #define	clrbit(a,i)	(((unsigned char *)(a))[(i)/NBBY] &= ~(1<<((i)%NBBY)))
 #define	isset(a,i)							\
 	(((const unsigned char *)(a))[(i)/NBBY] & (1<<((i)%NBBY)))
 #define	isclr(a,i)							\
 	((((const unsigned char *)(a))[(i)/NBBY] & (1<<((i)%NBBY))) == 0)
 
 /* Macros for counting and rounding. */
 #ifndef howmany
 #define	howmany(x, y)	(((x)+((y)-1))/(y))
 #endif
 #define	nitems(x)	(sizeof((x)) / sizeof((x)[0]))
 #define	rounddown(x, y)	(((x)/(y))*(y))
 #define	rounddown2(x, y) ((x)&(~((y)-1)))          /* if y is power of two */
 #define	roundup(x, y)	((((x)+((y)-1))/(y))*(y))  /* to any y */
 #define	roundup2(x, y)	(((x)+((y)-1))&(~((y)-1))) /* if y is powers of two */
 #define powerof2(x)	((((x)-1)&(x))==0)
 
 /* Macros for min/max. */
 #define	MIN(a,b) (((a)<(b))?(a):(b))
 #define	MAX(a,b) (((a)>(b))?(a):(b))
 
 #ifdef _KERNEL
 /*
  * Basic byte order function prototypes for non-inline functions.
  */
 #ifndef LOCORE
 #ifndef _BYTEORDER_PROTOTYPED
 #define	_BYTEORDER_PROTOTYPED
 __BEGIN_DECLS
 __uint32_t	 htonl(__uint32_t);
 __uint16_t	 htons(__uint16_t);
 __uint32_t	 ntohl(__uint32_t);
 __uint16_t	 ntohs(__uint16_t);
 __END_DECLS
 #endif
 #endif
 
 #ifndef _BYTEORDER_FUNC_DEFINED
 #define	_BYTEORDER_FUNC_DEFINED
 #define	htonl(x)	__htonl(x)
 #define	htons(x)	__htons(x)
 #define	ntohl(x)	__ntohl(x)
 #define	ntohs(x)	__ntohs(x)
 #endif /* !_BYTEORDER_FUNC_DEFINED */
 #endif /* _KERNEL */
 
 /*
  * Scale factor for scaled integers used to count %cpu time and load avgs.
  *
  * The number of CPU `tick's that map to a unique `%age' can be expressed
  * by the formula (1 / (2 ^ (FSHIFT - 11))).  The maximum load average that
  * can be calculated (assuming 32 bits) can be closely approximated using
  * the formula (2 ^ (2 * (16 - FSHIFT))) for (FSHIFT < 15).
  *
  * For the scheduler to maintain a 1:1 mapping of CPU `tick' to `%age',
  * FSHIFT must be at least 11; this gives us a maximum load avg of ~1024.
  */
 #define	FSHIFT	11		/* bits to right of fixed binary point */
 #define FSCALE	(1<<FSHIFT)
 
 #define dbtoc(db)			/* calculates devblks to pages */ \
 	((db + (ctodb(1) - 1)) >> (PAGE_SHIFT - DEV_BSHIFT))
 
 #define ctodb(db)			/* calculates pages to devblks */ \
 	((db) << (PAGE_SHIFT - DEV_BSHIFT))
 
 /*
  * Old spelling of __containerof().
  */
 #define	member2struct(s, m, x)						\
 	((struct s *)(void *)((char *)(x) - offsetof(struct s, m)))
 
 /*
  * Access a variable length array that has been declared as a fixed
  * length array.
  */
 #define __PAST_END(array, offset) (((__typeof__(*(array)) *)(array))[offset])
 
 #endif	/* _SYS_PARAM_H_ */