Index: stable/12/sys/dev/sfxge/common/ef10_filter.c
===================================================================
--- stable/12/sys/dev/sfxge/common/ef10_filter.c	(revision 342320)
+++ stable/12/sys/dev/sfxge/common/ef10_filter.c	(revision 342321)
@@ -1,1504 +1,1541 @@
 /*-
  * Copyright (c) 2007-2016 Solarflare Communications Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
  *
  * 1. Redistributions of source code must retain the above copyright notice,
  *    this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright notice,
  *    this list of conditions and the following disclaimer in the documentation
  *    and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  * The views and conclusions contained in the software and documentation are
  * those of the authors and should not be interpreted as representing official
  * policies, either expressed or implied, of the FreeBSD Project.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include "efx.h"
 #include "efx_impl.h"
 
 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD
 
 #if EFSYS_OPT_FILTER
 
 #define	EFE_SPEC(eftp, index)	((eftp)->eft_entry[(index)].efe_spec)
 
 static			efx_filter_spec_t *
 ef10_filter_entry_spec(
 	__in		const ef10_filter_table_t *eftp,
 	__in		unsigned int index)
 {
 	return ((efx_filter_spec_t *)(EFE_SPEC(eftp, index) &
 		~(uintptr_t)EFX_EF10_FILTER_FLAGS));
 }
 
 static			boolean_t
 ef10_filter_entry_is_busy(
 	__in		const ef10_filter_table_t *eftp,
 	__in		unsigned int index)
 {
 	if (EFE_SPEC(eftp, index) & EFX_EF10_FILTER_FLAG_BUSY)
 		return (B_TRUE);
 	else
 		return (B_FALSE);
 }
 
 static			boolean_t
 ef10_filter_entry_is_auto_old(
 	__in		const ef10_filter_table_t *eftp,
 	__in		unsigned int index)
 {
 	if (EFE_SPEC(eftp, index) & EFX_EF10_FILTER_FLAG_AUTO_OLD)
 		return (B_TRUE);
 	else
 		return (B_FALSE);
 }
 
 static			void
 ef10_filter_set_entry(
 	__inout		ef10_filter_table_t *eftp,
 	__in		unsigned int index,
 	__in_opt	const efx_filter_spec_t *efsp)
 {
 	EFE_SPEC(eftp, index) = (uintptr_t)efsp;
 }
 
 static			void
 ef10_filter_set_entry_busy(
 	__inout		ef10_filter_table_t *eftp,
 	__in		unsigned int index)
 {
 	EFE_SPEC(eftp, index) |= (uintptr_t)EFX_EF10_FILTER_FLAG_BUSY;
 }
 
 static			void
 ef10_filter_set_entry_not_busy(
 	__inout		ef10_filter_table_t *eftp,
 	__in		unsigned int index)
 {
 	EFE_SPEC(eftp, index) &= ~(uintptr_t)EFX_EF10_FILTER_FLAG_BUSY;
 }
 
 static			void
 ef10_filter_set_entry_auto_old(
 	__inout		ef10_filter_table_t *eftp,
 	__in		unsigned int index)
 {
 	EFSYS_ASSERT(ef10_filter_entry_spec(eftp, index) != NULL);
 	EFE_SPEC(eftp, index) |= (uintptr_t)EFX_EF10_FILTER_FLAG_AUTO_OLD;
 }
 
 static			void
 ef10_filter_set_entry_not_auto_old(
 	__inout		ef10_filter_table_t *eftp,
 	__in		unsigned int index)
 {
 	EFE_SPEC(eftp, index) &= ~(uintptr_t)EFX_EF10_FILTER_FLAG_AUTO_OLD;
 	EFSYS_ASSERT(ef10_filter_entry_spec(eftp, index) != NULL);
 }
 
 	__checkReturn	efx_rc_t
 ef10_filter_init(
 	__in		efx_nic_t *enp)
 {
 	efx_rc_t rc;
 	ef10_filter_table_t *eftp;
 
 	EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON ||
 		    enp->en_family == EFX_FAMILY_MEDFORD);
 
 #define	MATCH_MASK(match) (EFX_MASK32(match) << EFX_LOW_BIT(match))
 	EFX_STATIC_ASSERT(EFX_FILTER_MATCH_REM_HOST ==
 	    MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_SRC_IP));
 	EFX_STATIC_ASSERT(EFX_FILTER_MATCH_LOC_HOST ==
 	    MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_DST_IP));
 	EFX_STATIC_ASSERT(EFX_FILTER_MATCH_REM_MAC ==
 	    MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_SRC_MAC));
 	EFX_STATIC_ASSERT(EFX_FILTER_MATCH_REM_PORT ==
 	    MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_SRC_PORT));
 	EFX_STATIC_ASSERT(EFX_FILTER_MATCH_LOC_MAC ==
 	    MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_DST_MAC));
 	EFX_STATIC_ASSERT(EFX_FILTER_MATCH_LOC_PORT ==
 	    MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_DST_PORT));
 	EFX_STATIC_ASSERT(EFX_FILTER_MATCH_ETHER_TYPE ==
 	    MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_ETHER_TYPE));
 	EFX_STATIC_ASSERT(EFX_FILTER_MATCH_INNER_VID ==
 	    MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_INNER_VLAN));
 	EFX_STATIC_ASSERT(EFX_FILTER_MATCH_OUTER_VID ==
 	    MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_OUTER_VLAN));
 	EFX_STATIC_ASSERT(EFX_FILTER_MATCH_IP_PROTO ==
 	    MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_IP_PROTO));
+	EFX_STATIC_ASSERT(EFX_FILTER_MATCH_IFRM_UNKNOWN_MCAST_DST ==
+	    MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_MCAST_DST));
+	EFX_STATIC_ASSERT(EFX_FILTER_MATCH_IFRM_UNKNOWN_UCAST_DST ==
+	    MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_UCAST_DST));
 	EFX_STATIC_ASSERT(EFX_FILTER_MATCH_UNKNOWN_MCAST_DST ==
 	    MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_MCAST_DST));
 	EFX_STATIC_ASSERT((uint32_t)EFX_FILTER_MATCH_UNKNOWN_UCAST_DST ==
 	    MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_UCAST_DST));
 #undef MATCH_MASK
 
 	EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (ef10_filter_table_t), eftp);
 
 	if (!eftp) {
 		rc = ENOMEM;
 		goto fail1;
 	}
 
 	enp->en_filter.ef_ef10_filter_table = eftp;
 
 	return (0);
 
 fail1:
 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
 
 	return (rc);
 }
 
 			void
 ef10_filter_fini(
 	__in		efx_nic_t *enp)
 {
 	EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON ||
 		    enp->en_family == EFX_FAMILY_MEDFORD);
 
 	if (enp->en_filter.ef_ef10_filter_table != NULL) {
 		EFSYS_KMEM_FREE(enp->en_esip, sizeof (ef10_filter_table_t),
 		    enp->en_filter.ef_ef10_filter_table);
 	}
 }
 
 static	__checkReturn	efx_rc_t
 efx_mcdi_filter_op_add(
 	__in		efx_nic_t *enp,
 	__in		efx_filter_spec_t *spec,
 	__in		unsigned int filter_op,
 	__inout		ef10_filter_handle_t *handle)
 {
 	efx_mcdi_req_t req;
 	uint8_t payload[MAX(MC_CMD_FILTER_OP_EXT_IN_LEN,
 			    MC_CMD_FILTER_OP_EXT_OUT_LEN)];
 	efx_rc_t rc;
 
 	memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_FILTER_OP;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_FILTER_OP_EXT_IN_LEN;
 	req.emr_out_buf = payload;
 	req.emr_out_length = MC_CMD_FILTER_OP_EXT_OUT_LEN;
 
 	switch (filter_op) {
 	case MC_CMD_FILTER_OP_IN_OP_REPLACE:
 		MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_HANDLE_LO,
 		    handle->efh_lo);
 		MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_HANDLE_HI,
 		    handle->efh_hi);
 		/* Fall through */
 	case MC_CMD_FILTER_OP_IN_OP_INSERT:
 	case MC_CMD_FILTER_OP_IN_OP_SUBSCRIBE:
 		MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_OP, filter_op);
 		break;
 	default:
 		EFSYS_ASSERT(0);
 		rc = EINVAL;
 		goto fail1;
 	}
 
 	MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_PORT_ID,
 	    EVB_PORT_ID_ASSIGNED);
 	MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_MATCH_FIELDS,
 	    spec->efs_match_flags);
 	MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_RX_DEST,
 	    MC_CMD_FILTER_OP_EXT_IN_RX_DEST_HOST);
 	MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_RX_QUEUE,
 	    spec->efs_dmaq_id);
 	if (spec->efs_flags & EFX_FILTER_FLAG_RX_RSS) {
 		MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_RX_CONTEXT,
 		    spec->efs_rss_context);
 	}
 	MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_RX_MODE,
 	    spec->efs_flags & EFX_FILTER_FLAG_RX_RSS ?
 	    MC_CMD_FILTER_OP_EXT_IN_RX_MODE_RSS :
 	    MC_CMD_FILTER_OP_EXT_IN_RX_MODE_SIMPLE);
 	MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_TX_DEST,
 	    MC_CMD_FILTER_OP_EXT_IN_TX_DEST_DEFAULT);
 
 	if (filter_op != MC_CMD_FILTER_OP_IN_OP_REPLACE) {
 		/*
 		 * NOTE: Unlike most MCDI requests, the filter fields
 		 * are presented in network (big endian) byte order.
 		 */
 		memcpy(MCDI_IN2(req, uint8_t, FILTER_OP_EXT_IN_SRC_MAC),
 		    spec->efs_rem_mac, EFX_MAC_ADDR_LEN);
 		memcpy(MCDI_IN2(req, uint8_t, FILTER_OP_EXT_IN_DST_MAC),
 		    spec->efs_loc_mac, EFX_MAC_ADDR_LEN);
 
 		MCDI_IN_SET_WORD(req, FILTER_OP_EXT_IN_SRC_PORT,
 		    __CPU_TO_BE_16(spec->efs_rem_port));
 		MCDI_IN_SET_WORD(req, FILTER_OP_EXT_IN_DST_PORT,
 		    __CPU_TO_BE_16(spec->efs_loc_port));
 
 		MCDI_IN_SET_WORD(req, FILTER_OP_EXT_IN_ETHER_TYPE,
 		    __CPU_TO_BE_16(spec->efs_ether_type));
 
 		MCDI_IN_SET_WORD(req, FILTER_OP_EXT_IN_INNER_VLAN,
 		    __CPU_TO_BE_16(spec->efs_inner_vid));
 		MCDI_IN_SET_WORD(req, FILTER_OP_EXT_IN_OUTER_VLAN,
 		    __CPU_TO_BE_16(spec->efs_outer_vid));
 
 		/* IP protocol (in low byte, high byte is zero) */
 		MCDI_IN_SET_BYTE(req, FILTER_OP_EXT_IN_IP_PROTO,
 		    spec->efs_ip_proto);
 
 		EFX_STATIC_ASSERT(sizeof (spec->efs_rem_host) ==
 		    MC_CMD_FILTER_OP_EXT_IN_SRC_IP_LEN);
 		EFX_STATIC_ASSERT(sizeof (spec->efs_loc_host) ==
 		    MC_CMD_FILTER_OP_EXT_IN_DST_IP_LEN);
 
 		memcpy(MCDI_IN2(req, uint8_t, FILTER_OP_EXT_IN_SRC_IP),
 		    &spec->efs_rem_host.eo_byte[0],
 		    MC_CMD_FILTER_OP_EXT_IN_SRC_IP_LEN);
 		memcpy(MCDI_IN2(req, uint8_t, FILTER_OP_EXT_IN_DST_IP),
 		    &spec->efs_loc_host.eo_byte[0],
 		    MC_CMD_FILTER_OP_EXT_IN_DST_IP_LEN);
+
+		/*
+		 * On Medford, filters for encapsulated packets match based on
+		 * the ether type and IP protocol in the outer frame.  In
+		 * addition we need to fill in the VNI or VSID type field.
+		 */
+		switch (spec->efs_encap_type) {
+		case EFX_TUNNEL_PROTOCOL_NONE:
+			break;
+		case EFX_TUNNEL_PROTOCOL_VXLAN:
+		case EFX_TUNNEL_PROTOCOL_GENEVE:
+			MCDI_IN_POPULATE_DWORD_1(req,
+			    FILTER_OP_EXT_IN_VNI_OR_VSID,
+			    FILTER_OP_EXT_IN_VNI_TYPE,
+			    spec->efs_encap_type == EFX_TUNNEL_PROTOCOL_VXLAN ?
+				    MC_CMD_FILTER_OP_EXT_IN_VNI_TYPE_VXLAN :
+				    MC_CMD_FILTER_OP_EXT_IN_VNI_TYPE_GENEVE);
+			break;
+		case EFX_TUNNEL_PROTOCOL_NVGRE:
+			MCDI_IN_POPULATE_DWORD_1(req,
+			    FILTER_OP_EXT_IN_VNI_OR_VSID,
+			    FILTER_OP_EXT_IN_VSID_TYPE,
+			    MC_CMD_FILTER_OP_EXT_IN_VSID_TYPE_NVGRE);
+			break;
+		default:
+			EFSYS_ASSERT(0);
+			rc = EINVAL;
+			goto fail2;
+		}
 	}
 
 	efx_mcdi_execute(enp, &req);
 
 	if (req.emr_rc != 0) {
 		rc = req.emr_rc;
-		goto fail2;
+		goto fail3;
 	}
 
 	if (req.emr_out_length_used < MC_CMD_FILTER_OP_EXT_OUT_LEN) {
 		rc = EMSGSIZE;
-		goto fail3;
+		goto fail4;
 	}
 
 	handle->efh_lo = MCDI_OUT_DWORD(req, FILTER_OP_EXT_OUT_HANDLE_LO);
 	handle->efh_hi = MCDI_OUT_DWORD(req, FILTER_OP_EXT_OUT_HANDLE_HI);
 
 	return (0);
 
+fail4:
+	EFSYS_PROBE(fail4);
 fail3:
 	EFSYS_PROBE(fail3);
 fail2:
 	EFSYS_PROBE(fail2);
 fail1:
 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
 
 	return (rc);
 
 }
 
 static	__checkReturn	efx_rc_t
 efx_mcdi_filter_op_delete(
 	__in		efx_nic_t *enp,
 	__in		unsigned int filter_op,
 	__inout		ef10_filter_handle_t *handle)
 {
 	efx_mcdi_req_t req;
 	uint8_t payload[MAX(MC_CMD_FILTER_OP_EXT_IN_LEN,
 			    MC_CMD_FILTER_OP_EXT_OUT_LEN)];
 	efx_rc_t rc;
 
 	memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_FILTER_OP;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_FILTER_OP_EXT_IN_LEN;
 	req.emr_out_buf = payload;
 	req.emr_out_length = MC_CMD_FILTER_OP_EXT_OUT_LEN;
 
 	switch (filter_op) {
 	case MC_CMD_FILTER_OP_IN_OP_REMOVE:
 		MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_OP,
 		    MC_CMD_FILTER_OP_IN_OP_REMOVE);
 		break;
 	case MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE:
 		MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_OP,
 		    MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE);
 		break;
 	default:
 		EFSYS_ASSERT(0);
 		rc = EINVAL;
 		goto fail1;
 	}
 
 	MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_HANDLE_LO, handle->efh_lo);
 	MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_HANDLE_HI, handle->efh_hi);
 
 	efx_mcdi_execute_quiet(enp, &req);
 
 	if (req.emr_rc != 0) {
 		rc = req.emr_rc;
 		goto fail2;
 	}
 
 	if (req.emr_out_length_used < MC_CMD_FILTER_OP_EXT_OUT_LEN) {
 		rc = EMSGSIZE;
 		goto fail3;
 	}
 
 	return (0);
 
 fail3:
 	EFSYS_PROBE(fail3);
 
 fail2:
 	EFSYS_PROBE(fail2);
 fail1:
 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
 
 	return (rc);
 }
 
 static	__checkReturn	boolean_t
 ef10_filter_equal(
 	__in		const efx_filter_spec_t *left,
 	__in		const efx_filter_spec_t *right)
 {
 	/* FIXME: Consider rx vs tx filters (look at efs_flags) */
 	if (left->efs_match_flags != right->efs_match_flags)
 		return (B_FALSE);
 	if (!EFX_OWORD_IS_EQUAL(left->efs_rem_host, right->efs_rem_host))
 		return (B_FALSE);
 	if (!EFX_OWORD_IS_EQUAL(left->efs_loc_host, right->efs_loc_host))
 		return (B_FALSE);
 	if (memcmp(left->efs_rem_mac, right->efs_rem_mac, EFX_MAC_ADDR_LEN))
 		return (B_FALSE);
 	if (memcmp(left->efs_loc_mac, right->efs_loc_mac, EFX_MAC_ADDR_LEN))
 		return (B_FALSE);
 	if (left->efs_rem_port != right->efs_rem_port)
 		return (B_FALSE);
 	if (left->efs_loc_port != right->efs_loc_port)
 		return (B_FALSE);
 	if (left->efs_inner_vid != right->efs_inner_vid)
 		return (B_FALSE);
 	if (left->efs_outer_vid != right->efs_outer_vid)
 		return (B_FALSE);
 	if (left->efs_ether_type != right->efs_ether_type)
 		return (B_FALSE);
 	if (left->efs_ip_proto != right->efs_ip_proto)
+		return (B_FALSE);
+	if (left->efs_encap_type != right->efs_encap_type)
 		return (B_FALSE);
 
 	return (B_TRUE);
 
 }
 
 static	__checkReturn	boolean_t
 ef10_filter_same_dest(
 	__in		const efx_filter_spec_t *left,
 	__in		const efx_filter_spec_t *right)
 {
 	if ((left->efs_flags & EFX_FILTER_FLAG_RX_RSS) &&
 	    (right->efs_flags & EFX_FILTER_FLAG_RX_RSS)) {
 		if (left->efs_rss_context == right->efs_rss_context)
 			return (B_TRUE);
 	} else if ((~(left->efs_flags) & EFX_FILTER_FLAG_RX_RSS) &&
 	    (~(right->efs_flags) & EFX_FILTER_FLAG_RX_RSS)) {
 		if (left->efs_dmaq_id == right->efs_dmaq_id)
 			return (B_TRUE);
 	}
 	return (B_FALSE);
 }
 
 static	__checkReturn	uint32_t
 ef10_filter_hash(
 	__in		efx_filter_spec_t *spec)
 {
 	EFX_STATIC_ASSERT((sizeof (efx_filter_spec_t) % sizeof (uint32_t))
 			    == 0);
 	EFX_STATIC_ASSERT((EFX_FIELD_OFFSET(efx_filter_spec_t, efs_outer_vid) %
 			    sizeof (uint32_t)) == 0);
 
 	/*
 	 * As the area of the efx_filter_spec_t we need to hash is DWORD
 	 * aligned and an exact number of DWORDs in size we can use the
 	 * optimised efx_hash_dwords() rather than efx_hash_bytes()
 	 */
 	return (efx_hash_dwords((const uint32_t *)&spec->efs_outer_vid,
 			(sizeof (efx_filter_spec_t) -
 			EFX_FIELD_OFFSET(efx_filter_spec_t, efs_outer_vid)) /
 			sizeof (uint32_t), 0));
 }
 
 /*
  * Decide whether a filter should be exclusive or else should allow
  * delivery to additional recipients.  Currently we decide that
  * filters for specific local unicast MAC and IP addresses are
  * exclusive.
  */
 static	__checkReturn	boolean_t
 ef10_filter_is_exclusive(
 	__in		efx_filter_spec_t *spec)
 {
 	if ((spec->efs_match_flags & EFX_FILTER_MATCH_LOC_MAC) &&
 	    !EFX_MAC_ADDR_IS_MULTICAST(spec->efs_loc_mac))
 		return (B_TRUE);
 
 	if ((spec->efs_match_flags &
 		(EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_LOC_HOST)) ==
 	    (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_LOC_HOST)) {
 		if ((spec->efs_ether_type == EFX_ETHER_TYPE_IPV4) &&
 		    ((spec->efs_loc_host.eo_u8[0] & 0xf) != 0xe))
 			return (B_TRUE);
 		if ((spec->efs_ether_type == EFX_ETHER_TYPE_IPV6) &&
 		    (spec->efs_loc_host.eo_u8[0] != 0xff))
 			return (B_TRUE);
 	}
 
 	return (B_FALSE);
 }
 
 	__checkReturn	efx_rc_t
 ef10_filter_restore(
 	__in		efx_nic_t *enp)
 {
 	int tbl_id;
 	efx_filter_spec_t *spec;
 	ef10_filter_table_t *eftp = enp->en_filter.ef_ef10_filter_table;
 	boolean_t restoring;
 	efsys_lock_state_t state;
 	efx_rc_t rc;
 
 	EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON ||
 		    enp->en_family == EFX_FAMILY_MEDFORD);
 
 	for (tbl_id = 0; tbl_id < EFX_EF10_FILTER_TBL_ROWS; tbl_id++) {
 
 		EFSYS_LOCK(enp->en_eslp, state);
 
 		spec = ef10_filter_entry_spec(eftp, tbl_id);
 		if (spec == NULL) {
 			restoring = B_FALSE;
 		} else if (ef10_filter_entry_is_busy(eftp, tbl_id)) {
 			/* Ignore busy entries. */
 			restoring = B_FALSE;
 		} else {
 			ef10_filter_set_entry_busy(eftp, tbl_id);
 			restoring = B_TRUE;
 		}
 
 		EFSYS_UNLOCK(enp->en_eslp, state);
 
 		if (restoring == B_FALSE)
 			continue;
 
 		if (ef10_filter_is_exclusive(spec)) {
 			rc = efx_mcdi_filter_op_add(enp, spec,
 			    MC_CMD_FILTER_OP_IN_OP_INSERT,
 			    &eftp->eft_entry[tbl_id].efe_handle);
 		} else {
 			rc = efx_mcdi_filter_op_add(enp, spec,
 			    MC_CMD_FILTER_OP_IN_OP_SUBSCRIBE,
 			    &eftp->eft_entry[tbl_id].efe_handle);
 		}
 
 		if (rc != 0)
 			goto fail1;
 
 		EFSYS_LOCK(enp->en_eslp, state);
 
 		ef10_filter_set_entry_not_busy(eftp, tbl_id);
 
 		EFSYS_UNLOCK(enp->en_eslp, state);
 	}
 
 	return (0);
 
 fail1:
 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
 
 	return (rc);
 }
 
 /*
  * An arbitrary search limit for the software hash table. As per the linux net
  * driver.
  */
 #define	EF10_FILTER_SEARCH_LIMIT 200
 
 static	__checkReturn	efx_rc_t
 ef10_filter_add_internal(
 	__in		efx_nic_t *enp,
 	__inout		efx_filter_spec_t *spec,
 	__in		boolean_t may_replace,
 	__out_opt	uint32_t *filter_id)
 {
 	efx_rc_t rc;
 	ef10_filter_table_t *eftp = enp->en_filter.ef_ef10_filter_table;
 	efx_filter_spec_t *saved_spec;
 	uint32_t hash;
 	unsigned int depth;
 	int ins_index;
 	boolean_t replacing = B_FALSE;
 	unsigned int i;
 	efsys_lock_state_t state;
 	boolean_t locked = B_FALSE;
 
 	EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON ||
 		    enp->en_family == EFX_FAMILY_MEDFORD);
 
 #if EFSYS_OPT_RX_SCALE
 	spec->efs_rss_context = enp->en_rss_context;
 #endif
 
 	hash = ef10_filter_hash(spec);
 
 	/*
 	 * FIXME: Add support for inserting filters of different priorities
 	 * and removing lower priority multicast filters (bug 42378)
 	 */
 
 	/*
 	 * Find any existing filters with the same match tuple or
 	 * else a free slot to insert at.  If any of them are busy,
 	 * we have to wait and retry.
 	 */
 	for (;;) {
 		ins_index = -1;
 		depth = 1;
 		EFSYS_LOCK(enp->en_eslp, state);
 		locked = B_TRUE;
 
 		for (;;) {
 			i = (hash + depth) & (EFX_EF10_FILTER_TBL_ROWS - 1);
 			saved_spec = ef10_filter_entry_spec(eftp, i);
 
 			if (!saved_spec) {
 				if (ins_index < 0) {
 					ins_index = i;
 				}
 			} else if (ef10_filter_equal(spec, saved_spec)) {
 				if (ef10_filter_entry_is_busy(eftp, i))
 					break;
 				if (saved_spec->efs_priority
 					    == EFX_FILTER_PRI_AUTO) {
 					ins_index = i;
 					goto found;
 				} else if (ef10_filter_is_exclusive(spec)) {
 					if (may_replace) {
 						ins_index = i;
 						goto found;
 					} else {
 						rc = EEXIST;
 						goto fail1;
 					}
 				}
 
 				/* Leave existing */
 			}
 
 			/*
 			 * Once we reach the maximum search depth, use
 			 * the first suitable slot or return EBUSY if
 			 * there was none.
 			 */
 			if (depth == EF10_FILTER_SEARCH_LIMIT) {
 				if (ins_index < 0) {
 					rc = EBUSY;
 					goto fail2;
 				}
 				goto found;
 			}
 			depth++;
 		}
 		EFSYS_UNLOCK(enp->en_eslp, state);
 		locked = B_FALSE;
 	}
 
 found:
 	/*
 	 * Create a software table entry if necessary, and mark it
 	 * busy.  We might yet fail to insert, but any attempt to
 	 * insert a conflicting filter while we're waiting for the
 	 * firmware must find the busy entry.
 	 */
 	saved_spec = ef10_filter_entry_spec(eftp, ins_index);
 	if (saved_spec) {
 		if (saved_spec->efs_priority == EFX_FILTER_PRI_AUTO) {
 			/* This is a filter we are refreshing */
 			ef10_filter_set_entry_not_auto_old(eftp, ins_index);
 			goto out_unlock;
 
 		}
 		replacing = B_TRUE;
 	} else {
 		EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (*spec), saved_spec);
 		if (!saved_spec) {
 			rc = ENOMEM;
 			goto fail3;
 		}
 		*saved_spec = *spec;
 		ef10_filter_set_entry(eftp, ins_index, saved_spec);
 	}
 	ef10_filter_set_entry_busy(eftp, ins_index);
 
 	EFSYS_UNLOCK(enp->en_eslp, state);
 	locked = B_FALSE;
 
 	/*
 	 * On replacing the filter handle may change after after a successful
 	 * replace operation.
 	 */
 	if (replacing) {
 		rc = efx_mcdi_filter_op_add(enp, spec,
 		    MC_CMD_FILTER_OP_IN_OP_REPLACE,
 		    &eftp->eft_entry[ins_index].efe_handle);
 	} else if (ef10_filter_is_exclusive(spec)) {
 		rc = efx_mcdi_filter_op_add(enp, spec,
 		    MC_CMD_FILTER_OP_IN_OP_INSERT,
 		    &eftp->eft_entry[ins_index].efe_handle);
 	} else {
 		rc = efx_mcdi_filter_op_add(enp, spec,
 		    MC_CMD_FILTER_OP_IN_OP_SUBSCRIBE,
 		    &eftp->eft_entry[ins_index].efe_handle);
 	}
 
 	if (rc != 0)
 		goto fail4;
 
 	EFSYS_LOCK(enp->en_eslp, state);
 	locked = B_TRUE;
 
 	if (replacing) {
 		/* Update the fields that may differ */
 		saved_spec->efs_priority = spec->efs_priority;
 		saved_spec->efs_flags = spec->efs_flags;
 		saved_spec->efs_rss_context = spec->efs_rss_context;
 		saved_spec->efs_dmaq_id = spec->efs_dmaq_id;
 	}
 
 	ef10_filter_set_entry_not_busy(eftp, ins_index);
 
 out_unlock:
 
 	EFSYS_UNLOCK(enp->en_eslp, state);
 	locked = B_FALSE;
 
 	if (filter_id)
 		*filter_id = ins_index;
 
 	return (0);
 
 fail4:
 	EFSYS_PROBE(fail4);
 
 	if (!replacing) {
 		EFSYS_KMEM_FREE(enp->en_esip, sizeof (*spec), saved_spec);
 		saved_spec = NULL;
 	}
 	ef10_filter_set_entry_not_busy(eftp, ins_index);
 	ef10_filter_set_entry(eftp, ins_index, NULL);
 
 fail3:
 	EFSYS_PROBE(fail3);
 
 fail2:
 	EFSYS_PROBE(fail2);
 
 fail1:
 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
 
 	if (locked)
 		EFSYS_UNLOCK(enp->en_eslp, state);
 
 	return (rc);
 }
 
 	__checkReturn	efx_rc_t
 ef10_filter_add(
 	__in		efx_nic_t *enp,
 	__inout		efx_filter_spec_t *spec,
 	__in		boolean_t may_replace)
 {
 	efx_rc_t rc;
 
 	rc = ef10_filter_add_internal(enp, spec, may_replace, NULL);
 	if (rc != 0)
 		goto fail1;
 
 	return (0);
 
 fail1:
 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
 
 	return (rc);
 }
 
 
 static	__checkReturn	efx_rc_t
 ef10_filter_delete_internal(
 	__in		efx_nic_t *enp,
 	__in		uint32_t filter_id)
 {
 	efx_rc_t rc;
 	ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table;
 	efx_filter_spec_t *spec;
 	efsys_lock_state_t state;
 	uint32_t filter_idx = filter_id % EFX_EF10_FILTER_TBL_ROWS;
 
 	/*
 	 * Find the software table entry and mark it busy.  Don't
 	 * remove it yet; any attempt to update while we're waiting
 	 * for the firmware must find the busy entry.
 	 *
 	 * FIXME: What if the busy flag is never cleared?
 	 */
 	EFSYS_LOCK(enp->en_eslp, state);
 	while (ef10_filter_entry_is_busy(table, filter_idx)) {
 		EFSYS_UNLOCK(enp->en_eslp, state);
 		EFSYS_SPIN(1);
 		EFSYS_LOCK(enp->en_eslp, state);
 	}
 	if ((spec = ef10_filter_entry_spec(table, filter_idx)) != NULL) {
 		ef10_filter_set_entry_busy(table, filter_idx);
 	}
 	EFSYS_UNLOCK(enp->en_eslp, state);
 
 	if (spec == NULL) {
 		rc = ENOENT;
 		goto fail1;
 	}
 
 	/*
 	 * Try to remove the hardware filter. This may fail if the MC has
 	 * rebooted (which frees all hardware filter resources).
 	 */
 	if (ef10_filter_is_exclusive(spec)) {
 		rc = efx_mcdi_filter_op_delete(enp,
 		    MC_CMD_FILTER_OP_IN_OP_REMOVE,
 		    &table->eft_entry[filter_idx].efe_handle);
 	} else {
 		rc = efx_mcdi_filter_op_delete(enp,
 		    MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE,
 		    &table->eft_entry[filter_idx].efe_handle);
 	}
 
 	/* Free the software table entry */
 	EFSYS_LOCK(enp->en_eslp, state);
 	ef10_filter_set_entry_not_busy(table, filter_idx);
 	ef10_filter_set_entry(table, filter_idx, NULL);
 	EFSYS_UNLOCK(enp->en_eslp, state);
 
 	EFSYS_KMEM_FREE(enp->en_esip, sizeof (*spec), spec);
 
 	/* Check result of hardware filter removal */
 	if (rc != 0)
 		goto fail2;
 
 	return (0);
 
 fail2:
 	EFSYS_PROBE(fail2);
 
 fail1:
 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
 
 	return (rc);
 }
 
 	__checkReturn	efx_rc_t
 ef10_filter_delete(
 	__in		efx_nic_t *enp,
 	__inout		efx_filter_spec_t *spec)
 {
 	efx_rc_t rc;
 	ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table;
 	efx_filter_spec_t *saved_spec;
 	unsigned int hash;
 	unsigned int depth;
 	unsigned int i;
 	efsys_lock_state_t state;
 	boolean_t locked = B_FALSE;
 
 	EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON ||
 		    enp->en_family == EFX_FAMILY_MEDFORD);
 
 	hash = ef10_filter_hash(spec);
 
 	EFSYS_LOCK(enp->en_eslp, state);
 	locked = B_TRUE;
 
 	depth = 1;
 	for (;;) {
 		i = (hash + depth) & (EFX_EF10_FILTER_TBL_ROWS - 1);
 		saved_spec = ef10_filter_entry_spec(table, i);
 		if (saved_spec && ef10_filter_equal(spec, saved_spec) &&
 		    ef10_filter_same_dest(spec, saved_spec)) {
 			break;
 		}
 		if (depth == EF10_FILTER_SEARCH_LIMIT) {
 			rc = ENOENT;
 			goto fail1;
 		}
 		depth++;
 	}
 
 	EFSYS_UNLOCK(enp->en_eslp, state);
 	locked = B_FALSE;
 
 	rc = ef10_filter_delete_internal(enp, i);
 	if (rc != 0)
 		goto fail2;
 
 	return (0);
 
 fail2:
 	EFSYS_PROBE(fail2);
 
 fail1:
 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
 
 	if (locked)
 		EFSYS_UNLOCK(enp->en_eslp, state);
 
 	return (rc);
 }
 
 static	__checkReturn	efx_rc_t
 efx_mcdi_get_parser_disp_info(
 	__in				efx_nic_t *enp,
 	__out_ecount(buffer_length)	uint32_t *buffer,
 	__in				size_t buffer_length,
 	__out				size_t *list_lengthp)
 {
 	efx_mcdi_req_t req;
 	uint8_t payload[MAX(MC_CMD_GET_PARSER_DISP_INFO_IN_LEN,
 			    MC_CMD_GET_PARSER_DISP_INFO_OUT_LENMAX)];
 	size_t matches_count;
 	size_t list_size;
 	efx_rc_t rc;
 
 	(void) memset(payload, 0, sizeof (payload));
 	req.emr_cmd = MC_CMD_GET_PARSER_DISP_INFO;
 	req.emr_in_buf = payload;
 	req.emr_in_length = MC_CMD_GET_PARSER_DISP_INFO_IN_LEN;
 	req.emr_out_buf = payload;
 	req.emr_out_length = MC_CMD_GET_PARSER_DISP_INFO_OUT_LENMAX;
 
 	MCDI_IN_SET_DWORD(req, GET_PARSER_DISP_INFO_OUT_OP,
 	    MC_CMD_GET_PARSER_DISP_INFO_IN_OP_GET_SUPPORTED_RX_MATCHES);
 
 	efx_mcdi_execute(enp, &req);
 
 	if (req.emr_rc != 0) {
 		rc = req.emr_rc;
 		goto fail1;
 	}
 
 	matches_count = MCDI_OUT_DWORD(req,
 	    GET_PARSER_DISP_INFO_OUT_NUM_SUPPORTED_MATCHES);
 
 	if (req.emr_out_length_used <
 	    MC_CMD_GET_PARSER_DISP_INFO_OUT_LEN(matches_count)) {
 		rc = EMSGSIZE;
 		goto fail2;
 	}
 
 	*list_lengthp = matches_count;
 
 	if (buffer_length < matches_count) {
 		rc = ENOSPC;
 		goto fail3;
 	}
 
 	/*
 	 * Check that the elements in the list in the MCDI response are the size
 	 * we expect, so we can just copy them directly. Any conversion of the
 	 * flags is handled by the caller.
 	 */
 	EFX_STATIC_ASSERT(sizeof (uint32_t) ==
 	    MC_CMD_GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES_LEN);
 
 	list_size = matches_count *
 		MC_CMD_GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES_LEN;
 	memcpy(buffer,
 	    MCDI_OUT2(req, uint32_t,
 		    GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES),
 	    list_size);
 
 	return (0);
 
 fail3:
 	EFSYS_PROBE(fail3);
 fail2:
 	EFSYS_PROBE(fail2);
 fail1:
 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
 
 	return (rc);
 }
 
 	__checkReturn	efx_rc_t
 ef10_filter_supported_filters(
 	__in				efx_nic_t *enp,
 	__out_ecount(buffer_length)	uint32_t *buffer,
 	__in				size_t buffer_length,
 	__out				size_t *list_lengthp)
 {
 
 	size_t mcdi_list_length;
 	size_t list_length;
 	uint32_t i;
 	efx_rc_t rc;
 	uint32_t all_filter_flags =
 	    (EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_LOC_HOST |
 	    EFX_FILTER_MATCH_REM_MAC | EFX_FILTER_MATCH_REM_PORT |
 	    EFX_FILTER_MATCH_LOC_MAC | EFX_FILTER_MATCH_LOC_PORT |
 	    EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_INNER_VID |
 	    EFX_FILTER_MATCH_OUTER_VID | EFX_FILTER_MATCH_IP_PROTO |
 	    EFX_FILTER_MATCH_UNKNOWN_MCAST_DST |
 	    EFX_FILTER_MATCH_UNKNOWN_UCAST_DST);
 
 	rc = efx_mcdi_get_parser_disp_info(enp, buffer, buffer_length,
 					    &mcdi_list_length);
 	if (rc != 0) {
 		if (rc == ENOSPC) {
 			/* Pass through mcdi_list_length for the list length */
 			*list_lengthp = mcdi_list_length;
 		}
 		goto fail1;
 	}
 
 	/*
 	 * The static assertions in ef10_filter_init() ensure that the values of
 	 * the EFX_FILTER_MATCH flags match those used by MCDI, so they don't
 	 * need to be converted.
 	 *
 	 * In case support is added to MCDI for additional flags, remove any
 	 * matches from the list which include flags we don't support. The order
 	 * of the matches is preserved as they are ordered from highest to
 	 * lowest priority.
 	 */
 	EFSYS_ASSERT(mcdi_list_length <= buffer_length);
 	list_length = 0;
 	for (i = 0; i < mcdi_list_length; i++) {
 		if ((buffer[i] & ~all_filter_flags) == 0) {
 			buffer[list_length] = buffer[i];
 			list_length++;
 		}
 	}
 
 	*list_lengthp = list_length;
 
 	return (0);
 
 fail1:
 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
 
 	return (rc);
 }
 
 static	__checkReturn	efx_rc_t
 ef10_filter_insert_unicast(
 	__in				efx_nic_t *enp,
 	__in_ecount(6)			uint8_t const *addr,
 	__in				efx_filter_flags_t filter_flags)
 {
 	ef10_filter_table_t *eftp = enp->en_filter.ef_ef10_filter_table;
 	efx_filter_spec_t spec;
 	efx_rc_t rc;
 
 	/* Insert the filter for the local station address */
 	efx_filter_spec_init_rx(&spec, EFX_FILTER_PRI_AUTO,
 	    filter_flags,
 	    eftp->eft_default_rxq);
 	efx_filter_spec_set_eth_local(&spec, EFX_FILTER_SPEC_VID_UNSPEC, addr);
 
 	rc = ef10_filter_add_internal(enp, &spec, B_TRUE,
 	    &eftp->eft_unicst_filter_indexes[eftp->eft_unicst_filter_count]);
 	if (rc != 0)
 		goto fail1;
 
 	eftp->eft_unicst_filter_count++;
 	EFSYS_ASSERT(eftp->eft_unicst_filter_count <=
 		    EFX_EF10_FILTER_UNICAST_FILTERS_MAX);
 
 	return (0);
 
 fail1:
 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
 	return (rc);
 }
 
 static	__checkReturn	efx_rc_t
 ef10_filter_insert_all_unicast(
 	__in				efx_nic_t *enp,
 	__in				efx_filter_flags_t filter_flags)
 {
 	ef10_filter_table_t *eftp = enp->en_filter.ef_ef10_filter_table;
 	efx_filter_spec_t spec;
 	efx_rc_t rc;
 
 	/* Insert the unknown unicast filter */
 	efx_filter_spec_init_rx(&spec, EFX_FILTER_PRI_AUTO,
 	    filter_flags,
 	    eftp->eft_default_rxq);
 	efx_filter_spec_set_uc_def(&spec);
 	rc = ef10_filter_add_internal(enp, &spec, B_TRUE,
 	    &eftp->eft_unicst_filter_indexes[eftp->eft_unicst_filter_count]);
 	if (rc != 0)
 		goto fail1;
 
 	eftp->eft_unicst_filter_count++;
 	EFSYS_ASSERT(eftp->eft_unicst_filter_count <=
 		    EFX_EF10_FILTER_UNICAST_FILTERS_MAX);
 
 	return (0);
 
 fail1:
 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
 	return (rc);
 }
 
 static	__checkReturn	efx_rc_t
 ef10_filter_insert_multicast_list(
 	__in				efx_nic_t *enp,
 	__in				boolean_t mulcst,
 	__in				boolean_t brdcst,
 	__in_ecount(6*count)		uint8_t const *addrs,
 	__in				uint32_t count,
 	__in				efx_filter_flags_t filter_flags,
 	__in				boolean_t rollback)
 {
 	ef10_filter_table_t *eftp = enp->en_filter.ef_ef10_filter_table;
 	efx_filter_spec_t spec;
 	uint8_t addr[6];
 	uint32_t i;
 	uint32_t filter_index;
 	uint32_t filter_count;
 	efx_rc_t rc;
 
 	if (mulcst == B_FALSE)
 		count = 0;
 
 	if (count + (brdcst ? 1 : 0) >
 	    EFX_ARRAY_SIZE(eftp->eft_mulcst_filter_indexes)) {
 		/* Too many MAC addresses */
 		rc = EINVAL;
 		goto fail1;
 	}
 
 	/* Insert/renew multicast address list filters */
 	filter_count = 0;
 	for (i = 0; i < count; i++) {
 		efx_filter_spec_init_rx(&spec,
 		    EFX_FILTER_PRI_AUTO,
 		    filter_flags,
 		    eftp->eft_default_rxq);
 
 		efx_filter_spec_set_eth_local(&spec,
 		    EFX_FILTER_SPEC_VID_UNSPEC,
 		    &addrs[i * EFX_MAC_ADDR_LEN]);
 
 		rc = ef10_filter_add_internal(enp, &spec, B_TRUE,
 					    &filter_index);
 
 		if (rc == 0) {
 			eftp->eft_mulcst_filter_indexes[filter_count] =
 				filter_index;
 			filter_count++;
 		} else if (rollback == B_TRUE) {
 			/* Only stop upon failure if told to rollback */
 			goto rollback;
 		}
 
 	}
 
 	if (brdcst == B_TRUE) {
 		/* Insert/renew broadcast address filter */
 		efx_filter_spec_init_rx(&spec, EFX_FILTER_PRI_AUTO,
 		    filter_flags,
 		    eftp->eft_default_rxq);
 
 		EFX_MAC_BROADCAST_ADDR_SET(addr);
 		efx_filter_spec_set_eth_local(&spec, EFX_FILTER_SPEC_VID_UNSPEC,
 		    addr);
 
 		rc = ef10_filter_add_internal(enp, &spec, B_TRUE,
 					    &filter_index);
 
 		if (rc == 0) {
 			eftp->eft_mulcst_filter_indexes[filter_count] =
 				filter_index;
 			filter_count++;
 		} else if (rollback == B_TRUE) {
 			/* Only stop upon failure if told to rollback */
 			goto rollback;
 		}
 	}
 
 	eftp->eft_mulcst_filter_count = filter_count;
 	eftp->eft_using_all_mulcst = B_FALSE;
 
 	return (0);
 
 rollback:
 	/* Remove any filters we have inserted */
 	i = filter_count;
 	while (i--) {
 		(void) ef10_filter_delete_internal(enp,
 		    eftp->eft_mulcst_filter_indexes[i]);
 	}
 	eftp->eft_mulcst_filter_count = 0;
 
 fail1:
 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
 
 	return (rc);
 }
 
 static	__checkReturn	efx_rc_t
 ef10_filter_insert_all_multicast(
 	__in				efx_nic_t *enp,
 	__in				efx_filter_flags_t filter_flags)
 {
 	ef10_filter_table_t *eftp = enp->en_filter.ef_ef10_filter_table;
 	efx_filter_spec_t spec;
 	efx_rc_t rc;
 
 	/* Insert the unknown multicast filter */
 	efx_filter_spec_init_rx(&spec, EFX_FILTER_PRI_AUTO,
 	    filter_flags,
 	    eftp->eft_default_rxq);
 	efx_filter_spec_set_mc_def(&spec);
 
 	rc = ef10_filter_add_internal(enp, &spec, B_TRUE,
 	    &eftp->eft_mulcst_filter_indexes[0]);
 	if (rc != 0)
 		goto fail1;
 
 	eftp->eft_mulcst_filter_count = 1;
 	eftp->eft_using_all_mulcst = B_TRUE;
 
 	/*
 	 * FIXME: If brdcst == B_FALSE, add a filter to drop broadcast traffic.
 	 */
 
 	return (0);
 
 fail1:
 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
 
 	return (rc);
 }
 
 static			void
 ef10_filter_remove_old(
 	__in		efx_nic_t *enp)
 {
 	ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table;
 	uint32_t i;
 
 	for (i = 0; i < EFX_ARRAY_SIZE(table->eft_entry); i++) {
 		if (ef10_filter_entry_is_auto_old(table, i)) {
 			(void) ef10_filter_delete_internal(enp, i);
 		}
 	}
 }
 
 
 static	__checkReturn	efx_rc_t
 ef10_filter_get_workarounds(
 	__in				efx_nic_t *enp)
 {
 	efx_nic_cfg_t *encp = &enp->en_nic_cfg;
 	uint32_t implemented = 0;
 	uint32_t enabled = 0;
 	efx_rc_t rc;
 
 	rc = efx_mcdi_get_workarounds(enp, &implemented, &enabled);
 	if (rc == 0) {
 		/* Check if chained multicast filter support is enabled */
 		if (implemented & enabled & MC_CMD_GET_WORKAROUNDS_OUT_BUG26807)
 			encp->enc_bug26807_workaround = B_TRUE;
 		else
 			encp->enc_bug26807_workaround = B_FALSE;
 	} else if (rc == ENOTSUP) {
 		/*
 		 * Firmware is too old to support GET_WORKAROUNDS, and support
 		 * for this workaround was implemented later.
 		 */
 		encp->enc_bug26807_workaround = B_FALSE;
 	} else {
 		goto fail1;
 	}
 
 	return (0);
 
 fail1:
 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
 
 	return (rc);
 
 }
 
 
 /*
  * Reconfigure all filters.
  * If all_unicst and/or all mulcst filters cannot be applied then
  * return ENOTSUP (Note the filters for the specified addresses are
  * still applied in this case).
  */
 	__checkReturn	efx_rc_t
 ef10_filter_reconfigure(
 	__in				efx_nic_t *enp,
 	__in_ecount(6)			uint8_t const *mac_addr,
 	__in				boolean_t all_unicst,
 	__in				boolean_t mulcst,
 	__in				boolean_t all_mulcst,
 	__in				boolean_t brdcst,
 	__in_ecount(6*count)		uint8_t const *addrs,
 	__in				uint32_t count)
 {
 	efx_nic_cfg_t *encp = &enp->en_nic_cfg;
 	ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table;
 	efx_filter_flags_t filter_flags;
 	unsigned int i;
 	efx_rc_t all_unicst_rc = 0;
 	efx_rc_t all_mulcst_rc = 0;
 	efx_rc_t rc;
 
 	if (table->eft_default_rxq == NULL) {
 		/*
 		 * Filters direct traffic to the default RXQ, and so cannot be
 		 * inserted until it is available. Any currently configured
 		 * filters must be removed (ignore errors in case the MC
 		 * has rebooted, which removes hardware filters).
 		 */
 		for (i = 0; i < table->eft_unicst_filter_count; i++) {
 			(void) ef10_filter_delete_internal(enp,
 					table->eft_unicst_filter_indexes[i]);
 		}
 		table->eft_unicst_filter_count = 0;
 
 		for (i = 0; i < table->eft_mulcst_filter_count; i++) {
 			(void) ef10_filter_delete_internal(enp,
 					table->eft_mulcst_filter_indexes[i]);
 		}
 		table->eft_mulcst_filter_count = 0;
 
 		return (0);
 	}
 
 	if (table->eft_using_rss)
 		filter_flags = EFX_FILTER_FLAG_RX_RSS;
 	else
 		filter_flags = 0;
 
 	/* Mark old filters which may need to be removed */
 	for (i = 0; i < table->eft_unicst_filter_count; i++) {
 		ef10_filter_set_entry_auto_old(table,
 					table->eft_unicst_filter_indexes[i]);
 	}
 	for (i = 0; i < table->eft_mulcst_filter_count; i++) {
 		ef10_filter_set_entry_auto_old(table,
 					table->eft_mulcst_filter_indexes[i]);
 	}
 
 	/*
 	 * Insert or renew unicast filters.
 	 *
 	 * Frimware does not perform chaining on unicast filters. As traffic is
 	 * therefore only delivered to the first matching filter, we should
 	 * always insert the specific filter for our MAC address, to try and
 	 * ensure we get that traffic.
 	 *
 	 * (If the filter for our MAC address has already been inserted by
 	 * another function, we won't receive traffic sent to us, even if we
 	 * insert a unicast mismatch filter. To prevent traffic stealing, this
 	 * therefore relies on the privilege model only allowing functions to
 	 * insert filters for their own MAC address unless explicitly given
 	 * additional privileges by the user. This also means that, even on a
 	 * priviliged function, inserting a unicast mismatch filter may not
 	 * catch all traffic in multi PCI function scenarios.)
 	 */
 	table->eft_unicst_filter_count = 0;
 	rc = ef10_filter_insert_unicast(enp, mac_addr, filter_flags);
 	if (all_unicst || (rc != 0)) {
 		all_unicst_rc = ef10_filter_insert_all_unicast(enp,
 						    filter_flags);
 		if ((rc != 0) && (all_unicst_rc != 0))
 			goto fail1;
 	}
 
 	/*
 	 * WORKAROUND_BUG26807 controls firmware support for chained multicast
 	 * filters, and can only be enabled or disabled when the hardware filter
 	 * table is empty.
 	 *
 	 * Chained multicast filters require support from the datapath firmware,
 	 * and may not be available (e.g. low-latency variants or old Huntington
 	 * firmware).
 	 *
 	 * Firmware will reset (FLR) functions which have inserted filters in
 	 * the hardware filter table when the workaround is enabled/disabled.
 	 * Functions without any hardware filters are not reset.
 	 *
 	 * Re-check if the workaround is enabled after adding unicast hardware
 	 * filters. This ensures that encp->enc_bug26807_workaround matches the
 	 * firmware state, and that later changes to enable/disable the
 	 * workaround will result in this function seeing a reset (FLR).
 	 *
 	 * In common-code drivers, we only support multiple PCI function
 	 * scenarios with firmware that supports multicast chaining, so we can
 	 * assume it is enabled for such cases and hence simplify the filter
 	 * insertion logic. Firmware that does not support multicast chaining
 	 * does not support multiple PCI function configurations either, so
 	 * filter insertion is much simpler and the same strategies can still be
 	 * used.
 	 */
 	if ((rc = ef10_filter_get_workarounds(enp)) != 0)
 		goto fail2;
 
 	if ((table->eft_using_all_mulcst != all_mulcst) &&
 	    (encp->enc_bug26807_workaround == B_TRUE)) {
 		/*
 		 * Multicast filter chaining is enabled, so traffic that matches
 		 * more than one multicast filter will be replicated and
 		 * delivered to multiple recipients.  To avoid this duplicate
 		 * delivery, remove old multicast filters before inserting new
 		 * multicast filters.
 		 */
 		ef10_filter_remove_old(enp);
 	}
 
 	/* Insert or renew multicast filters */
 	if (all_mulcst == B_TRUE) {
 		/*
 		 * Insert the all multicast filter. If that fails, try to insert
 		 * all of our multicast filters (but without rollback on
 		 * failure).
 		 */
 		all_mulcst_rc = ef10_filter_insert_all_multicast(enp,
 							    filter_flags);
 		if (all_mulcst_rc != 0) {
 			rc = ef10_filter_insert_multicast_list(enp, B_TRUE,
 			    brdcst, addrs, count, filter_flags, B_FALSE);
 			if (rc != 0)
 				goto fail3;
 		}
 	} else {
 		/*
 		 * Insert filters for multicast addresses.
 		 * If any insertion fails, then rollback and try to insert the
 		 * all multicast filter instead.
 		 * If that also fails, try to insert all of the multicast
 		 * filters (but without rollback on failure).
 		 */
 		rc = ef10_filter_insert_multicast_list(enp, mulcst, brdcst,
 			    addrs, count, filter_flags, B_TRUE);
 		if (rc != 0) {
 			if ((table->eft_using_all_mulcst == B_FALSE) &&
 			    (encp->enc_bug26807_workaround == B_TRUE)) {
 				/*
 				 * Multicast filter chaining is on, so remove
 				 * old filters before inserting the multicast
 				 * all filter to avoid duplicate delivery caused
 				 * by packets matching multiple filters.
 				 */
 				ef10_filter_remove_old(enp);
 			}
 
 			rc = ef10_filter_insert_all_multicast(enp,
 							    filter_flags);
 			if (rc != 0) {
 				rc = ef10_filter_insert_multicast_list(enp,
 				    mulcst, brdcst,
 				    addrs, count, filter_flags, B_FALSE);
 				if (rc != 0)
 					goto fail4;
 			}
 		}
 	}
 
 	/* Remove old filters which were not renewed */
 	ef10_filter_remove_old(enp);
 
 	/* report if any optional flags were rejected */
 	if (((all_unicst != B_FALSE) && (all_unicst_rc != 0)) ||
 	    ((all_mulcst != B_FALSE) && (all_mulcst_rc != 0))) {
 		rc = ENOTSUP;
 	}
 
 	return (rc);
 
 fail4:
 	EFSYS_PROBE(fail4);
 fail3:
 	EFSYS_PROBE(fail3);
 fail2:
 	EFSYS_PROBE(fail2);
 fail1:
 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
 
 	/* Clear auto old flags */
 	for (i = 0; i < EFX_ARRAY_SIZE(table->eft_entry); i++) {
 		if (ef10_filter_entry_is_auto_old(table, i)) {
 			ef10_filter_set_entry_not_auto_old(table, i);
 		}
 	}
 
 	return (rc);
 }
 
 		void
 ef10_filter_get_default_rxq(
 	__in		efx_nic_t *enp,
 	__out		efx_rxq_t **erpp,
 	__out		boolean_t *using_rss)
 {
 	ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table;
 
 	*erpp = table->eft_default_rxq;
 	*using_rss = table->eft_using_rss;
 }
 
 
 		void
 ef10_filter_default_rxq_set(
 	__in		efx_nic_t *enp,
 	__in		efx_rxq_t *erp,
 	__in		boolean_t using_rss)
 {
 	ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table;
 
 #if EFSYS_OPT_RX_SCALE
 	EFSYS_ASSERT((using_rss == B_FALSE) ||
 	    (enp->en_rss_context != EF10_RSS_CONTEXT_INVALID));
 	table->eft_using_rss = using_rss;
 #else
 	EFSYS_ASSERT(using_rss == B_FALSE);
 	table->eft_using_rss = B_FALSE;
 #endif
 	table->eft_default_rxq = erp;
 }
 
 		void
 ef10_filter_default_rxq_clear(
 	__in		efx_nic_t *enp)
 {
 	ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table;
 
 	table->eft_default_rxq = NULL;
 	table->eft_using_rss = B_FALSE;
 }
 
 
 #endif /* EFSYS_OPT_FILTER */
 
 #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */
Index: stable/12/sys/dev/sfxge/common/efx.h
===================================================================
--- stable/12/sys/dev/sfxge/common/efx.h	(revision 342320)
+++ stable/12/sys/dev/sfxge/common/efx.h	(revision 342321)
@@ -1,2462 +1,2490 @@
 /*-
  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
  *
  * Copyright (c) 2006-2016 Solarflare Communications Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
  *
  * 1. Redistributions of source code must retain the above copyright notice,
  *    this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright notice,
  *    this list of conditions and the following disclaimer in the documentation
  *    and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  * The views and conclusions contained in the software and documentation are
  * those of the authors and should not be interpreted as representing official
  * policies, either expressed or implied, of the FreeBSD Project.
  *
  * $FreeBSD$
  */
 
 #ifndef	_SYS_EFX_H
 #define	_SYS_EFX_H
 
 #include "efsys.h"
 #include "efx_check.h"
 #include "efx_phy_ids.h"
 
 #ifdef	__cplusplus
 extern "C" {
 #endif
 
 #define	EFX_STATIC_ASSERT(_cond)		\
 	((void)sizeof(char[(_cond) ? 1 : -1]))
 
 #define	EFX_ARRAY_SIZE(_array)			\
 	(sizeof(_array) / sizeof((_array)[0]))
 
 #define	EFX_FIELD_OFFSET(_type, _field)		\
 	((size_t) &(((_type *)0)->_field))
 
 /* Return codes */
 
 typedef __success(return == 0) int efx_rc_t;
 
 
 /* Chip families */
 
 typedef enum efx_family_e {
 	EFX_FAMILY_INVALID,
 	EFX_FAMILY_FALCON,	/* Obsolete and not supported */
 	EFX_FAMILY_SIENA,
 	EFX_FAMILY_HUNTINGTON,
 	EFX_FAMILY_MEDFORD,
 	EFX_FAMILY_NTYPES
 } efx_family_t;
 
 extern	__checkReturn	efx_rc_t
 efx_family(
 	__in		uint16_t venid,
 	__in		uint16_t devid,
 	__out		efx_family_t *efp);
 
 
 #define	EFX_PCI_VENID_SFC			0x1924
 
 #define	EFX_PCI_DEVID_FALCON			0x0710	/* SFC4000 */
 
 #define	EFX_PCI_DEVID_BETHPAGE			0x0803	/* SFC9020 */
 #define	EFX_PCI_DEVID_SIENA			0x0813	/* SFL9021 */
 #define	EFX_PCI_DEVID_SIENA_F1_UNINIT		0x0810
 
 #define	EFX_PCI_DEVID_HUNTINGTON_PF_UNINIT	0x0901
 #define	EFX_PCI_DEVID_FARMINGDALE		0x0903	/* SFC9120 PF */
 #define	EFX_PCI_DEVID_GREENPORT			0x0923	/* SFC9140 PF */
 
 #define	EFX_PCI_DEVID_FARMINGDALE_VF		0x1903	/* SFC9120 VF */
 #define	EFX_PCI_DEVID_GREENPORT_VF		0x1923	/* SFC9140 VF */
 
 #define	EFX_PCI_DEVID_MEDFORD_PF_UNINIT		0x0913
 #define	EFX_PCI_DEVID_MEDFORD			0x0A03	/* SFC9240 PF */
 #define	EFX_PCI_DEVID_MEDFORD_VF		0x1A03	/* SFC9240 VF */
 
 #define	EFX_MEM_BAR	2
 
 /* Error codes */
 
 enum {
 	EFX_ERR_INVALID,
 	EFX_ERR_SRAM_OOB,
 	EFX_ERR_BUFID_DC_OOB,
 	EFX_ERR_MEM_PERR,
 	EFX_ERR_RBUF_OWN,
 	EFX_ERR_TBUF_OWN,
 	EFX_ERR_RDESQ_OWN,
 	EFX_ERR_TDESQ_OWN,
 	EFX_ERR_EVQ_OWN,
 	EFX_ERR_EVFF_OFLO,
 	EFX_ERR_ILL_ADDR,
 	EFX_ERR_SRAM_PERR,
 	EFX_ERR_NCODES
 };
 
 /* Calculate the IEEE 802.3 CRC32 of a MAC addr */
 extern	__checkReturn		uint32_t
 efx_crc32_calculate(
 	__in			uint32_t crc_init,
 	__in_ecount(length)	uint8_t const *input,
 	__in			int length);
 
 
 /* Type prototypes */
 
 typedef struct efx_rxq_s	efx_rxq_t;
 
 /* NIC */
 
 typedef struct efx_nic_s	efx_nic_t;
 
 extern	__checkReturn	efx_rc_t
 efx_nic_create(
 	__in		efx_family_t family,
 	__in		efsys_identifier_t *esip,
 	__in		efsys_bar_t *esbp,
 	__in		efsys_lock_t *eslp,
 	__deref_out	efx_nic_t **enpp);
 
 extern	__checkReturn	efx_rc_t
 efx_nic_probe(
 	__in		efx_nic_t *enp);
 
 extern	__checkReturn	efx_rc_t
 efx_nic_init(
 	__in		efx_nic_t *enp);
 
 extern	__checkReturn	efx_rc_t
 efx_nic_reset(
 	__in		efx_nic_t *enp);
 
 #if EFSYS_OPT_DIAG
 
 extern	__checkReturn	efx_rc_t
 efx_nic_register_test(
 	__in		efx_nic_t *enp);
 
 #endif	/* EFSYS_OPT_DIAG */
 
 extern		void
 efx_nic_fini(
 	__in		efx_nic_t *enp);
 
 extern		void
 efx_nic_unprobe(
 	__in		efx_nic_t *enp);
 
 extern		void
 efx_nic_destroy(
 	__in	efx_nic_t *enp);
 
 #define	EFX_PCIE_LINK_SPEED_GEN1		1
 #define	EFX_PCIE_LINK_SPEED_GEN2		2
 #define	EFX_PCIE_LINK_SPEED_GEN3		3
 
 typedef enum efx_pcie_link_performance_e {
 	EFX_PCIE_LINK_PERFORMANCE_UNKNOWN_BANDWIDTH,
 	EFX_PCIE_LINK_PERFORMANCE_SUBOPTIMAL_BANDWIDTH,
 	EFX_PCIE_LINK_PERFORMANCE_SUBOPTIMAL_LATENCY,
 	EFX_PCIE_LINK_PERFORMANCE_OPTIMAL
 } efx_pcie_link_performance_t;
 
 extern	__checkReturn	efx_rc_t
 efx_nic_calculate_pcie_link_bandwidth(
 	__in		uint32_t pcie_link_width,
 	__in		uint32_t pcie_link_gen,
 	__out		uint32_t *bandwidth_mbpsp);
 
 extern	__checkReturn	efx_rc_t
 efx_nic_check_pcie_link_speed(
 	__in		efx_nic_t *enp,
 	__in		uint32_t pcie_link_width,
 	__in		uint32_t pcie_link_gen,
 	__out		efx_pcie_link_performance_t *resultp);
 
 #if EFSYS_OPT_MCDI
 
 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD
 /* Huntington and Medford require MCDIv2 commands */
 #define	WITH_MCDI_V2 1
 #endif
 
 typedef struct efx_mcdi_req_s efx_mcdi_req_t;
 
 typedef enum efx_mcdi_exception_e {
 	EFX_MCDI_EXCEPTION_MC_REBOOT,
 	EFX_MCDI_EXCEPTION_MC_BADASSERT,
 } efx_mcdi_exception_t;
 
 #if EFSYS_OPT_MCDI_LOGGING
 typedef enum efx_log_msg_e {
 	EFX_LOG_INVALID,
 	EFX_LOG_MCDI_REQUEST,
 	EFX_LOG_MCDI_RESPONSE,
 } efx_log_msg_t;
 #endif /* EFSYS_OPT_MCDI_LOGGING */
 
 typedef struct efx_mcdi_transport_s {
 	void		*emt_context;
 	efsys_mem_t	*emt_dma_mem;
 	void		(*emt_execute)(void *, efx_mcdi_req_t *);
 	void		(*emt_ev_cpl)(void *);
 	void		(*emt_exception)(void *, efx_mcdi_exception_t);
 #if EFSYS_OPT_MCDI_LOGGING
 	void		(*emt_logger)(void *, efx_log_msg_t,
 					void *, size_t, void *, size_t);
 #endif /* EFSYS_OPT_MCDI_LOGGING */
 #if EFSYS_OPT_MCDI_PROXY_AUTH
 	void		(*emt_ev_proxy_response)(void *, uint32_t, efx_rc_t);
 #endif /* EFSYS_OPT_MCDI_PROXY_AUTH */
 } efx_mcdi_transport_t;
 
 extern	__checkReturn	efx_rc_t
 efx_mcdi_init(
 	__in		efx_nic_t *enp,
 	__in		const efx_mcdi_transport_t *mtp);
 
 extern	__checkReturn	efx_rc_t
 efx_mcdi_reboot(
 	__in		efx_nic_t *enp);
 
 			void
 efx_mcdi_new_epoch(
 	__in		efx_nic_t *enp);
 
 extern			void
 efx_mcdi_get_timeout(
 	__in		efx_nic_t *enp,
 	__in		efx_mcdi_req_t *emrp,
 	__out		uint32_t *usec_timeoutp);
 
 extern			void
 efx_mcdi_request_start(
 	__in		efx_nic_t *enp,
 	__in		efx_mcdi_req_t *emrp,
 	__in		boolean_t ev_cpl);
 
 extern	__checkReturn	boolean_t
 efx_mcdi_request_poll(
 	__in		efx_nic_t *enp);
 
 extern	__checkReturn	boolean_t
 efx_mcdi_request_abort(
 	__in		efx_nic_t *enp);
 
 extern			void
 efx_mcdi_fini(
 	__in		efx_nic_t *enp);
 
 #endif	/* EFSYS_OPT_MCDI */
 
 /* INTR */
 
 #define	EFX_NINTR_SIENA 1024
 
 typedef enum efx_intr_type_e {
 	EFX_INTR_INVALID = 0,
 	EFX_INTR_LINE,
 	EFX_INTR_MESSAGE,
 	EFX_INTR_NTYPES
 } efx_intr_type_t;
 
 #define	EFX_INTR_SIZE	(sizeof (efx_oword_t))
 
 extern	__checkReturn	efx_rc_t
 efx_intr_init(
 	__in		efx_nic_t *enp,
 	__in		efx_intr_type_t type,
 	__in		efsys_mem_t *esmp);
 
 extern			void
 efx_intr_enable(
 	__in		efx_nic_t *enp);
 
 extern			void
 efx_intr_disable(
 	__in		efx_nic_t *enp);
 
 extern			void
 efx_intr_disable_unlocked(
 	__in		efx_nic_t *enp);
 
 #define	EFX_INTR_NEVQS	32
 
 extern	__checkReturn	efx_rc_t
 efx_intr_trigger(
 	__in		efx_nic_t *enp,
 	__in		unsigned int level);
 
 extern			void
 efx_intr_status_line(
 	__in		efx_nic_t *enp,
 	__out		boolean_t *fatalp,
 	__out		uint32_t *maskp);
 
 extern			void
 efx_intr_status_message(
 	__in		efx_nic_t *enp,
 	__in		unsigned int message,
 	__out		boolean_t *fatalp);
 
 extern			void
 efx_intr_fatal(
 	__in		efx_nic_t *enp);
 
 extern			void
 efx_intr_fini(
 	__in		efx_nic_t *enp);
 
 /* MAC */
 
 #if EFSYS_OPT_MAC_STATS
 
 /* START MKCONFIG GENERATED EfxHeaderMacBlock e323546097fd7c65 */
 typedef enum efx_mac_stat_e {
 	EFX_MAC_RX_OCTETS,
 	EFX_MAC_RX_PKTS,
 	EFX_MAC_RX_UNICST_PKTS,
 	EFX_MAC_RX_MULTICST_PKTS,
 	EFX_MAC_RX_BRDCST_PKTS,
 	EFX_MAC_RX_PAUSE_PKTS,
 	EFX_MAC_RX_LE_64_PKTS,
 	EFX_MAC_RX_65_TO_127_PKTS,
 	EFX_MAC_RX_128_TO_255_PKTS,
 	EFX_MAC_RX_256_TO_511_PKTS,
 	EFX_MAC_RX_512_TO_1023_PKTS,
 	EFX_MAC_RX_1024_TO_15XX_PKTS,
 	EFX_MAC_RX_GE_15XX_PKTS,
 	EFX_MAC_RX_ERRORS,
 	EFX_MAC_RX_FCS_ERRORS,
 	EFX_MAC_RX_DROP_EVENTS,
 	EFX_MAC_RX_FALSE_CARRIER_ERRORS,
 	EFX_MAC_RX_SYMBOL_ERRORS,
 	EFX_MAC_RX_ALIGN_ERRORS,
 	EFX_MAC_RX_INTERNAL_ERRORS,
 	EFX_MAC_RX_JABBER_PKTS,
 	EFX_MAC_RX_LANE0_CHAR_ERR,
 	EFX_MAC_RX_LANE1_CHAR_ERR,
 	EFX_MAC_RX_LANE2_CHAR_ERR,
 	EFX_MAC_RX_LANE3_CHAR_ERR,
 	EFX_MAC_RX_LANE0_DISP_ERR,
 	EFX_MAC_RX_LANE1_DISP_ERR,
 	EFX_MAC_RX_LANE2_DISP_ERR,
 	EFX_MAC_RX_LANE3_DISP_ERR,
 	EFX_MAC_RX_MATCH_FAULT,
 	EFX_MAC_RX_NODESC_DROP_CNT,
 	EFX_MAC_TX_OCTETS,
 	EFX_MAC_TX_PKTS,
 	EFX_MAC_TX_UNICST_PKTS,
 	EFX_MAC_TX_MULTICST_PKTS,
 	EFX_MAC_TX_BRDCST_PKTS,
 	EFX_MAC_TX_PAUSE_PKTS,
 	EFX_MAC_TX_LE_64_PKTS,
 	EFX_MAC_TX_65_TO_127_PKTS,
 	EFX_MAC_TX_128_TO_255_PKTS,
 	EFX_MAC_TX_256_TO_511_PKTS,
 	EFX_MAC_TX_512_TO_1023_PKTS,
 	EFX_MAC_TX_1024_TO_15XX_PKTS,
 	EFX_MAC_TX_GE_15XX_PKTS,
 	EFX_MAC_TX_ERRORS,
 	EFX_MAC_TX_SGL_COL_PKTS,
 	EFX_MAC_TX_MULT_COL_PKTS,
 	EFX_MAC_TX_EX_COL_PKTS,
 	EFX_MAC_TX_LATE_COL_PKTS,
 	EFX_MAC_TX_DEF_PKTS,
 	EFX_MAC_TX_EX_DEF_PKTS,
 	EFX_MAC_PM_TRUNC_BB_OVERFLOW,
 	EFX_MAC_PM_DISCARD_BB_OVERFLOW,
 	EFX_MAC_PM_TRUNC_VFIFO_FULL,
 	EFX_MAC_PM_DISCARD_VFIFO_FULL,
 	EFX_MAC_PM_TRUNC_QBB,
 	EFX_MAC_PM_DISCARD_QBB,
 	EFX_MAC_PM_DISCARD_MAPPING,
 	EFX_MAC_RXDP_Q_DISABLED_PKTS,
 	EFX_MAC_RXDP_DI_DROPPED_PKTS,
 	EFX_MAC_RXDP_STREAMING_PKTS,
 	EFX_MAC_RXDP_HLB_FETCH,
 	EFX_MAC_RXDP_HLB_WAIT,
 	EFX_MAC_VADAPTER_RX_UNICAST_PACKETS,
 	EFX_MAC_VADAPTER_RX_UNICAST_BYTES,
 	EFX_MAC_VADAPTER_RX_MULTICAST_PACKETS,
 	EFX_MAC_VADAPTER_RX_MULTICAST_BYTES,
 	EFX_MAC_VADAPTER_RX_BROADCAST_PACKETS,
 	EFX_MAC_VADAPTER_RX_BROADCAST_BYTES,
 	EFX_MAC_VADAPTER_RX_BAD_PACKETS,
 	EFX_MAC_VADAPTER_RX_BAD_BYTES,
 	EFX_MAC_VADAPTER_RX_OVERFLOW,
 	EFX_MAC_VADAPTER_TX_UNICAST_PACKETS,
 	EFX_MAC_VADAPTER_TX_UNICAST_BYTES,
 	EFX_MAC_VADAPTER_TX_MULTICAST_PACKETS,
 	EFX_MAC_VADAPTER_TX_MULTICAST_BYTES,
 	EFX_MAC_VADAPTER_TX_BROADCAST_PACKETS,
 	EFX_MAC_VADAPTER_TX_BROADCAST_BYTES,
 	EFX_MAC_VADAPTER_TX_BAD_PACKETS,
 	EFX_MAC_VADAPTER_TX_BAD_BYTES,
 	EFX_MAC_VADAPTER_TX_OVERFLOW,
 	EFX_MAC_NSTATS
 } efx_mac_stat_t;
 
 /* END MKCONFIG GENERATED EfxHeaderMacBlock */
 
 #endif	/* EFSYS_OPT_MAC_STATS */
 
 typedef enum efx_link_mode_e {
 	EFX_LINK_UNKNOWN = 0,
 	EFX_LINK_DOWN,
 	EFX_LINK_10HDX,
 	EFX_LINK_10FDX,
 	EFX_LINK_100HDX,
 	EFX_LINK_100FDX,
 	EFX_LINK_1000HDX,
 	EFX_LINK_1000FDX,
 	EFX_LINK_10000FDX,
 	EFX_LINK_40000FDX,
 	EFX_LINK_NMODES
 } efx_link_mode_t;
 
 #define	EFX_MAC_ADDR_LEN 6
 
 #define	EFX_MAC_ADDR_IS_MULTICAST(_address) (((uint8_t *)_address)[0] & 0x01)
 
 #define	EFX_MAC_MULTICAST_LIST_MAX	256
 
 #define	EFX_MAC_SDU_MAX	9202
 
 #define	EFX_MAC_PDU_ADJUSTMENT					\
 	(/* EtherII */ 14					\
 	    + /* VLAN */ 4					\
 	    + /* CRC */ 4					\
 	    + /* bug16011 */ 16)				\
 
 #define	EFX_MAC_PDU(_sdu)					\
 	P2ROUNDUP((_sdu) + EFX_MAC_PDU_ADJUSTMENT, 8)
 
 /*
  * Due to the P2ROUNDUP in EFX_MAC_PDU(), EFX_MAC_SDU_FROM_PDU() may give
  * the SDU rounded up slightly.
  */
 #define	EFX_MAC_SDU_FROM_PDU(_pdu)	((_pdu) - EFX_MAC_PDU_ADJUSTMENT)
 
 #define	EFX_MAC_PDU_MIN	60
 #define	EFX_MAC_PDU_MAX	EFX_MAC_PDU(EFX_MAC_SDU_MAX)
 
 extern	__checkReturn	efx_rc_t
 efx_mac_pdu_get(
 	__in		efx_nic_t *enp,
 	__out		size_t *pdu);
 
 extern	__checkReturn	efx_rc_t
 efx_mac_pdu_set(
 	__in		efx_nic_t *enp,
 	__in		size_t pdu);
 
 extern	__checkReturn	efx_rc_t
 efx_mac_addr_set(
 	__in		efx_nic_t *enp,
 	__in		uint8_t *addr);
 
 extern	__checkReturn			efx_rc_t
 efx_mac_filter_set(
 	__in				efx_nic_t *enp,
 	__in				boolean_t all_unicst,
 	__in				boolean_t mulcst,
 	__in				boolean_t all_mulcst,
 	__in				boolean_t brdcst);
 
 extern	__checkReturn	efx_rc_t
 efx_mac_multicast_list_set(
 	__in				efx_nic_t *enp,
 	__in_ecount(6*count)		uint8_t const *addrs,
 	__in				int count);
 
 extern	__checkReturn	efx_rc_t
 efx_mac_filter_default_rxq_set(
 	__in		efx_nic_t *enp,
 	__in		efx_rxq_t *erp,
 	__in		boolean_t using_rss);
 
 extern			void
 efx_mac_filter_default_rxq_clear(
 	__in		efx_nic_t *enp);
 
 extern	__checkReturn	efx_rc_t
 efx_mac_drain(
 	__in		efx_nic_t *enp,
 	__in		boolean_t enabled);
 
 extern	__checkReturn	efx_rc_t
 efx_mac_up(
 	__in		efx_nic_t *enp,
 	__out		boolean_t *mac_upp);
 
 #define	EFX_FCNTL_RESPOND	0x00000001
 #define	EFX_FCNTL_GENERATE	0x00000002
 
 extern	__checkReturn	efx_rc_t
 efx_mac_fcntl_set(
 	__in		efx_nic_t *enp,
 	__in		unsigned int fcntl,
 	__in		boolean_t autoneg);
 
 extern			void
 efx_mac_fcntl_get(
 	__in		efx_nic_t *enp,
 	__out		unsigned int *fcntl_wantedp,
 	__out		unsigned int *fcntl_linkp);
 
 
 #if EFSYS_OPT_MAC_STATS
 
 #if EFSYS_OPT_NAMES
 
 extern	__checkReturn			const char *
 efx_mac_stat_name(
 	__in				efx_nic_t *enp,
 	__in				unsigned int id);
 
 #endif	/* EFSYS_OPT_NAMES */
 
 #define	EFX_MAC_STATS_MASK_BITS_PER_PAGE	(8 * sizeof (uint32_t))
 
 #define	EFX_MAC_STATS_MASK_NPAGES	\
 	(P2ROUNDUP(EFX_MAC_NSTATS, EFX_MAC_STATS_MASK_BITS_PER_PAGE) / \
 	    EFX_MAC_STATS_MASK_BITS_PER_PAGE)
 
 /*
  * Get mask of MAC statistics supported by the hardware.
  *
  * If mask_size is insufficient to return the mask, EINVAL error is
  * returned. EFX_MAC_STATS_MASK_NPAGES multiplied by size of the page
  * (which is sizeof (uint32_t)) is sufficient.
  */
 extern	__checkReturn			efx_rc_t
 efx_mac_stats_get_mask(
 	__in				efx_nic_t *enp,
 	__out_bcount(mask_size)		uint32_t *maskp,
 	__in				size_t mask_size);
 
 #define	EFX_MAC_STAT_SUPPORTED(_mask, _stat)	\
 	((_mask)[(_stat) / EFX_MAC_STATS_MASK_BITS_PER_PAGE] &	\
 	 (1ULL << ((_stat) & (EFX_MAC_STATS_MASK_BITS_PER_PAGE - 1))))
 
 #define	EFX_MAC_STATS_SIZE 0x400
 
 /*
  * Upload mac statistics supported by the hardware into the given buffer.
  *
  * The reference buffer must be at least %EFX_MAC_STATS_SIZE bytes,
  * and page aligned.
  *
  * The hardware will only DMA statistics that it understands (of course).
  * Drivers should not make any assumptions about which statistics are
  * supported, especially when the statistics are generated by firmware.
  *
  * Thus, drivers should zero this buffer before use, so that not-understood
  * statistics read back as zero.
  */
 extern	__checkReturn			efx_rc_t
 efx_mac_stats_upload(
 	__in				efx_nic_t *enp,
 	__in				efsys_mem_t *esmp);
 
 extern	__checkReturn			efx_rc_t
 efx_mac_stats_periodic(
 	__in				efx_nic_t *enp,
 	__in				efsys_mem_t *esmp,
 	__in				uint16_t period_ms,
 	__in				boolean_t events);
 
 extern	__checkReturn			efx_rc_t
 efx_mac_stats_update(
 	__in				efx_nic_t *enp,
 	__in				efsys_mem_t *esmp,
 	__inout_ecount(EFX_MAC_NSTATS)	efsys_stat_t *stat,
 	__inout_opt			uint32_t *generationp);
 
 #endif	/* EFSYS_OPT_MAC_STATS */
 
 /* MON */
 
 typedef enum efx_mon_type_e {
 	EFX_MON_INVALID = 0,
 	EFX_MON_SFC90X0,
 	EFX_MON_SFC91X0,
 	EFX_MON_SFC92X0,
 	EFX_MON_NTYPES
 } efx_mon_type_t;
 
 #if EFSYS_OPT_NAMES
 
 extern		const char *
 efx_mon_name(
 	__in	efx_nic_t *enp);
 
 #endif	/* EFSYS_OPT_NAMES */
 
 extern	__checkReturn	efx_rc_t
 efx_mon_init(
 	__in		efx_nic_t *enp);
 
 #if EFSYS_OPT_MON_STATS
 
 #define	EFX_MON_STATS_PAGE_SIZE 0x100
 #define	EFX_MON_MASK_ELEMENT_SIZE 32
 
 /* START MKCONFIG GENERATED MonitorHeaderStatsBlock 5d4ee5185e419abe */
 typedef enum efx_mon_stat_e {
 	EFX_MON_STAT_2_5V,
 	EFX_MON_STAT_VCCP1,
 	EFX_MON_STAT_VCC,
 	EFX_MON_STAT_5V,
 	EFX_MON_STAT_12V,
 	EFX_MON_STAT_VCCP2,
 	EFX_MON_STAT_EXT_TEMP,
 	EFX_MON_STAT_INT_TEMP,
 	EFX_MON_STAT_AIN1,
 	EFX_MON_STAT_AIN2,
 	EFX_MON_STAT_INT_COOLING,
 	EFX_MON_STAT_EXT_COOLING,
 	EFX_MON_STAT_1V,
 	EFX_MON_STAT_1_2V,
 	EFX_MON_STAT_1_8V,
 	EFX_MON_STAT_3_3V,
 	EFX_MON_STAT_1_2VA,
 	EFX_MON_STAT_VREF,
 	EFX_MON_STAT_VAOE,
 	EFX_MON_STAT_AOE_TEMP,
 	EFX_MON_STAT_PSU_AOE_TEMP,
 	EFX_MON_STAT_PSU_TEMP,
 	EFX_MON_STAT_FAN0,
 	EFX_MON_STAT_FAN1,
 	EFX_MON_STAT_FAN2,
 	EFX_MON_STAT_FAN3,
 	EFX_MON_STAT_FAN4,
 	EFX_MON_STAT_VAOE_IN,
 	EFX_MON_STAT_IAOE,
 	EFX_MON_STAT_IAOE_IN,
 	EFX_MON_STAT_NIC_POWER,
 	EFX_MON_STAT_0_9V,
 	EFX_MON_STAT_I0_9V,
 	EFX_MON_STAT_I1_2V,
 	EFX_MON_STAT_0_9V_ADC,
 	EFX_MON_STAT_INT_TEMP2,
 	EFX_MON_STAT_VREG_TEMP,
 	EFX_MON_STAT_VREG_0_9V_TEMP,
 	EFX_MON_STAT_VREG_1_2V_TEMP,
 	EFX_MON_STAT_INT_VPTAT,
 	EFX_MON_STAT_INT_ADC_TEMP,
 	EFX_MON_STAT_EXT_VPTAT,
 	EFX_MON_STAT_EXT_ADC_TEMP,
 	EFX_MON_STAT_AMBIENT_TEMP,
 	EFX_MON_STAT_AIRFLOW,
 	EFX_MON_STAT_VDD08D_VSS08D_CSR,
 	EFX_MON_STAT_VDD08D_VSS08D_CSR_EXTADC,
 	EFX_MON_STAT_HOTPOINT_TEMP,
 	EFX_MON_STAT_PHY_POWER_SWITCH_PORT0,
 	EFX_MON_STAT_PHY_POWER_SWITCH_PORT1,
 	EFX_MON_STAT_MUM_VCC,
 	EFX_MON_STAT_0V9_A,
 	EFX_MON_STAT_I0V9_A,
 	EFX_MON_STAT_0V9_A_TEMP,
 	EFX_MON_STAT_0V9_B,
 	EFX_MON_STAT_I0V9_B,
 	EFX_MON_STAT_0V9_B_TEMP,
 	EFX_MON_STAT_CCOM_AVREG_1V2_SUPPLY,
 	EFX_MON_STAT_CCOM_AVREG_1V2_SUPPLY_EXT_ADC,
 	EFX_MON_STAT_CCOM_AVREG_1V8_SUPPLY,
 	EFX_MON_STAT_CCOM_AVREG_1V8_SUPPLY_EXT_ADC,
 	EFX_MON_STAT_CONTROLLER_MASTER_VPTAT,
 	EFX_MON_STAT_CONTROLLER_MASTER_INTERNAL_TEMP,
 	EFX_MON_STAT_CONTROLLER_MASTER_VPTAT_EXT_ADC,
 	EFX_MON_STAT_CONTROLLER_MASTER_INTERNAL_TEMP_EXT_ADC,
 	EFX_MON_STAT_CONTROLLER_SLAVE_VPTAT,
 	EFX_MON_STAT_CONTROLLER_SLAVE_INTERNAL_TEMP,
 	EFX_MON_STAT_CONTROLLER_SLAVE_VPTAT_EXT_ADC,
 	EFX_MON_STAT_CONTROLLER_SLAVE_INTERNAL_TEMP_EXT_ADC,
 	EFX_MON_STAT_SODIMM_VOUT,
 	EFX_MON_STAT_SODIMM_0_TEMP,
 	EFX_MON_STAT_SODIMM_1_TEMP,
 	EFX_MON_STAT_PHY0_VCC,
 	EFX_MON_STAT_PHY1_VCC,
 	EFX_MON_STAT_CONTROLLER_TDIODE_TEMP,
 	EFX_MON_STAT_BOARD_FRONT_TEMP,
 	EFX_MON_STAT_BOARD_BACK_TEMP,
 	EFX_MON_NSTATS
 } efx_mon_stat_t;
 
 /* END MKCONFIG GENERATED MonitorHeaderStatsBlock */
 
 typedef enum efx_mon_stat_state_e {
 	EFX_MON_STAT_STATE_OK = 0,
 	EFX_MON_STAT_STATE_WARNING = 1,
 	EFX_MON_STAT_STATE_FATAL = 2,
 	EFX_MON_STAT_STATE_BROKEN = 3,
 	EFX_MON_STAT_STATE_NO_READING = 4,
 } efx_mon_stat_state_t;
 
 typedef struct efx_mon_stat_value_s {
 	uint16_t	emsv_value;
 	uint16_t	emsv_state;
 } efx_mon_stat_value_t;
 
 #if EFSYS_OPT_NAMES
 
 extern					const char *
 efx_mon_stat_name(
 	__in				efx_nic_t *enp,
 	__in				efx_mon_stat_t id);
 
 #endif	/* EFSYS_OPT_NAMES */
 
 extern	__checkReturn			efx_rc_t
 efx_mon_stats_update(
 	__in				efx_nic_t *enp,
 	__in				efsys_mem_t *esmp,
 	__inout_ecount(EFX_MON_NSTATS)	efx_mon_stat_value_t *values);
 
 #endif	/* EFSYS_OPT_MON_STATS */
 
 extern		void
 efx_mon_fini(
 	__in	efx_nic_t *enp);
 
 /* PHY */
 
 extern	__checkReturn	efx_rc_t
 efx_phy_verify(
 	__in		efx_nic_t *enp);
 
 #if EFSYS_OPT_PHY_LED_CONTROL
 
 typedef enum efx_phy_led_mode_e {
 	EFX_PHY_LED_DEFAULT = 0,
 	EFX_PHY_LED_OFF,
 	EFX_PHY_LED_ON,
 	EFX_PHY_LED_FLASH,
 	EFX_PHY_LED_NMODES
 } efx_phy_led_mode_t;
 
 extern	__checkReturn	efx_rc_t
 efx_phy_led_set(
 	__in	efx_nic_t *enp,
 	__in	efx_phy_led_mode_t mode);
 
 #endif	/* EFSYS_OPT_PHY_LED_CONTROL */
 
 extern	__checkReturn	efx_rc_t
 efx_port_init(
 	__in		efx_nic_t *enp);
 
 #if EFSYS_OPT_LOOPBACK
 
 typedef enum efx_loopback_type_e {
 	EFX_LOOPBACK_OFF = 0,
 	EFX_LOOPBACK_DATA = 1,
 	EFX_LOOPBACK_GMAC = 2,
 	EFX_LOOPBACK_XGMII = 3,
 	EFX_LOOPBACK_XGXS = 4,
 	EFX_LOOPBACK_XAUI = 5,
 	EFX_LOOPBACK_GMII = 6,
 	EFX_LOOPBACK_SGMII = 7,
 	EFX_LOOPBACK_XGBR = 8,
 	EFX_LOOPBACK_XFI = 9,
 	EFX_LOOPBACK_XAUI_FAR = 10,
 	EFX_LOOPBACK_GMII_FAR = 11,
 	EFX_LOOPBACK_SGMII_FAR = 12,
 	EFX_LOOPBACK_XFI_FAR = 13,
 	EFX_LOOPBACK_GPHY = 14,
 	EFX_LOOPBACK_PHY_XS = 15,
 	EFX_LOOPBACK_PCS = 16,
 	EFX_LOOPBACK_PMA_PMD = 17,
 	EFX_LOOPBACK_XPORT = 18,
 	EFX_LOOPBACK_XGMII_WS = 19,
 	EFX_LOOPBACK_XAUI_WS = 20,
 	EFX_LOOPBACK_XAUI_WS_FAR = 21,
 	EFX_LOOPBACK_XAUI_WS_NEAR = 22,
 	EFX_LOOPBACK_GMII_WS = 23,
 	EFX_LOOPBACK_XFI_WS = 24,
 	EFX_LOOPBACK_XFI_WS_FAR = 25,
 	EFX_LOOPBACK_PHYXS_WS = 26,
 	EFX_LOOPBACK_PMA_INT = 27,
 	EFX_LOOPBACK_SD_NEAR = 28,
 	EFX_LOOPBACK_SD_FAR = 29,
 	EFX_LOOPBACK_PMA_INT_WS = 30,
 	EFX_LOOPBACK_SD_FEP2_WS = 31,
 	EFX_LOOPBACK_SD_FEP1_5_WS = 32,
 	EFX_LOOPBACK_SD_FEP_WS = 33,
 	EFX_LOOPBACK_SD_FES_WS = 34,
 	EFX_LOOPBACK_NTYPES
 } efx_loopback_type_t;
 
 typedef enum efx_loopback_kind_e {
 	EFX_LOOPBACK_KIND_OFF = 0,
 	EFX_LOOPBACK_KIND_ALL,
 	EFX_LOOPBACK_KIND_MAC,
 	EFX_LOOPBACK_KIND_PHY,
 	EFX_LOOPBACK_NKINDS
 } efx_loopback_kind_t;
 
 extern			void
 efx_loopback_mask(
 	__in	efx_loopback_kind_t loopback_kind,
 	__out	efx_qword_t *maskp);
 
 extern	__checkReturn	efx_rc_t
 efx_port_loopback_set(
 	__in	efx_nic_t *enp,
 	__in	efx_link_mode_t link_mode,
 	__in	efx_loopback_type_t type);
 
 #if EFSYS_OPT_NAMES
 
 extern	__checkReturn	const char *
 efx_loopback_type_name(
 	__in		efx_nic_t *enp,
 	__in		efx_loopback_type_t type);
 
 #endif	/* EFSYS_OPT_NAMES */
 
 #endif	/* EFSYS_OPT_LOOPBACK */
 
 extern	__checkReturn	efx_rc_t
 efx_port_poll(
 	__in		efx_nic_t *enp,
 	__out_opt	efx_link_mode_t	*link_modep);
 
 extern		void
 efx_port_fini(
 	__in	efx_nic_t *enp);
 
 typedef enum efx_phy_cap_type_e {
 	EFX_PHY_CAP_INVALID = 0,
 	EFX_PHY_CAP_10HDX,
 	EFX_PHY_CAP_10FDX,
 	EFX_PHY_CAP_100HDX,
 	EFX_PHY_CAP_100FDX,
 	EFX_PHY_CAP_1000HDX,
 	EFX_PHY_CAP_1000FDX,
 	EFX_PHY_CAP_10000FDX,
 	EFX_PHY_CAP_PAUSE,
 	EFX_PHY_CAP_ASYM,
 	EFX_PHY_CAP_AN,
 	EFX_PHY_CAP_40000FDX,
 	EFX_PHY_CAP_NTYPES
 } efx_phy_cap_type_t;
 
 
 #define	EFX_PHY_CAP_CURRENT	0x00000000
 #define	EFX_PHY_CAP_DEFAULT	0x00000001
 #define	EFX_PHY_CAP_PERM	0x00000002
 
 extern		void
 efx_phy_adv_cap_get(
 	__in		efx_nic_t *enp,
 	__in		uint32_t flag,
 	__out		uint32_t *maskp);
 
 extern	__checkReturn	efx_rc_t
 efx_phy_adv_cap_set(
 	__in		efx_nic_t *enp,
 	__in		uint32_t mask);
 
 extern			void
 efx_phy_lp_cap_get(
 	__in		efx_nic_t *enp,
 	__out		uint32_t *maskp);
 
 extern	__checkReturn	efx_rc_t
 efx_phy_oui_get(
 	__in		efx_nic_t *enp,
 	__out		uint32_t *ouip);
 
 typedef enum efx_phy_media_type_e {
 	EFX_PHY_MEDIA_INVALID = 0,
 	EFX_PHY_MEDIA_XAUI,
 	EFX_PHY_MEDIA_CX4,
 	EFX_PHY_MEDIA_KX4,
 	EFX_PHY_MEDIA_XFP,
 	EFX_PHY_MEDIA_SFP_PLUS,
 	EFX_PHY_MEDIA_BASE_T,
 	EFX_PHY_MEDIA_QSFP_PLUS,
 	EFX_PHY_MEDIA_NTYPES
 } efx_phy_media_type_t;
 
 /* Get the type of medium currently used.  If the board has ports for
  * modules, a module is present, and we recognise the media type of
  * the module, then this will be the media type of the module.
  * Otherwise it will be the media type of the port.
  */
 extern			void
 efx_phy_media_type_get(
 	__in		efx_nic_t *enp,
 	__out		efx_phy_media_type_t *typep);
 
 extern	__checkReturn		efx_rc_t
 efx_phy_module_get_info(
 	__in			efx_nic_t *enp,
 	__in			uint8_t dev_addr,
 	__in			uint8_t offset,
 	__in			uint8_t len,
 	__out_bcount(len)	uint8_t *data);
 
 #if EFSYS_OPT_PHY_STATS
 
 /* START MKCONFIG GENERATED PhyHeaderStatsBlock 30ed56ad501f8e36 */
 typedef enum efx_phy_stat_e {
 	EFX_PHY_STAT_OUI,
 	EFX_PHY_STAT_PMA_PMD_LINK_UP,
 	EFX_PHY_STAT_PMA_PMD_RX_FAULT,
 	EFX_PHY_STAT_PMA_PMD_TX_FAULT,
 	EFX_PHY_STAT_PMA_PMD_REV_A,
 	EFX_PHY_STAT_PMA_PMD_REV_B,
 	EFX_PHY_STAT_PMA_PMD_REV_C,
 	EFX_PHY_STAT_PMA_PMD_REV_D,
 	EFX_PHY_STAT_PCS_LINK_UP,
 	EFX_PHY_STAT_PCS_RX_FAULT,
 	EFX_PHY_STAT_PCS_TX_FAULT,
 	EFX_PHY_STAT_PCS_BER,
 	EFX_PHY_STAT_PCS_BLOCK_ERRORS,
 	EFX_PHY_STAT_PHY_XS_LINK_UP,
 	EFX_PHY_STAT_PHY_XS_RX_FAULT,
 	EFX_PHY_STAT_PHY_XS_TX_FAULT,
 	EFX_PHY_STAT_PHY_XS_ALIGN,
 	EFX_PHY_STAT_PHY_XS_SYNC_A,
 	EFX_PHY_STAT_PHY_XS_SYNC_B,
 	EFX_PHY_STAT_PHY_XS_SYNC_C,
 	EFX_PHY_STAT_PHY_XS_SYNC_D,
 	EFX_PHY_STAT_AN_LINK_UP,
 	EFX_PHY_STAT_AN_MASTER,
 	EFX_PHY_STAT_AN_LOCAL_RX_OK,
 	EFX_PHY_STAT_AN_REMOTE_RX_OK,
 	EFX_PHY_STAT_CL22EXT_LINK_UP,
 	EFX_PHY_STAT_SNR_A,
 	EFX_PHY_STAT_SNR_B,
 	EFX_PHY_STAT_SNR_C,
 	EFX_PHY_STAT_SNR_D,
 	EFX_PHY_STAT_PMA_PMD_SIGNAL_A,
 	EFX_PHY_STAT_PMA_PMD_SIGNAL_B,
 	EFX_PHY_STAT_PMA_PMD_SIGNAL_C,
 	EFX_PHY_STAT_PMA_PMD_SIGNAL_D,
 	EFX_PHY_STAT_AN_COMPLETE,
 	EFX_PHY_STAT_PMA_PMD_REV_MAJOR,
 	EFX_PHY_STAT_PMA_PMD_REV_MINOR,
 	EFX_PHY_STAT_PMA_PMD_REV_MICRO,
 	EFX_PHY_STAT_PCS_FW_VERSION_0,
 	EFX_PHY_STAT_PCS_FW_VERSION_1,
 	EFX_PHY_STAT_PCS_FW_VERSION_2,
 	EFX_PHY_STAT_PCS_FW_VERSION_3,
 	EFX_PHY_STAT_PCS_FW_BUILD_YY,
 	EFX_PHY_STAT_PCS_FW_BUILD_MM,
 	EFX_PHY_STAT_PCS_FW_BUILD_DD,
 	EFX_PHY_STAT_PCS_OP_MODE,
 	EFX_PHY_NSTATS
 } efx_phy_stat_t;
 
 /* END MKCONFIG GENERATED PhyHeaderStatsBlock */
 
 #if EFSYS_OPT_NAMES
 
 extern					const char *
 efx_phy_stat_name(
 	__in				efx_nic_t *enp,
 	__in				efx_phy_stat_t stat);
 
 #endif	/* EFSYS_OPT_NAMES */
 
 #define	EFX_PHY_STATS_SIZE 0x100
 
 extern	__checkReturn			efx_rc_t
 efx_phy_stats_update(
 	__in				efx_nic_t *enp,
 	__in				efsys_mem_t *esmp,
 	__inout_ecount(EFX_PHY_NSTATS)	uint32_t *stat);
 
 #endif	/* EFSYS_OPT_PHY_STATS */
 
 
 #if EFSYS_OPT_BIST
 
 typedef enum efx_bist_type_e {
 	EFX_BIST_TYPE_UNKNOWN,
 	EFX_BIST_TYPE_PHY_NORMAL,
 	EFX_BIST_TYPE_PHY_CABLE_SHORT,
 	EFX_BIST_TYPE_PHY_CABLE_LONG,
 	EFX_BIST_TYPE_MC_MEM,	/* Test the MC DMEM and IMEM */
 	EFX_BIST_TYPE_SAT_MEM,	/* Test the DMEM and IMEM of satellite cpus*/
 	EFX_BIST_TYPE_REG,	/* Test the register memories */
 	EFX_BIST_TYPE_NTYPES,
 } efx_bist_type_t;
 
 typedef enum efx_bist_result_e {
 	EFX_BIST_RESULT_UNKNOWN,
 	EFX_BIST_RESULT_RUNNING,
 	EFX_BIST_RESULT_PASSED,
 	EFX_BIST_RESULT_FAILED,
 } efx_bist_result_t;
 
 typedef enum efx_phy_cable_status_e {
 	EFX_PHY_CABLE_STATUS_OK,
 	EFX_PHY_CABLE_STATUS_INVALID,
 	EFX_PHY_CABLE_STATUS_OPEN,
 	EFX_PHY_CABLE_STATUS_INTRAPAIRSHORT,
 	EFX_PHY_CABLE_STATUS_INTERPAIRSHORT,
 	EFX_PHY_CABLE_STATUS_BUSY,
 } efx_phy_cable_status_t;
 
 typedef enum efx_bist_value_e {
 	EFX_BIST_PHY_CABLE_LENGTH_A,
 	EFX_BIST_PHY_CABLE_LENGTH_B,
 	EFX_BIST_PHY_CABLE_LENGTH_C,
 	EFX_BIST_PHY_CABLE_LENGTH_D,
 	EFX_BIST_PHY_CABLE_STATUS_A,
 	EFX_BIST_PHY_CABLE_STATUS_B,
 	EFX_BIST_PHY_CABLE_STATUS_C,
 	EFX_BIST_PHY_CABLE_STATUS_D,
 	EFX_BIST_FAULT_CODE,
 	/* Memory BIST specific values. These match to the MC_CMD_BIST_POLL
 	 * response. */
 	EFX_BIST_MEM_TEST,
 	EFX_BIST_MEM_ADDR,
 	EFX_BIST_MEM_BUS,
 	EFX_BIST_MEM_EXPECT,
 	EFX_BIST_MEM_ACTUAL,
 	EFX_BIST_MEM_ECC,
 	EFX_BIST_MEM_ECC_PARITY,
 	EFX_BIST_MEM_ECC_FATAL,
 	EFX_BIST_NVALUES,
 } efx_bist_value_t;
 
 extern	__checkReturn		efx_rc_t
 efx_bist_enable_offline(
 	__in			efx_nic_t *enp);
 
 extern	__checkReturn		efx_rc_t
 efx_bist_start(
 	__in			efx_nic_t *enp,
 	__in			efx_bist_type_t type);
 
 extern	__checkReturn		efx_rc_t
 efx_bist_poll(
 	__in			efx_nic_t *enp,
 	__in			efx_bist_type_t type,
 	__out			efx_bist_result_t *resultp,
 	__out_opt		uint32_t *value_maskp,
 	__out_ecount_opt(count)	unsigned long *valuesp,
 	__in			size_t count);
 
 extern				void
 efx_bist_stop(
 	__in			efx_nic_t *enp,
 	__in			efx_bist_type_t type);
 
 #endif	/* EFSYS_OPT_BIST */
 
 #define	EFX_FEATURE_IPV6		0x00000001
 #define	EFX_FEATURE_LFSR_HASH_INSERT	0x00000002
 #define	EFX_FEATURE_LINK_EVENTS		0x00000004
 #define	EFX_FEATURE_PERIODIC_MAC_STATS	0x00000008
 #define	EFX_FEATURE_MCDI		0x00000020
 #define	EFX_FEATURE_LOOKAHEAD_SPLIT	0x00000040
 #define	EFX_FEATURE_MAC_HEADER_FILTERS	0x00000080
 #define	EFX_FEATURE_TURBO		0x00000100
 #define	EFX_FEATURE_MCDI_DMA		0x00000200
 #define	EFX_FEATURE_TX_SRC_FILTERS	0x00000400
 #define	EFX_FEATURE_PIO_BUFFERS		0x00000800
 #define	EFX_FEATURE_FW_ASSISTED_TSO	0x00001000
 #define	EFX_FEATURE_FW_ASSISTED_TSO_V2	0x00002000
 
 typedef enum efx_tunnel_protocol_e {
 	EFX_TUNNEL_PROTOCOL_NONE = 0,
 	EFX_TUNNEL_PROTOCOL_VXLAN,
 	EFX_TUNNEL_PROTOCOL_GENEVE,
 	EFX_TUNNEL_PROTOCOL_NVGRE,
 	EFX_TUNNEL_NPROTOS
 } efx_tunnel_protocol_t;
 
 typedef struct efx_nic_cfg_s {
 	uint32_t		enc_board_type;
 	uint32_t		enc_phy_type;
 #if EFSYS_OPT_NAMES
 	char			enc_phy_name[21];
 #endif
 	char			enc_phy_revision[21];
 	efx_mon_type_t		enc_mon_type;
 #if EFSYS_OPT_MON_STATS
 	uint32_t		enc_mon_stat_dma_buf_size;
 	uint32_t		enc_mon_stat_mask[(EFX_MON_NSTATS + 31) / 32];
 #endif
 	unsigned int		enc_features;
 	uint8_t			enc_mac_addr[6];
 	uint8_t			enc_port;	/* PHY port number */
 	uint32_t		enc_intr_vec_base;
 	uint32_t		enc_intr_limit;
 	uint32_t		enc_evq_limit;
 	uint32_t		enc_txq_limit;
 	uint32_t		enc_rxq_limit;
 	uint32_t		enc_txq_max_ndescs;
 	uint32_t		enc_buftbl_limit;
 	uint32_t		enc_piobuf_limit;
 	uint32_t		enc_piobuf_size;
 	uint32_t		enc_piobuf_min_alloc_size;
 	uint32_t		enc_evq_timer_quantum_ns;
 	uint32_t		enc_evq_timer_max_us;
 	uint32_t		enc_clk_mult;
 	uint32_t		enc_rx_prefix_size;
 	uint32_t		enc_rx_buf_align_start;
 	uint32_t		enc_rx_buf_align_end;
 #if EFSYS_OPT_LOOPBACK
 	efx_qword_t		enc_loopback_types[EFX_LINK_NMODES];
 #endif	/* EFSYS_OPT_LOOPBACK */
 #if EFSYS_OPT_PHY_FLAGS
 	uint32_t		enc_phy_flags_mask;
 #endif	/* EFSYS_OPT_PHY_FLAGS */
 #if EFSYS_OPT_PHY_LED_CONTROL
 	uint32_t		enc_led_mask;
 #endif	/* EFSYS_OPT_PHY_LED_CONTROL */
 #if EFSYS_OPT_PHY_STATS
 	uint64_t		enc_phy_stat_mask;
 #endif	/* EFSYS_OPT_PHY_STATS */
 #if EFSYS_OPT_MCDI
 	uint8_t			enc_mcdi_mdio_channel;
 #if EFSYS_OPT_PHY_STATS
 	uint32_t		enc_mcdi_phy_stat_mask;
 #endif	/* EFSYS_OPT_PHY_STATS */
 #if EFSYS_OPT_MON_STATS
 	uint32_t		*enc_mcdi_sensor_maskp;
 	uint32_t		enc_mcdi_sensor_mask_size;
 #endif	/* EFSYS_OPT_MON_STATS */
 #endif	/* EFSYS_OPT_MCDI */
 #if EFSYS_OPT_BIST
 	uint32_t		enc_bist_mask;
 #endif	/* EFSYS_OPT_BIST */
 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD
 	uint32_t		enc_pf;
 	uint32_t		enc_vf;
 	uint32_t		enc_privilege_mask;
 #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */
 	boolean_t		enc_bug26807_workaround;
 	boolean_t		enc_bug35388_workaround;
 	boolean_t		enc_bug41750_workaround;
 	boolean_t		enc_bug61265_workaround;
 	boolean_t		enc_rx_batching_enabled;
 	/* Maximum number of descriptors completed in an rx event. */
 	uint32_t		enc_rx_batch_max;
 	/* Number of rx descriptors the hardware requires for a push. */
 	uint32_t		enc_rx_push_align;
 	/* Maximum amount of data in DMA descriptor */
 	uint32_t		enc_tx_dma_desc_size_max;
 	/*
 	 * Boundary which DMA descriptor data must not cross or 0 if no
 	 * limitation.
 	 */
 	uint32_t		enc_tx_dma_desc_boundary;
 	/*
 	 * Maximum number of bytes into the packet the TCP header can start for
 	 * the hardware to apply TSO packet edits.
 	 */
 	uint32_t		enc_tx_tso_tcp_header_offset_limit;
 	boolean_t		enc_fw_assisted_tso_enabled;
 	boolean_t		enc_fw_assisted_tso_v2_enabled;
 	/* Number of TSO contexts on the NIC (FATSOv2) */
 	uint32_t		enc_fw_assisted_tso_v2_n_contexts;
 	boolean_t		enc_hw_tx_insert_vlan_enabled;
 	/* Number of PFs on the NIC */
 	uint32_t		enc_hw_pf_count;
 	/* Datapath firmware vadapter/vport/vswitch support */
 	boolean_t		enc_datapath_cap_evb;
 	boolean_t		enc_rx_disable_scatter_supported;
 	boolean_t		enc_allow_set_mac_with_installed_filters;
 	boolean_t		enc_enhanced_set_mac_supported;
 	boolean_t		enc_init_evq_v2_supported;
 	boolean_t		enc_pm_and_rxdp_counters;
 	boolean_t		enc_mac_stats_40g_tx_size_bins;
 	uint32_t		enc_tunnel_encapsulations_supported;
 	/* External port identifier */
 	uint8_t			enc_external_port;
 	uint32_t		enc_mcdi_max_payload_length;
 	/* VPD may be per-PF or global */
 	boolean_t		enc_vpd_is_global;
 	/* Minimum unidirectional bandwidth in Mb/s to max out all ports */
 	uint32_t		enc_required_pcie_bandwidth_mbps;
 	uint32_t		enc_max_pcie_link_gen;
 	/* Firmware verifies integrity of NVRAM updates */
 	uint32_t		enc_fw_verified_nvram_update_required;
 } efx_nic_cfg_t;
 
 #define	EFX_PCI_FUNCTION_IS_PF(_encp)	((_encp)->enc_vf == 0xffff)
 #define	EFX_PCI_FUNCTION_IS_VF(_encp)	((_encp)->enc_vf != 0xffff)
 
 #define	EFX_PCI_FUNCTION(_encp)	\
 	(EFX_PCI_FUNCTION_IS_PF(_encp) ? (_encp)->enc_pf : (_encp)->enc_vf)
 
 #define	EFX_PCI_VF_PARENT(_encp)	((_encp)->enc_pf)
 
 extern			const efx_nic_cfg_t *
 efx_nic_cfg_get(
 	__in		efx_nic_t *enp);
 
 /* Driver resource limits (minimum required/maximum usable). */
 typedef struct efx_drv_limits_s {
 	uint32_t	edl_min_evq_count;
 	uint32_t	edl_max_evq_count;
 
 	uint32_t	edl_min_rxq_count;
 	uint32_t	edl_max_rxq_count;
 
 	uint32_t	edl_min_txq_count;
 	uint32_t	edl_max_txq_count;
 
 	/* PIO blocks (sub-allocated from piobuf) */
 	uint32_t	edl_min_pio_alloc_size;
 	uint32_t	edl_max_pio_alloc_count;
 } efx_drv_limits_t;
 
 extern	__checkReturn	efx_rc_t
 efx_nic_set_drv_limits(
 	__inout		efx_nic_t *enp,
 	__in		efx_drv_limits_t *edlp);
 
 typedef enum efx_nic_region_e {
 	EFX_REGION_VI,			/* Memory BAR UC mapping */
 	EFX_REGION_PIO_WRITE_VI,	/* Memory BAR WC mapping */
 } efx_nic_region_t;
 
 extern	__checkReturn	efx_rc_t
 efx_nic_get_bar_region(
 	__in		efx_nic_t *enp,
 	__in		efx_nic_region_t region,
 	__out		uint32_t *offsetp,
 	__out		size_t *sizep);
 
 extern	__checkReturn	efx_rc_t
 efx_nic_get_vi_pool(
 	__in		efx_nic_t *enp,
 	__out		uint32_t *evq_countp,
 	__out		uint32_t *rxq_countp,
 	__out		uint32_t *txq_countp);
 
 
 #if EFSYS_OPT_VPD
 
 typedef enum efx_vpd_tag_e {
 	EFX_VPD_ID = 0x02,
 	EFX_VPD_END = 0x0f,
 	EFX_VPD_RO = 0x10,
 	EFX_VPD_RW = 0x11,
 } efx_vpd_tag_t;
 
 typedef uint16_t efx_vpd_keyword_t;
 
 typedef struct efx_vpd_value_s {
 	efx_vpd_tag_t		evv_tag;
 	efx_vpd_keyword_t	evv_keyword;
 	uint8_t			evv_length;
 	uint8_t			evv_value[0x100];
 } efx_vpd_value_t;
 
 
 #define	EFX_VPD_KEYWORD(x, y) ((x) | ((y) << 8))
 
 extern	__checkReturn		efx_rc_t
 efx_vpd_init(
 	__in			efx_nic_t *enp);
 
 extern	__checkReturn		efx_rc_t
 efx_vpd_size(
 	__in			efx_nic_t *enp,
 	__out			size_t *sizep);
 
 extern	__checkReturn		efx_rc_t
 efx_vpd_read(
 	__in			efx_nic_t *enp,
 	__out_bcount(size)	caddr_t data,
 	__in			size_t size);
 
 extern	__checkReturn		efx_rc_t
 efx_vpd_verify(
 	__in			efx_nic_t *enp,
 	__in_bcount(size)	caddr_t data,
 	__in			size_t size);
 
 extern	__checkReturn		efx_rc_t
 efx_vpd_reinit(
 	__in			efx_nic_t *enp,
 	__in_bcount(size)	caddr_t data,
 	__in			size_t size);
 
 extern	__checkReturn		efx_rc_t
 efx_vpd_get(
 	__in			efx_nic_t *enp,
 	__in_bcount(size)	caddr_t data,
 	__in			size_t size,
 	__inout			efx_vpd_value_t *evvp);
 
 extern	__checkReturn		efx_rc_t
 efx_vpd_set(
 	__in			efx_nic_t *enp,
 	__inout_bcount(size)	caddr_t data,
 	__in			size_t size,
 	__in			efx_vpd_value_t *evvp);
 
 extern	__checkReturn		efx_rc_t
 efx_vpd_next(
 	__in			efx_nic_t *enp,
 	__inout_bcount(size)	caddr_t data,
 	__in			size_t size,
 	__out			efx_vpd_value_t *evvp,
 	__inout			unsigned int *contp);
 
 extern	__checkReturn		efx_rc_t
 efx_vpd_write(
 	__in			efx_nic_t *enp,
 	__in_bcount(size)	caddr_t data,
 	__in			size_t size);
 
 extern				void
 efx_vpd_fini(
 	__in			efx_nic_t *enp);
 
 #endif	/* EFSYS_OPT_VPD */
 
 /* NVRAM */
 
 #if EFSYS_OPT_NVRAM
 
 typedef enum efx_nvram_type_e {
 	EFX_NVRAM_INVALID = 0,
 	EFX_NVRAM_BOOTROM,
 	EFX_NVRAM_BOOTROM_CFG,
 	EFX_NVRAM_MC_FIRMWARE,
 	EFX_NVRAM_MC_GOLDEN,
 	EFX_NVRAM_PHY,
 	EFX_NVRAM_NULLPHY,
 	EFX_NVRAM_FPGA,
 	EFX_NVRAM_FCFW,
 	EFX_NVRAM_CPLD,
 	EFX_NVRAM_FPGA_BACKUP,
 	EFX_NVRAM_DYNAMIC_CFG,
 	EFX_NVRAM_LICENSE,
 	EFX_NVRAM_UEFIROM,
 	EFX_NVRAM_NTYPES,
 } efx_nvram_type_t;
 
 extern	__checkReturn		efx_rc_t
 efx_nvram_init(
 	__in			efx_nic_t *enp);
 
 #if EFSYS_OPT_DIAG
 
 extern	__checkReturn		efx_rc_t
 efx_nvram_test(
 	__in			efx_nic_t *enp);
 
 #endif	/* EFSYS_OPT_DIAG */
 
 extern	__checkReturn		efx_rc_t
 efx_nvram_size(
 	__in			efx_nic_t *enp,
 	__in			efx_nvram_type_t type,
 	__out			size_t *sizep);
 
 extern	__checkReturn		efx_rc_t
 efx_nvram_rw_start(
 	__in			efx_nic_t *enp,
 	__in			efx_nvram_type_t type,
 	__out_opt		size_t *pref_chunkp);
 
 extern	__checkReturn		efx_rc_t
 efx_nvram_rw_finish(
 	__in			efx_nic_t *enp,
 	__in			efx_nvram_type_t type);
 
 extern	__checkReturn		efx_rc_t
 efx_nvram_get_version(
 	__in			efx_nic_t *enp,
 	__in			efx_nvram_type_t type,
 	__out			uint32_t *subtypep,
 	__out_ecount(4)		uint16_t version[4]);
 
 extern	__checkReturn		efx_rc_t
 efx_nvram_read_chunk(
 	__in			efx_nic_t *enp,
 	__in			efx_nvram_type_t type,
 	__in			unsigned int offset,
 	__out_bcount(size)	caddr_t data,
 	__in			size_t size);
 
 extern	__checkReturn		efx_rc_t
 efx_nvram_set_version(
 	__in			efx_nic_t *enp,
 	__in			efx_nvram_type_t type,
 	__in_ecount(4)		uint16_t version[4]);
 
 extern	__checkReturn		efx_rc_t
 efx_nvram_validate(
 	__in			efx_nic_t *enp,
 	__in			efx_nvram_type_t type,
 	__in_bcount(partn_size)	caddr_t partn_data,
 	__in			size_t partn_size);
 
 extern	 __checkReturn		efx_rc_t
 efx_nvram_erase(
 	__in			efx_nic_t *enp,
 	__in			efx_nvram_type_t type);
 
 extern	__checkReturn		efx_rc_t
 efx_nvram_write_chunk(
 	__in			efx_nic_t *enp,
 	__in			efx_nvram_type_t type,
 	__in			unsigned int offset,
 	__in_bcount(size)	caddr_t data,
 	__in			size_t size);
 
 extern				void
 efx_nvram_fini(
 	__in			efx_nic_t *enp);
 
 #endif	/* EFSYS_OPT_NVRAM */
 
 #if EFSYS_OPT_BOOTCFG
 
 /* Report size and offset of bootcfg sector in NVRAM partition. */
 extern	__checkReturn		efx_rc_t
 efx_bootcfg_sector_info(
 	__in			efx_nic_t *enp,
 	__in			uint32_t pf,
 	__out_opt		uint32_t *sector_countp,
 	__out			size_t *offsetp,
 	__out			size_t *max_sizep);
 
 /*
  * Copy bootcfg sector data to a target buffer which may differ in size.
  * Optionally corrects format errors in source buffer.
  */
 extern				efx_rc_t
 efx_bootcfg_copy_sector(
 	__in			efx_nic_t *enp,
 	__inout_bcount(sector_length)
 				uint8_t *sector,
 	__in			size_t sector_length,
 	__out_bcount(data_size)	uint8_t *data,
 	__in			size_t data_size,
 	__in			boolean_t handle_format_errors);
 
 extern				efx_rc_t
 efx_bootcfg_read(
 	__in			efx_nic_t *enp,
 	__out_bcount(size)	caddr_t data,
 	__in			size_t size);
 
 extern				efx_rc_t
 efx_bootcfg_write(
 	__in			efx_nic_t *enp,
 	__in_bcount(size)	caddr_t data,
 	__in			size_t size);
 
 #endif	/* EFSYS_OPT_BOOTCFG */
 
 #if EFSYS_OPT_DIAG
 
 typedef enum efx_pattern_type_t {
 	EFX_PATTERN_BYTE_INCREMENT = 0,
 	EFX_PATTERN_ALL_THE_SAME,
 	EFX_PATTERN_BIT_ALTERNATE,
 	EFX_PATTERN_BYTE_ALTERNATE,
 	EFX_PATTERN_BYTE_CHANGING,
 	EFX_PATTERN_BIT_SWEEP,
 	EFX_PATTERN_NTYPES
 } efx_pattern_type_t;
 
 typedef			void
 (*efx_sram_pattern_fn_t)(
 	__in		size_t row,
 	__in		boolean_t negate,
 	__out		efx_qword_t *eqp);
 
 extern	__checkReturn	efx_rc_t
 efx_sram_test(
 	__in		efx_nic_t *enp,
 	__in		efx_pattern_type_t type);
 
 #endif	/* EFSYS_OPT_DIAG */
 
 extern	__checkReturn	efx_rc_t
 efx_sram_buf_tbl_set(
 	__in		efx_nic_t *enp,
 	__in		uint32_t id,
 	__in		efsys_mem_t *esmp,
 	__in		size_t n);
 
 extern		void
 efx_sram_buf_tbl_clear(
 	__in	efx_nic_t *enp,
 	__in	uint32_t id,
 	__in	size_t n);
 
 #define	EFX_BUF_TBL_SIZE	0x20000
 
 #define	EFX_BUF_SIZE		4096
 
 /* EV */
 
 typedef struct efx_evq_s	efx_evq_t;
 
 #if EFSYS_OPT_QSTATS
 
 /* START MKCONFIG GENERATED EfxHeaderEventQueueBlock 6f3843f5fe7cc843 */
 typedef enum efx_ev_qstat_e {
 	EV_ALL,
 	EV_RX,
 	EV_RX_OK,
 	EV_RX_FRM_TRUNC,
 	EV_RX_TOBE_DISC,
 	EV_RX_PAUSE_FRM_ERR,
 	EV_RX_BUF_OWNER_ID_ERR,
 	EV_RX_IPV4_HDR_CHKSUM_ERR,
 	EV_RX_TCP_UDP_CHKSUM_ERR,
 	EV_RX_ETH_CRC_ERR,
 	EV_RX_IP_FRAG_ERR,
 	EV_RX_MCAST_PKT,
 	EV_RX_MCAST_HASH_MATCH,
 	EV_RX_TCP_IPV4,
 	EV_RX_TCP_IPV6,
 	EV_RX_UDP_IPV4,
 	EV_RX_UDP_IPV6,
 	EV_RX_OTHER_IPV4,
 	EV_RX_OTHER_IPV6,
 	EV_RX_NON_IP,
 	EV_RX_BATCH,
 	EV_TX,
 	EV_TX_WQ_FF_FULL,
 	EV_TX_PKT_ERR,
 	EV_TX_PKT_TOO_BIG,
 	EV_TX_UNEXPECTED,
 	EV_GLOBAL,
 	EV_GLOBAL_MNT,
 	EV_DRIVER,
 	EV_DRIVER_SRM_UPD_DONE,
 	EV_DRIVER_TX_DESCQ_FLS_DONE,
 	EV_DRIVER_RX_DESCQ_FLS_DONE,
 	EV_DRIVER_RX_DESCQ_FLS_FAILED,
 	EV_DRIVER_RX_DSC_ERROR,
 	EV_DRIVER_TX_DSC_ERROR,
 	EV_DRV_GEN,
 	EV_MCDI_RESPONSE,
 	EV_NQSTATS
 } efx_ev_qstat_t;
 
 /* END MKCONFIG GENERATED EfxHeaderEventQueueBlock */
 
 #endif	/* EFSYS_OPT_QSTATS */
 
 extern	__checkReturn	efx_rc_t
 efx_ev_init(
 	__in		efx_nic_t *enp);
 
 extern		void
 efx_ev_fini(
 	__in		efx_nic_t *enp);
 
 #define	EFX_EVQ_MAXNEVS		32768
 #define	EFX_EVQ_MINNEVS		512
 
 #define	EFX_EVQ_SIZE(_nevs)	((_nevs) * sizeof (efx_qword_t))
 #define	EFX_EVQ_NBUFS(_nevs)	(EFX_EVQ_SIZE(_nevs) / EFX_BUF_SIZE)
 
 #define	EFX_EVQ_FLAGS_TYPE_MASK		(0x3)
 #define	EFX_EVQ_FLAGS_TYPE_AUTO		(0x0)
 #define	EFX_EVQ_FLAGS_TYPE_THROUGHPUT	(0x1)
 #define	EFX_EVQ_FLAGS_TYPE_LOW_LATENCY	(0x2)
 
 #define	EFX_EVQ_FLAGS_NOTIFY_MASK	(0xC)
 #define	EFX_EVQ_FLAGS_NOTIFY_INTERRUPT	(0x0)	/* Interrupting (default) */
 #define	EFX_EVQ_FLAGS_NOTIFY_DISABLED	(0x4)	/* Non-interrupting */
 
 extern	__checkReturn	efx_rc_t
 efx_ev_qcreate(
 	__in		efx_nic_t *enp,
 	__in		unsigned int index,
 	__in		efsys_mem_t *esmp,
 	__in		size_t n,
 	__in		uint32_t id,
 	__in		uint32_t us,
 	__in		uint32_t flags,
 	__deref_out	efx_evq_t **eepp);
 
 extern		void
 efx_ev_qpost(
 	__in		efx_evq_t *eep,
 	__in		uint16_t data);
 
 typedef __checkReturn	boolean_t
 (*efx_initialized_ev_t)(
 	__in_opt	void *arg);
 
 #define	EFX_PKT_UNICAST		0x0004
 #define	EFX_PKT_START		0x0008
 
 #define	EFX_PKT_VLAN_TAGGED	0x0010
 #define	EFX_CKSUM_TCPUDP	0x0020
 #define	EFX_CKSUM_IPV4		0x0040
 #define	EFX_PKT_CONT		0x0080
 
 #define	EFX_CHECK_VLAN		0x0100
 #define	EFX_PKT_TCP		0x0200
 #define	EFX_PKT_UDP		0x0400
 #define	EFX_PKT_IPV4		0x0800
 
 #define	EFX_PKT_IPV6		0x1000
 #define	EFX_PKT_PREFIX_LEN	0x2000
 #define	EFX_ADDR_MISMATCH	0x4000
 #define	EFX_DISCARD		0x8000
 
 #define	EFX_EV_RX_NLABELS	32
 #define	EFX_EV_TX_NLABELS	32
 
 typedef	__checkReturn	boolean_t
 (*efx_rx_ev_t)(
 	__in_opt	void *arg,
 	__in		uint32_t label,
 	__in		uint32_t id,
 	__in		uint32_t size,
 	__in		uint16_t flags);
 
 typedef	__checkReturn	boolean_t
 (*efx_tx_ev_t)(
 	__in_opt	void *arg,
 	__in		uint32_t label,
 	__in		uint32_t id);
 
 #define	EFX_EXCEPTION_RX_RECOVERY	0x00000001
 #define	EFX_EXCEPTION_RX_DSC_ERROR	0x00000002
 #define	EFX_EXCEPTION_TX_DSC_ERROR	0x00000003
 #define	EFX_EXCEPTION_UNKNOWN_SENSOREVT	0x00000004
 #define	EFX_EXCEPTION_FWALERT_SRAM	0x00000005
 #define	EFX_EXCEPTION_UNKNOWN_FWALERT	0x00000006
 #define	EFX_EXCEPTION_RX_ERROR		0x00000007
 #define	EFX_EXCEPTION_TX_ERROR		0x00000008
 #define	EFX_EXCEPTION_EV_ERROR		0x00000009
 
 typedef	__checkReturn	boolean_t
 (*efx_exception_ev_t)(
 	__in_opt	void *arg,
 	__in		uint32_t label,
 	__in		uint32_t data);
 
 typedef	__checkReturn	boolean_t
 (*efx_rxq_flush_done_ev_t)(
 	__in_opt	void *arg,
 	__in		uint32_t rxq_index);
 
 typedef	__checkReturn	boolean_t
 (*efx_rxq_flush_failed_ev_t)(
 	__in_opt	void *arg,
 	__in		uint32_t rxq_index);
 
 typedef	__checkReturn	boolean_t
 (*efx_txq_flush_done_ev_t)(
 	__in_opt	void *arg,
 	__in		uint32_t txq_index);
 
 typedef	__checkReturn	boolean_t
 (*efx_software_ev_t)(
 	__in_opt	void *arg,
 	__in		uint16_t magic);
 
 typedef	__checkReturn	boolean_t
 (*efx_sram_ev_t)(
 	__in_opt	void *arg,
 	__in		uint32_t code);
 
 #define	EFX_SRAM_CLEAR		0
 #define	EFX_SRAM_UPDATE		1
 #define	EFX_SRAM_ILLEGAL_CLEAR	2
 
 typedef	__checkReturn	boolean_t
 (*efx_wake_up_ev_t)(
 	__in_opt	void *arg,
 	__in		uint32_t label);
 
 typedef	__checkReturn	boolean_t
 (*efx_timer_ev_t)(
 	__in_opt	void *arg,
 	__in		uint32_t label);
 
 typedef __checkReturn	boolean_t
 (*efx_link_change_ev_t)(
 	__in_opt	void *arg,
 	__in		efx_link_mode_t	link_mode);
 
 #if EFSYS_OPT_MON_STATS
 
 typedef __checkReturn	boolean_t
 (*efx_monitor_ev_t)(
 	__in_opt	void *arg,
 	__in		efx_mon_stat_t id,
 	__in		efx_mon_stat_value_t value);
 
 #endif	/* EFSYS_OPT_MON_STATS */
 
 #if EFSYS_OPT_MAC_STATS
 
 typedef __checkReturn	boolean_t
 (*efx_mac_stats_ev_t)(
 	__in_opt	void *arg,
 	__in		uint32_t generation
 	);
 
 #endif	/* EFSYS_OPT_MAC_STATS */
 
 typedef struct efx_ev_callbacks_s {
 	efx_initialized_ev_t		eec_initialized;
 	efx_rx_ev_t			eec_rx;
 	efx_tx_ev_t			eec_tx;
 	efx_exception_ev_t		eec_exception;
 	efx_rxq_flush_done_ev_t		eec_rxq_flush_done;
 	efx_rxq_flush_failed_ev_t	eec_rxq_flush_failed;
 	efx_txq_flush_done_ev_t		eec_txq_flush_done;
 	efx_software_ev_t		eec_software;
 	efx_sram_ev_t			eec_sram;
 	efx_wake_up_ev_t		eec_wake_up;
 	efx_timer_ev_t			eec_timer;
 	efx_link_change_ev_t		eec_link_change;
 #if EFSYS_OPT_MON_STATS
 	efx_monitor_ev_t		eec_monitor;
 #endif	/* EFSYS_OPT_MON_STATS */
 #if EFSYS_OPT_MAC_STATS
 	efx_mac_stats_ev_t		eec_mac_stats;
 #endif	/* EFSYS_OPT_MAC_STATS */
 } efx_ev_callbacks_t;
 
 extern	__checkReturn	boolean_t
 efx_ev_qpending(
 	__in		efx_evq_t *eep,
 	__in		unsigned int count);
 
 #if EFSYS_OPT_EV_PREFETCH
 
 extern			void
 efx_ev_qprefetch(
 	__in		efx_evq_t *eep,
 	__in		unsigned int count);
 
 #endif	/* EFSYS_OPT_EV_PREFETCH */
 
 extern			void
 efx_ev_qpoll(
 	__in		efx_evq_t *eep,
 	__inout		unsigned int *countp,
 	__in		const efx_ev_callbacks_t *eecp,
 	__in_opt	void *arg);
 
 extern	__checkReturn	efx_rc_t
 efx_ev_usecs_to_ticks(
 	__in		efx_nic_t *enp,
 	__in		unsigned int usecs,
 	__out		unsigned int *ticksp);
 
 extern	__checkReturn	efx_rc_t
 efx_ev_qmoderate(
 	__in		efx_evq_t *eep,
 	__in		unsigned int us);
 
 extern	__checkReturn	efx_rc_t
 efx_ev_qprime(
 	__in		efx_evq_t *eep,
 	__in		unsigned int count);
 
 #if EFSYS_OPT_QSTATS
 
 #if EFSYS_OPT_NAMES
 
 extern		const char *
 efx_ev_qstat_name(
 	__in	efx_nic_t *enp,
 	__in	unsigned int id);
 
 #endif	/* EFSYS_OPT_NAMES */
 
 extern					void
 efx_ev_qstats_update(
 	__in				efx_evq_t *eep,
 	__inout_ecount(EV_NQSTATS)	efsys_stat_t *stat);
 
 #endif	/* EFSYS_OPT_QSTATS */
 
 extern		void
 efx_ev_qdestroy(
 	__in	efx_evq_t *eep);
 
 /* RX */
 
 extern	__checkReturn	efx_rc_t
 efx_rx_init(
 	__inout		efx_nic_t *enp);
 
 extern		void
 efx_rx_fini(
 	__in		efx_nic_t *enp);
 
 #if EFSYS_OPT_RX_SCATTER
 	__checkReturn	efx_rc_t
 efx_rx_scatter_enable(
 	__in		efx_nic_t *enp,
 	__in		unsigned int buf_size);
 #endif	/* EFSYS_OPT_RX_SCATTER */
 
 /* Handle to represent use of the default RSS context. */
 #define	EFX_RSS_CONTEXT_DEFAULT	0xffffffff
 
 #if EFSYS_OPT_RX_SCALE
 
 typedef enum efx_rx_hash_alg_e {
 	EFX_RX_HASHALG_LFSR = 0,
 	EFX_RX_HASHALG_TOEPLITZ
 } efx_rx_hash_alg_t;
 
 #define	EFX_RX_HASH_IPV4	(1U << 0)
 #define	EFX_RX_HASH_TCPIPV4	(1U << 1)
 #define	EFX_RX_HASH_IPV6	(1U << 2)
 #define	EFX_RX_HASH_TCPIPV6	(1U << 3)
 
 typedef unsigned int efx_rx_hash_type_t;
 
 typedef enum efx_rx_hash_support_e {
 	EFX_RX_HASH_UNAVAILABLE = 0,	/* Hardware hash not inserted */
 	EFX_RX_HASH_AVAILABLE		/* Insert hash with/without RSS */
 } efx_rx_hash_support_t;
 
 #define	EFX_RSS_TBL_SIZE	128	/* Rows in RX indirection table */
 #define	EFX_MAXRSS		64	/* RX indirection entry range */
 #define	EFX_MAXRSS_LEGACY	16	/* See bug16611 and bug17213 */
 
 typedef enum efx_rx_scale_support_e {
 	EFX_RX_SCALE_UNAVAILABLE = 0,	/* Not supported */
 	EFX_RX_SCALE_EXCLUSIVE,		/* Writable key/indirection table */
 	EFX_RX_SCALE_SHARED		/* Read-only key/indirection table */
 } efx_rx_scale_support_t;
 
 extern	__checkReturn	efx_rc_t
 efx_rx_hash_support_get(
 	__in		efx_nic_t *enp,
 	__out		efx_rx_hash_support_t *supportp);
 
 
 extern	__checkReturn	efx_rc_t
 efx_rx_scale_support_get(
 	__in		efx_nic_t *enp,
 	__out		efx_rx_scale_support_t *supportp);
 
 extern	__checkReturn	efx_rc_t
 efx_rx_scale_mode_set(
 	__in	efx_nic_t *enp,
 	__in	efx_rx_hash_alg_t alg,
 	__in	efx_rx_hash_type_t type,
 	__in	boolean_t insert);
 
 extern	__checkReturn	efx_rc_t
 efx_rx_scale_tbl_set(
 	__in		efx_nic_t *enp,
 	__in_ecount(n)	unsigned int *table,
 	__in		size_t n);
 
 extern	__checkReturn	efx_rc_t
 efx_rx_scale_key_set(
 	__in		efx_nic_t *enp,
 	__in_ecount(n)	uint8_t *key,
 	__in		size_t n);
 
 extern	__checkReturn	uint32_t
 efx_pseudo_hdr_hash_get(
 	__in		efx_rxq_t *erp,
 	__in		efx_rx_hash_alg_t func,
 	__in		uint8_t *buffer);
 
 #endif	/* EFSYS_OPT_RX_SCALE */
 
 extern	__checkReturn	efx_rc_t
 efx_pseudo_hdr_pkt_length_get(
 	__in		efx_rxq_t *erp,
 	__in		uint8_t *buffer,
 	__out		uint16_t *pkt_lengthp);
 
 #define	EFX_RXQ_MAXNDESCS		4096
 #define	EFX_RXQ_MINNDESCS		512
 
 #define	EFX_RXQ_SIZE(_ndescs)		((_ndescs) * sizeof (efx_qword_t))
 #define	EFX_RXQ_NBUFS(_ndescs)		(EFX_RXQ_SIZE(_ndescs) / EFX_BUF_SIZE)
 #define	EFX_RXQ_LIMIT(_ndescs)		((_ndescs) - 16)
 #define	EFX_RXQ_DC_NDESCS(_dcsize)	(8 << _dcsize)
 
 typedef enum efx_rxq_type_e {
 	EFX_RXQ_TYPE_DEFAULT,
 	EFX_RXQ_TYPE_SCATTER,
 	EFX_RXQ_NTYPES
 } efx_rxq_type_t;
 
 extern	__checkReturn	efx_rc_t
 efx_rx_qcreate(
 	__in		efx_nic_t *enp,
 	__in		unsigned int index,
 	__in		unsigned int label,
 	__in		efx_rxq_type_t type,
 	__in		efsys_mem_t *esmp,
 	__in		size_t n,
 	__in		uint32_t id,
 	__in		efx_evq_t *eep,
 	__deref_out	efx_rxq_t **erpp);
 
 typedef struct efx_buffer_s {
 	efsys_dma_addr_t	eb_addr;
 	size_t			eb_size;
 	boolean_t		eb_eop;
 } efx_buffer_t;
 
 typedef struct efx_desc_s {
 	efx_qword_t ed_eq;
 } efx_desc_t;
 
 extern			void
 efx_rx_qpost(
 	__in		efx_rxq_t *erp,
 	__in_ecount(n)	efsys_dma_addr_t *addrp,
 	__in		size_t size,
 	__in		unsigned int n,
 	__in		unsigned int completed,
 	__in		unsigned int added);
 
 extern		void
 efx_rx_qpush(
 	__in	efx_rxq_t *erp,
 	__in	unsigned int added,
 	__inout	unsigned int *pushedp);
 
 extern	__checkReturn	efx_rc_t
 efx_rx_qflush(
 	__in	efx_rxq_t *erp);
 
 extern		void
 efx_rx_qenable(
 	__in	efx_rxq_t *erp);
 
 extern		void
 efx_rx_qdestroy(
 	__in	efx_rxq_t *erp);
 
 /* TX */
 
 typedef struct efx_txq_s	efx_txq_t;
 
 #if EFSYS_OPT_QSTATS
 
 /* START MKCONFIG GENERATED EfxHeaderTransmitQueueBlock 12dff8778598b2db */
 typedef enum efx_tx_qstat_e {
 	TX_POST,
 	TX_POST_PIO,
 	TX_NQSTATS
 } efx_tx_qstat_t;
 
 /* END MKCONFIG GENERATED EfxHeaderTransmitQueueBlock */
 
 #endif	/* EFSYS_OPT_QSTATS */
 
 extern	__checkReturn	efx_rc_t
 efx_tx_init(
 	__in		efx_nic_t *enp);
 
 extern		void
 efx_tx_fini(
 	__in	efx_nic_t *enp);
 
 #define	EFX_TXQ_MINNDESCS		512
 
 #define	EFX_TXQ_SIZE(_ndescs)		((_ndescs) * sizeof (efx_qword_t))
 #define	EFX_TXQ_NBUFS(_ndescs)		(EFX_TXQ_SIZE(_ndescs) / EFX_BUF_SIZE)
 #define	EFX_TXQ_LIMIT(_ndescs)		((_ndescs) - 16)
 #define	EFX_TXQ_DC_NDESCS(_dcsize)	(8 << _dcsize)
 
 #define	EFX_TXQ_MAX_BUFS 8 /* Maximum independent of EFX_BUG35388_WORKAROUND. */
 
 #define	EFX_TXQ_CKSUM_IPV4	0x0001
 #define	EFX_TXQ_CKSUM_TCPUDP	0x0002
 #define	EFX_TXQ_FATSOV2		0x0004
 
 extern	__checkReturn	efx_rc_t
 efx_tx_qcreate(
 	__in		efx_nic_t *enp,
 	__in		unsigned int index,
 	__in		unsigned int label,
 	__in		efsys_mem_t *esmp,
 	__in		size_t n,
 	__in		uint32_t id,
 	__in		uint16_t flags,
 	__in		efx_evq_t *eep,
 	__deref_out	efx_txq_t **etpp,
 	__out		unsigned int *addedp);
 
 extern	__checkReturn	efx_rc_t
 efx_tx_qpost(
 	__in		efx_txq_t *etp,
 	__in_ecount(n)	efx_buffer_t *eb,
 	__in		unsigned int n,
 	__in		unsigned int completed,
 	__inout		unsigned int *addedp);
 
 extern	__checkReturn	efx_rc_t
 efx_tx_qpace(
 	__in		efx_txq_t *etp,
 	__in		unsigned int ns);
 
 extern			void
 efx_tx_qpush(
 	__in		efx_txq_t *etp,
 	__in		unsigned int added,
 	__in		unsigned int pushed);
 
 extern	__checkReturn	efx_rc_t
 efx_tx_qflush(
 	__in		efx_txq_t *etp);
 
 extern			void
 efx_tx_qenable(
 	__in		efx_txq_t *etp);
 
 extern	__checkReturn	efx_rc_t
 efx_tx_qpio_enable(
 	__in		efx_txq_t *etp);
 
 extern			void
 efx_tx_qpio_disable(
 	__in		efx_txq_t *etp);
 
 extern	__checkReturn	efx_rc_t
 efx_tx_qpio_write(
 	__in			efx_txq_t *etp,
 	__in_ecount(buf_length)	uint8_t *buffer,
 	__in			size_t buf_length,
 	__in			size_t pio_buf_offset);
 
 extern	__checkReturn	efx_rc_t
 efx_tx_qpio_post(
 	__in			efx_txq_t *etp,
 	__in			size_t pkt_length,
 	__in			unsigned int completed,
 	__inout			unsigned int *addedp);
 
 extern	__checkReturn	efx_rc_t
 efx_tx_qdesc_post(
 	__in		efx_txq_t *etp,
 	__in_ecount(n)	efx_desc_t *ed,
 	__in		unsigned int n,
 	__in		unsigned int completed,
 	__inout		unsigned int *addedp);
 
 extern	void
 efx_tx_qdesc_dma_create(
 	__in	efx_txq_t *etp,
 	__in	efsys_dma_addr_t addr,
 	__in	size_t size,
 	__in	boolean_t eop,
 	__out	efx_desc_t *edp);
 
 extern	void
 efx_tx_qdesc_tso_create(
 	__in	efx_txq_t *etp,
 	__in	uint16_t ipv4_id,
 	__in	uint32_t tcp_seq,
 	__in	uint8_t  tcp_flags,
 	__out	efx_desc_t *edp);
 
 /* Number of FATSOv2 option descriptors */
 #define	EFX_TX_FATSOV2_OPT_NDESCS		2
 
 /* Maximum number of DMA segments per TSO packet (not superframe) */
 #define	EFX_TX_FATSOV2_DMA_SEGS_PER_PKT_MAX	24
 
 extern	void
 efx_tx_qdesc_tso2_create(
 	__in			efx_txq_t *etp,
 	__in			uint16_t ipv4_id,
 	__in			uint32_t tcp_seq,
 	__in			uint16_t tcp_mss,
 	__out_ecount(count)	efx_desc_t *edp,
 	__in			int count);
 
 extern	void
 efx_tx_qdesc_vlantci_create(
 	__in	efx_txq_t *etp,
 	__in	uint16_t tci,
 	__out	efx_desc_t *edp);
 
 #if EFSYS_OPT_QSTATS
 
 #if EFSYS_OPT_NAMES
 
 extern		const char *
 efx_tx_qstat_name(
 	__in	efx_nic_t *etp,
 	__in	unsigned int id);
 
 #endif	/* EFSYS_OPT_NAMES */
 
 extern					void
 efx_tx_qstats_update(
 	__in				efx_txq_t *etp,
 	__inout_ecount(TX_NQSTATS)	efsys_stat_t *stat);
 
 #endif	/* EFSYS_OPT_QSTATS */
 
 extern		void
 efx_tx_qdestroy(
 	__in	efx_txq_t *etp);
 
 
 /* FILTER */
 
 #if EFSYS_OPT_FILTER
 
 #define	EFX_ETHER_TYPE_IPV4 0x0800
 #define	EFX_ETHER_TYPE_IPV6 0x86DD
 
 #define	EFX_IPPROTO_TCP 6
 #define	EFX_IPPROTO_UDP 17
+#define	EFX_IPPROTO_GRE	47
 
 /* Use RSS to spread across multiple queues */
 #define	EFX_FILTER_FLAG_RX_RSS		0x01
 /* Enable RX scatter */
 #define	EFX_FILTER_FLAG_RX_SCATTER	0x02
 /*
  * Override an automatic filter (priority EFX_FILTER_PRI_AUTO).
  * May only be set by the filter implementation for each type.
  * A removal request will restore the automatic filter in its place.
  */
 #define	EFX_FILTER_FLAG_RX_OVER_AUTO	0x04
 /* Filter is for RX */
 #define	EFX_FILTER_FLAG_RX		0x08
 /* Filter is for TX */
 #define	EFX_FILTER_FLAG_TX		0x10
 
 typedef unsigned int efx_filter_flags_t;
 
+/*
+ * Flags which specify the fields to match on. The values are the same as in the
+ * MC_CMD_FILTER_OP/MC_CMD_FILTER_OP_EXT commands.
+ */
 typedef enum efx_filter_match_flags_e {
 	EFX_FILTER_MATCH_REM_HOST = 0x0001,	/* Match by remote IP host
 						 * address */
 	EFX_FILTER_MATCH_LOC_HOST = 0x0002,	/* Match by local IP host
 						 * address */
 	EFX_FILTER_MATCH_REM_MAC = 0x0004,	/* Match by remote MAC address */
 	EFX_FILTER_MATCH_REM_PORT = 0x0008,	/* Match by remote TCP/UDP port */
 	EFX_FILTER_MATCH_LOC_MAC = 0x0010,	/* Match by remote TCP/UDP port */
 	EFX_FILTER_MATCH_LOC_PORT = 0x0020,	/* Match by local TCP/UDP port */
 	EFX_FILTER_MATCH_ETHER_TYPE = 0x0040,	/* Match by Ether-type */
 	EFX_FILTER_MATCH_INNER_VID = 0x0080,	/* Match by inner VLAN ID */
 	EFX_FILTER_MATCH_OUTER_VID = 0x0100,	/* Match by outer VLAN ID */
 	EFX_FILTER_MATCH_IP_PROTO = 0x0200,	/* Match by IP transport
 						 * protocol */
+	/* For encapsulated packets, match all multicast inner frames */
+	EFX_FILTER_MATCH_IFRM_UNKNOWN_MCAST_DST = 0x01000000,
+	/* For encapsulated packets, match all unicast inner frames */
+	EFX_FILTER_MATCH_IFRM_UNKNOWN_UCAST_DST = 0x02000000,
 	/* Match otherwise-unmatched multicast and broadcast packets */
 	EFX_FILTER_MATCH_UNKNOWN_MCAST_DST = 0x40000000,
 	/* Match otherwise-unmatched unicast packets */
 	EFX_FILTER_MATCH_UNKNOWN_UCAST_DST = 0x80000000,
 } efx_filter_match_flags_t;
 
 typedef enum efx_filter_priority_s {
 	EFX_FILTER_PRI_HINT = 0,	/* Performance hint */
 	EFX_FILTER_PRI_AUTO,		/* Automatic filter based on device
 					 * address list or hardware
 					 * requirements. This may only be used
 					 * by the filter implementation for
 					 * each NIC type. */
 	EFX_FILTER_PRI_MANUAL,		/* Manually configured filter */
 	EFX_FILTER_PRI_REQUIRED,	/* Required for correct behaviour of the
 					 * client (e.g. SR-IOV, HyperV VMQ etc.)
 					 */
 } efx_filter_priority_t;
 
 /*
  * FIXME: All these fields are assumed to be in little-endian byte order.
  * It may be better for some to be big-endian. See bug42804.
  */
 
 typedef struct efx_filter_spec_s {
-	uint32_t	efs_match_flags;
-	uint32_t	efs_priority:2;
-	uint32_t	efs_flags:6;
-	uint32_t	efs_dmaq_id:12;
-	uint32_t	efs_rss_context;
-	uint16_t	efs_outer_vid;
-	uint16_t	efs_inner_vid;
-	uint8_t		efs_loc_mac[EFX_MAC_ADDR_LEN];
-	uint8_t		efs_rem_mac[EFX_MAC_ADDR_LEN];
-	uint16_t	efs_ether_type;
-	uint8_t		efs_ip_proto;
-	uint16_t	efs_loc_port;
-	uint16_t	efs_rem_port;
-	efx_oword_t	efs_rem_host;
-	efx_oword_t	efs_loc_host;
+	uint32_t		efs_match_flags;
+	uint32_t		efs_priority:2;
+	uint32_t		efs_flags:6;
+	uint32_t		efs_dmaq_id:12;
+	uint32_t		efs_rss_context;
+	uint16_t		efs_outer_vid;
+	uint16_t		efs_inner_vid;
+	uint8_t			efs_loc_mac[EFX_MAC_ADDR_LEN];
+	uint8_t			efs_rem_mac[EFX_MAC_ADDR_LEN];
+	uint16_t		efs_ether_type;
+	uint8_t			efs_ip_proto;
+	efx_tunnel_protocol_t	efs_encap_type;
+	uint16_t		efs_loc_port;
+	uint16_t		efs_rem_port;
+	efx_oword_t		efs_rem_host;
+	efx_oword_t		efs_loc_host;
 } efx_filter_spec_t;
 
 
 /* Default values for use in filter specifications */
 #define	EFX_FILTER_SPEC_RX_DMAQ_ID_DROP		0xfff
 #define	EFX_FILTER_SPEC_VID_UNSPEC		0xffff
 
 extern	__checkReturn	efx_rc_t
 efx_filter_init(
 	__in		efx_nic_t *enp);
 
 extern			void
 efx_filter_fini(
 	__in		efx_nic_t *enp);
 
 extern	__checkReturn	efx_rc_t
 efx_filter_insert(
 	__in		efx_nic_t *enp,
 	__inout		efx_filter_spec_t *spec);
 
 extern	__checkReturn	efx_rc_t
 efx_filter_remove(
 	__in		efx_nic_t *enp,
 	__inout		efx_filter_spec_t *spec);
 
 extern	__checkReturn	efx_rc_t
 efx_filter_restore(
 	__in		efx_nic_t *enp);
 
 extern	__checkReturn	efx_rc_t
 efx_filter_supported_filters(
 	__in				efx_nic_t *enp,
 	__out_ecount(buffer_length)	uint32_t *buffer,
 	__in				size_t buffer_length,
 	__out				size_t *list_lengthp);
 
 extern			void
 efx_filter_spec_init_rx(
 	__out		efx_filter_spec_t *spec,
 	__in		efx_filter_priority_t priority,
 	__in		efx_filter_flags_t flags,
 	__in		efx_rxq_t *erp);
 
 extern			void
 efx_filter_spec_init_tx(
 	__out		efx_filter_spec_t *spec,
 	__in		efx_txq_t *etp);
 
 extern	__checkReturn	efx_rc_t
 efx_filter_spec_set_ipv4_local(
 	__inout		efx_filter_spec_t *spec,
 	__in		uint8_t proto,
 	__in		uint32_t host,
 	__in		uint16_t port);
 
 extern	__checkReturn	efx_rc_t
 efx_filter_spec_set_ipv4_full(
 	__inout		efx_filter_spec_t *spec,
 	__in		uint8_t proto,
 	__in		uint32_t lhost,
 	__in		uint16_t lport,
 	__in		uint32_t rhost,
 	__in		uint16_t rport);
 
 extern	__checkReturn	efx_rc_t
 efx_filter_spec_set_eth_local(
 	__inout		efx_filter_spec_t *spec,
 	__in		uint16_t vid,
 	__in		const uint8_t *addr);
 
+extern			void
+efx_filter_spec_set_ether_type(
+	__inout		efx_filter_spec_t *spec,
+	__in		uint16_t ether_type);
+
 extern	__checkReturn	efx_rc_t
 efx_filter_spec_set_uc_def(
 	__inout		efx_filter_spec_t *spec);
 
 extern	__checkReturn	efx_rc_t
 efx_filter_spec_set_mc_def(
 	__inout		efx_filter_spec_t *spec);
+
+typedef enum efx_filter_inner_frame_match_e {
+	EFX_FILTER_INNER_FRAME_MATCH_OTHER = 0,
+	EFX_FILTER_INNER_FRAME_MATCH_UNKNOWN_MCAST_DST,
+	EFX_FILTER_INNER_FRAME_MATCH_UNKNOWN_UCAST_DST
+} efx_filter_inner_frame_match_t;
+
+extern	__checkReturn	efx_rc_t
+efx_filter_spec_set_encap_type(
+	__inout		efx_filter_spec_t *spec,
+	__in		efx_tunnel_protocol_t encap_type,
+	__in		efx_filter_inner_frame_match_t inner_frame_match);
+
 
 #endif	/* EFSYS_OPT_FILTER */
 
 /* HASH */
 
 extern	__checkReturn		uint32_t
 efx_hash_dwords(
 	__in_ecount(count)	uint32_t const *input,
 	__in			size_t count,
 	__in			uint32_t init);
 
 extern	__checkReturn		uint32_t
 efx_hash_bytes(
 	__in_ecount(length)	uint8_t const *input,
 	__in			size_t length,
 	__in			uint32_t init);
 
 #if EFSYS_OPT_LICENSING
 
 /* LICENSING */
 
 typedef struct efx_key_stats_s {
 	uint32_t	eks_valid;
 	uint32_t	eks_invalid;
 	uint32_t	eks_blacklisted;
 	uint32_t	eks_unverifiable;
 	uint32_t	eks_wrong_node;
 	uint32_t	eks_licensed_apps_lo;
 	uint32_t	eks_licensed_apps_hi;
 	uint32_t	eks_licensed_features_lo;
 	uint32_t	eks_licensed_features_hi;
 } efx_key_stats_t;
 
 extern	__checkReturn		efx_rc_t
 efx_lic_init(
 	__in			efx_nic_t *enp);
 
 extern				void
 efx_lic_fini(
 	__in			efx_nic_t *enp);
 
 extern	__checkReturn	boolean_t
 efx_lic_check_support(
 	__in			efx_nic_t *enp);
 
 extern	__checkReturn	efx_rc_t
 efx_lic_update_licenses(
 	__in		efx_nic_t *enp);
 
 extern	__checkReturn	efx_rc_t
 efx_lic_get_key_stats(
 	__in		efx_nic_t *enp,
 	__out		efx_key_stats_t *ksp);
 
 extern	__checkReturn	efx_rc_t
 efx_lic_app_state(
 	__in		efx_nic_t *enp,
 	__in		uint64_t app_id,
 	__out		boolean_t *licensedp);
 
 extern	__checkReturn	efx_rc_t
 efx_lic_get_id(
 	__in		efx_nic_t *enp,
 	__in		size_t buffer_size,
 	__out		uint32_t *typep,
 	__out		size_t *lengthp,
 	__out_opt	uint8_t *bufferp);
 
 
 extern	__checkReturn		efx_rc_t
 efx_lic_find_start(
 	__in			efx_nic_t *enp,
 	__in_bcount(buffer_size)
 				caddr_t bufferp,
 	__in			size_t buffer_size,
 	__out			uint32_t *startp
 	);
 
 extern	__checkReturn		efx_rc_t
 efx_lic_find_end(
 	__in			efx_nic_t *enp,
 	__in_bcount(buffer_size)
 				caddr_t bufferp,
 	__in			size_t buffer_size,
 	__in			uint32_t offset,
 	__out			uint32_t *endp
 	);
 
 extern	__checkReturn	__success(return != B_FALSE)	boolean_t
 efx_lic_find_key(
 	__in			efx_nic_t *enp,
 	__in_bcount(buffer_size)
 				caddr_t bufferp,
 	__in			size_t buffer_size,
 	__in			uint32_t offset,
 	__out			uint32_t *startp,
 	__out			uint32_t *lengthp
 	);
 
 extern	__checkReturn	__success(return != B_FALSE)	boolean_t
 efx_lic_validate_key(
 	__in			efx_nic_t *enp,
 	__in_bcount(length)	caddr_t keyp,
 	__in			uint32_t length
 	);
 
 extern	__checkReturn		efx_rc_t
 efx_lic_read_key(
 	__in			efx_nic_t *enp,
 	__in_bcount(buffer_size)
 				caddr_t bufferp,
 	__in			size_t buffer_size,
 	__in			uint32_t offset,
 	__in			uint32_t length,
 	__out_bcount_part(key_max_size, *lengthp)
 				caddr_t keyp,
 	__in			size_t key_max_size,
 	__out			uint32_t *lengthp
 	);
 
 extern	__checkReturn		efx_rc_t
 efx_lic_write_key(
 	__in			efx_nic_t *enp,
 	__in_bcount(buffer_size)
 				caddr_t bufferp,
 	__in			size_t buffer_size,
 	__in			uint32_t offset,
 	__in_bcount(length)	caddr_t keyp,
 	__in			uint32_t length,
 	__out			uint32_t *lengthp
 	);
 
 	__checkReturn		efx_rc_t
 efx_lic_delete_key(
 	__in			efx_nic_t *enp,
 	__in_bcount(buffer_size)
 				caddr_t bufferp,
 	__in			size_t buffer_size,
 	__in			uint32_t offset,
 	__in			uint32_t length,
 	__in			uint32_t end,
 	__out			uint32_t *deltap
 	);
 
 extern	__checkReturn		efx_rc_t
 efx_lic_create_partition(
 	__in			efx_nic_t *enp,
 	__in_bcount(buffer_size)
 				caddr_t bufferp,
 	__in			size_t buffer_size
 	);
 
 extern	__checkReturn		efx_rc_t
 efx_lic_finish_partition(
 	__in			efx_nic_t *enp,
 	__in_bcount(buffer_size)
 				caddr_t bufferp,
 	__in			size_t buffer_size
 	);
 
 #endif	/* EFSYS_OPT_LICENSING */
 
 
 
 #ifdef	__cplusplus
 }
 #endif
 
 #endif	/* _SYS_EFX_H */
Index: stable/12/sys/dev/sfxge/common/efx_filter.c
===================================================================
--- stable/12/sys/dev/sfxge/common/efx_filter.c	(revision 342320)
+++ stable/12/sys/dev/sfxge/common/efx_filter.c	(revision 342321)
@@ -1,1429 +1,1497 @@
 /*-
  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
  *
  * Copyright (c) 2007-2016 Solarflare Communications Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
  *
  * 1. Redistributions of source code must retain the above copyright notice,
  *    this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright notice,
  *    this list of conditions and the following disclaimer in the documentation
  *    and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  * The views and conclusions contained in the software and documentation are
  * those of the authors and should not be interpreted as representing official
  * policies, either expressed or implied, of the FreeBSD Project.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include "efx.h"
 #include "efx_impl.h"
 
 
 #if EFSYS_OPT_FILTER
 
 #if EFSYS_OPT_SIENA
 
 static	__checkReturn	efx_rc_t
 siena_filter_init(
 	__in		efx_nic_t *enp);
 
 static			void
 siena_filter_fini(
 	__in		efx_nic_t *enp);
 
 static	__checkReturn	efx_rc_t
 siena_filter_restore(
 	__in		efx_nic_t *enp);
 
 static	__checkReturn	efx_rc_t
 siena_filter_add(
 	__in		efx_nic_t *enp,
 	__inout		efx_filter_spec_t *spec,
 	__in		boolean_t may_replace);
 
 static	__checkReturn	efx_rc_t
 siena_filter_delete(
 	__in		efx_nic_t *enp,
 	__inout		efx_filter_spec_t *spec);
 
 static	__checkReturn	efx_rc_t
 siena_filter_supported_filters(
 	__in				efx_nic_t *enp,
 	__out_ecount(buffer_length)	uint32_t *buffer,
 	__in				size_t buffer_length,
 	__out				size_t *list_lengthp);
 
 #endif /* EFSYS_OPT_SIENA */
 
 #if EFSYS_OPT_SIENA
 static const efx_filter_ops_t	__efx_filter_siena_ops = {
 	siena_filter_init,		/* efo_init */
 	siena_filter_fini,		/* efo_fini */
 	siena_filter_restore,		/* efo_restore */
 	siena_filter_add,		/* efo_add */
 	siena_filter_delete,		/* efo_delete */
 	siena_filter_supported_filters,	/* efo_supported_filters */
 	NULL,				/* efo_reconfigure */
 };
 #endif /* EFSYS_OPT_SIENA */
 
 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD
 static const efx_filter_ops_t	__efx_filter_ef10_ops = {
 	ef10_filter_init,		/* efo_init */
 	ef10_filter_fini,		/* efo_fini */
 	ef10_filter_restore,		/* efo_restore */
 	ef10_filter_add,		/* efo_add */
 	ef10_filter_delete,		/* efo_delete */
 	ef10_filter_supported_filters,	/* efo_supported_filters */
 	ef10_filter_reconfigure,	/* efo_reconfigure */
 };
 #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */
 
 	__checkReturn	efx_rc_t
 efx_filter_insert(
 	__in		efx_nic_t *enp,
 	__inout		efx_filter_spec_t *spec)
 {
 	const efx_filter_ops_t *efop = enp->en_efop;
 
 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 	EFSYS_ASSERT3P(spec, !=, NULL);
 	EFSYS_ASSERT3U(spec->efs_flags, &, EFX_FILTER_FLAG_RX);
 
 	return (efop->efo_add(enp, spec, B_FALSE));
 }
 
 	__checkReturn	efx_rc_t
 efx_filter_remove(
 	__in		efx_nic_t *enp,
 	__inout		efx_filter_spec_t *spec)
 {
 	const efx_filter_ops_t *efop = enp->en_efop;
 
 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 	EFSYS_ASSERT3P(spec, !=, NULL);
 	EFSYS_ASSERT3U(spec->efs_flags, &, EFX_FILTER_FLAG_RX);
 
 #if EFSYS_OPT_RX_SCALE
 	spec->efs_rss_context = enp->en_rss_context;
 #endif
 
 	return (efop->efo_delete(enp, spec));
 }
 
 	__checkReturn	efx_rc_t
 efx_filter_restore(
 	__in		efx_nic_t *enp)
 {
 	efx_rc_t rc;
 
 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 
 	if ((rc = enp->en_efop->efo_restore(enp)) != 0)
 		goto fail1;
 
 	return (0);
 
 fail1:
 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
 
 	return (rc);
 }
 
 	__checkReturn	efx_rc_t
 efx_filter_init(
 	__in		efx_nic_t *enp)
 {
 	const efx_filter_ops_t *efop;
 	efx_rc_t rc;
 
 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_FILTER));
 
 	switch (enp->en_family) {
 #if EFSYS_OPT_SIENA
 	case EFX_FAMILY_SIENA:
 		efop = &__efx_filter_siena_ops;
 		break;
 #endif /* EFSYS_OPT_SIENA */
 
 #if EFSYS_OPT_HUNTINGTON
 	case EFX_FAMILY_HUNTINGTON:
 		efop = &__efx_filter_ef10_ops;
 		break;
 #endif /* EFSYS_OPT_HUNTINGTON */
 
 #if EFSYS_OPT_MEDFORD
 	case EFX_FAMILY_MEDFORD:
 		efop = &__efx_filter_ef10_ops;
 		break;
 #endif /* EFSYS_OPT_MEDFORD */
 
 	default:
 		EFSYS_ASSERT(0);
 		rc = ENOTSUP;
 		goto fail1;
 	}
 
 	if ((rc = efop->efo_init(enp)) != 0)
 		goto fail2;
 
 	enp->en_efop = efop;
 	enp->en_mod_flags |= EFX_MOD_FILTER;
 	return (0);
 
 fail2:
 	EFSYS_PROBE(fail2);
 fail1:
 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
 
 	enp->en_efop = NULL;
 	enp->en_mod_flags &= ~EFX_MOD_FILTER;
 	return (rc);
 }
 
 			void
 efx_filter_fini(
 	__in		efx_nic_t *enp)
 {
 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 
 	enp->en_efop->efo_fini(enp);
 
 	enp->en_efop = NULL;
 	enp->en_mod_flags &= ~EFX_MOD_FILTER;
 }
 
 /*
  * Query the possible combinations of match flags which can be filtered on.
  * These are returned as a list, of which each 32 bit element is a bitmask
  * formed of EFX_FILTER_MATCH flags.
  *
  * The combinations are ordered in priority from highest to lowest.
  *
  * If the provided buffer is too short to hold the list, the call with fail with
  * ENOSPC and *list_lengthp will be set to the buffer length required.
  */
 	__checkReturn	efx_rc_t
 efx_filter_supported_filters(
 	__in				efx_nic_t *enp,
 	__out_ecount(buffer_length)	uint32_t *buffer,
 	__in				size_t buffer_length,
 	__out				size_t *list_lengthp)
 {
 	efx_rc_t rc;
 
 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 	EFSYS_ASSERT(enp->en_efop->efo_supported_filters != NULL);
 
 	if (buffer == NULL) {
 		rc = EINVAL;
 		goto fail1;
 	}
 
 	rc = enp->en_efop->efo_supported_filters(enp, buffer, buffer_length,
 						    list_lengthp);
 	if (rc != 0)
 		goto fail2;
 
 	return (0);
 
 fail2:
 	EFSYS_PROBE(fail2);
 fail1:
 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
 
 	return (rc);
 }
 
 	__checkReturn	efx_rc_t
 efx_filter_reconfigure(
 	__in				efx_nic_t *enp,
 	__in_ecount(6)			uint8_t const *mac_addr,
 	__in				boolean_t all_unicst,
 	__in				boolean_t mulcst,
 	__in				boolean_t all_mulcst,
 	__in				boolean_t brdcst,
 	__in_ecount(6*count)		uint8_t const *addrs,
 	__in				uint32_t count)
 {
 	efx_rc_t rc;
 
 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 
 	if (enp->en_efop->efo_reconfigure != NULL) {
 		if ((rc = enp->en_efop->efo_reconfigure(enp, mac_addr,
 							all_unicst, mulcst,
 							all_mulcst, brdcst,
 							addrs, count)) != 0)
 			goto fail1;
 	}
 
 	return (0);
 
 fail1:
 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
 
 	return (rc);
 }
 
 		void
 efx_filter_spec_init_rx(
 	__out		efx_filter_spec_t *spec,
 	__in		efx_filter_priority_t priority,
 	__in		efx_filter_flags_t flags,
 	__in		efx_rxq_t *erp)
 {
 	EFSYS_ASSERT3P(spec, !=, NULL);
 	EFSYS_ASSERT3P(erp, !=, NULL);
 	EFSYS_ASSERT((flags & ~(EFX_FILTER_FLAG_RX_RSS |
 				EFX_FILTER_FLAG_RX_SCATTER)) == 0);
 
 	memset(spec, 0, sizeof (*spec));
 	spec->efs_priority = priority;
 	spec->efs_flags = EFX_FILTER_FLAG_RX | flags;
 	spec->efs_rss_context = EFX_RSS_CONTEXT_DEFAULT;
 	spec->efs_dmaq_id = (uint16_t)erp->er_index;
 }
 
 		void
 efx_filter_spec_init_tx(
 	__out		efx_filter_spec_t *spec,
 	__in		efx_txq_t *etp)
 {
 	EFSYS_ASSERT3P(spec, !=, NULL);
 	EFSYS_ASSERT3P(etp, !=, NULL);
 
 	memset(spec, 0, sizeof (*spec));
 	spec->efs_priority = EFX_FILTER_PRI_REQUIRED;
 	spec->efs_flags = EFX_FILTER_FLAG_TX;
 	spec->efs_dmaq_id = (uint16_t)etp->et_index;
 }
 
 
 /*
  *  Specify IPv4 host, transport protocol and port in a filter specification
  */
 __checkReturn		efx_rc_t
 efx_filter_spec_set_ipv4_local(
 	__inout		efx_filter_spec_t *spec,
 	__in		uint8_t proto,
 	__in		uint32_t host,
 	__in		uint16_t port)
 {
 	EFSYS_ASSERT3P(spec, !=, NULL);
 
 	spec->efs_match_flags |=
 		EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 		EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT;
 	spec->efs_ether_type = EFX_ETHER_TYPE_IPV4;
 	spec->efs_ip_proto = proto;
 	spec->efs_loc_host.eo_u32[0] = host;
 	spec->efs_loc_port = port;
 	return (0);
 }
 
 /*
  * Specify IPv4 hosts, transport protocol and ports in a filter specification
  */
 __checkReturn		efx_rc_t
 efx_filter_spec_set_ipv4_full(
 	__inout		efx_filter_spec_t *spec,
 	__in		uint8_t proto,
 	__in		uint32_t lhost,
 	__in		uint16_t lport,
 	__in		uint32_t rhost,
 	__in		uint16_t rport)
 {
 	EFSYS_ASSERT3P(spec, !=, NULL);
 
 	spec->efs_match_flags |=
 		EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 		EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |
 		EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT;
 	spec->efs_ether_type = EFX_ETHER_TYPE_IPV4;
 	spec->efs_ip_proto = proto;
 	spec->efs_loc_host.eo_u32[0] = lhost;
 	spec->efs_loc_port = lport;
 	spec->efs_rem_host.eo_u32[0] = rhost;
 	spec->efs_rem_port = rport;
 	return (0);
 }
 
 /*
  * Specify local Ethernet address and/or VID in filter specification
  */
 __checkReturn		efx_rc_t
 efx_filter_spec_set_eth_local(
 	__inout		efx_filter_spec_t *spec,
 	__in		uint16_t vid,
 	__in		const uint8_t *addr)
 {
 	EFSYS_ASSERT3P(spec, !=, NULL);
 	EFSYS_ASSERT3P(addr, !=, NULL);
 
 	if (vid == EFX_FILTER_SPEC_VID_UNSPEC && addr == NULL)
 		return (EINVAL);
 
 	if (vid != EFX_FILTER_SPEC_VID_UNSPEC) {
 		spec->efs_match_flags |= EFX_FILTER_MATCH_OUTER_VID;
 		spec->efs_outer_vid = vid;
 	}
 	if (addr != NULL) {
 		spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_MAC;
 		memcpy(spec->efs_loc_mac, addr, EFX_MAC_ADDR_LEN);
 	}
 	return (0);
 }
 
+			void
+efx_filter_spec_set_ether_type(
+	__inout		efx_filter_spec_t *spec,
+	__in		uint16_t ether_type)
+{
+	EFSYS_ASSERT3P(spec, !=, NULL);
+
+	spec->efs_ether_type = ether_type;
+	spec->efs_match_flags |= EFX_FILTER_MATCH_ETHER_TYPE;
+}
+
 /*
  * Specify matching otherwise-unmatched unicast in a filter specification
  */
 __checkReturn		efx_rc_t
 efx_filter_spec_set_uc_def(
 	__inout		efx_filter_spec_t *spec)
 {
 	EFSYS_ASSERT3P(spec, !=, NULL);
 
 	spec->efs_match_flags |= EFX_FILTER_MATCH_UNKNOWN_UCAST_DST;
 	return (0);
 }
 
 /*
  * Specify matching otherwise-unmatched multicast in a filter specification
  */
 __checkReturn		efx_rc_t
 efx_filter_spec_set_mc_def(
 	__inout		efx_filter_spec_t *spec)
 {
 	EFSYS_ASSERT3P(spec, !=, NULL);
 
 	spec->efs_match_flags |= EFX_FILTER_MATCH_UNKNOWN_MCAST_DST;
 	return (0);
+}
+
+
+__checkReturn		efx_rc_t
+efx_filter_spec_set_encap_type(
+	__inout		efx_filter_spec_t *spec,
+	__in		efx_tunnel_protocol_t encap_type,
+	__in		efx_filter_inner_frame_match_t inner_frame_match)
+{
+	uint32_t match_flags = 0;
+	uint8_t ip_proto;
+	efx_rc_t rc;
+
+	EFSYS_ASSERT3P(spec, !=, NULL);
+
+	switch (encap_type) {
+	case EFX_TUNNEL_PROTOCOL_VXLAN:
+	case EFX_TUNNEL_PROTOCOL_GENEVE:
+		ip_proto = EFX_IPPROTO_UDP;
+		break;
+	case EFX_TUNNEL_PROTOCOL_NVGRE:
+		ip_proto = EFX_IPPROTO_GRE;
+		break;
+	default:
+		EFSYS_ASSERT(0);
+		rc = EINVAL;
+		goto fail1;
+	}
+
+	switch (inner_frame_match) {
+	case EFX_FILTER_INNER_FRAME_MATCH_UNKNOWN_MCAST_DST:
+		match_flags |= EFX_FILTER_MATCH_IFRM_UNKNOWN_MCAST_DST;
+		break;
+	case EFX_FILTER_INNER_FRAME_MATCH_UNKNOWN_UCAST_DST:
+		match_flags |= EFX_FILTER_MATCH_IFRM_UNKNOWN_UCAST_DST;
+		break;
+	case EFX_FILTER_INNER_FRAME_MATCH_OTHER:
+		/* This is for when specific inner frames are to be matched. */
+		break;
+	default:
+		EFSYS_ASSERT(0);
+		rc = EINVAL;
+		goto fail2;
+	}
+
+	spec->efs_encap_type = encap_type;
+	spec->efs_ip_proto = ip_proto;
+	spec->efs_match_flags |= (match_flags | EFX_FILTER_MATCH_IP_PROTO);
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, efx_rc_t, rc);
+
+	return (rc);
 }
 
 
 
 #if EFSYS_OPT_SIENA
 
 /*
  * "Fudge factors" - difference between programmed value and actual depth.
  * Due to pipelined implementation we need to program H/W with a value that
  * is larger than the hop limit we want.
  */
 #define	FILTER_CTL_SRCH_FUDGE_WILD 3
 #define	FILTER_CTL_SRCH_FUDGE_FULL 1
 
 /*
  * Hard maximum hop limit.  Hardware will time-out beyond 200-something.
  * We also need to avoid infinite loops in efx_filter_search() when the
  * table is full.
  */
 #define	FILTER_CTL_SRCH_MAX 200
 
 static	__checkReturn	efx_rc_t
 siena_filter_spec_from_gen_spec(
 	__out		siena_filter_spec_t *sf_spec,
 	__in		efx_filter_spec_t *gen_spec)
 {
 	efx_rc_t rc;
 	boolean_t is_full = B_FALSE;
 
 	if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX)
 		EFSYS_ASSERT3U(gen_spec->efs_flags, ==, EFX_FILTER_FLAG_TX);
 	else
 		EFSYS_ASSERT3U(gen_spec->efs_flags, &, EFX_FILTER_FLAG_RX);
 
 	/* Siena only has one RSS context */
 	if ((gen_spec->efs_flags & EFX_FILTER_FLAG_RX_RSS) &&
 	    gen_spec->efs_rss_context != EFX_RSS_CONTEXT_DEFAULT) {
 		rc = EINVAL;
 		goto fail1;
 	}
 
 	sf_spec->sfs_flags = gen_spec->efs_flags;
 	sf_spec->sfs_dmaq_id = gen_spec->efs_dmaq_id;
 
 	switch (gen_spec->efs_match_flags) {
 	case EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 	    EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |
 	    EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT:
 		is_full = B_TRUE;
 		/* Fall through */
 	case EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 	    EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT: {
 		uint32_t rhost, host1, host2;
 		uint16_t rport, port1, port2;
 
 		if (gen_spec->efs_ether_type != EFX_ETHER_TYPE_IPV4) {
 			rc = ENOTSUP;
 			goto fail2;
 		}
 		if (gen_spec->efs_loc_port == 0 ||
 		    (is_full && gen_spec->efs_rem_port == 0)) {
 			rc = EINVAL;
 			goto fail3;
 		}
 		switch (gen_spec->efs_ip_proto) {
 		case EFX_IPPROTO_TCP:
 			if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 				sf_spec->sfs_type = (is_full ?
 				    EFX_SIENA_FILTER_TX_TCP_FULL :
 				    EFX_SIENA_FILTER_TX_TCP_WILD);
 			} else {
 				sf_spec->sfs_type = (is_full ?
 				    EFX_SIENA_FILTER_RX_TCP_FULL :
 				    EFX_SIENA_FILTER_RX_TCP_WILD);
 			}
 			break;
 		case EFX_IPPROTO_UDP:
 			if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 				sf_spec->sfs_type = (is_full ?
 				    EFX_SIENA_FILTER_TX_UDP_FULL :
 				    EFX_SIENA_FILTER_TX_UDP_WILD);
 			} else {
 				sf_spec->sfs_type = (is_full ?
 				    EFX_SIENA_FILTER_RX_UDP_FULL :
 				    EFX_SIENA_FILTER_RX_UDP_WILD);
 			}
 			break;
 		default:
 			rc = ENOTSUP;
 			goto fail4;
 		}
 		/*
 		 * The filter is constructed in terms of source and destination,
 		 * with the odd wrinkle that the ports are swapped in a UDP
 		 * wildcard filter. We need to convert from local and remote
 		 * addresses (zero for a wildcard).
 		 */
 		rhost = is_full ? gen_spec->efs_rem_host.eo_u32[0] : 0;
 		rport = is_full ? gen_spec->efs_rem_port : 0;
 		if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 			host1 = gen_spec->efs_loc_host.eo_u32[0];
 			host2 = rhost;
 		} else {
 			host1 = rhost;
 			host2 = gen_spec->efs_loc_host.eo_u32[0];
 		}
 		if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 			if (sf_spec->sfs_type ==
 			    EFX_SIENA_FILTER_TX_UDP_WILD) {
 				port1 = rport;
 				port2 = gen_spec->efs_loc_port;
 			} else {
 				port1 = gen_spec->efs_loc_port;
 				port2 = rport;
 			}
 		} else {
 			if (sf_spec->sfs_type ==
 			    EFX_SIENA_FILTER_RX_UDP_WILD) {
 				port1 = gen_spec->efs_loc_port;
 				port2 = rport;
 			} else {
 				port1 = rport;
 				port2 = gen_spec->efs_loc_port;
 			}
 		}
 		sf_spec->sfs_dword[0] = (host1 << 16) | port1;
 		sf_spec->sfs_dword[1] = (port2 << 16) | (host1 >> 16);
 		sf_spec->sfs_dword[2] = host2;
 		break;
 	}
 
 	case EFX_FILTER_MATCH_LOC_MAC | EFX_FILTER_MATCH_OUTER_VID:
 		is_full = B_TRUE;
 		/* Fall through */
 	case EFX_FILTER_MATCH_LOC_MAC:
 		if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 			sf_spec->sfs_type = (is_full ?
 			    EFX_SIENA_FILTER_TX_MAC_FULL :
 			    EFX_SIENA_FILTER_TX_MAC_WILD);
 		} else {
 			sf_spec->sfs_type = (is_full ?
 			    EFX_SIENA_FILTER_RX_MAC_FULL :
 			    EFX_SIENA_FILTER_RX_MAC_WILD);
 		}
 		sf_spec->sfs_dword[0] = is_full ? gen_spec->efs_outer_vid : 0;
 		sf_spec->sfs_dword[1] =
 		    gen_spec->efs_loc_mac[2] << 24 |
 		    gen_spec->efs_loc_mac[3] << 16 |
 		    gen_spec->efs_loc_mac[4] <<  8 |
 		    gen_spec->efs_loc_mac[5];
 		sf_spec->sfs_dword[2] =
 		    gen_spec->efs_loc_mac[0] << 8 |
 		    gen_spec->efs_loc_mac[1];
 		break;
 
 	default:
 		EFSYS_ASSERT(B_FALSE);
 		rc = ENOTSUP;
 		goto fail5;
 	}
 
 	return (0);
 
 fail5:
 	EFSYS_PROBE(fail5);
 fail4:
 	EFSYS_PROBE(fail4);
 fail3:
 	EFSYS_PROBE(fail3);
 fail2:
 	EFSYS_PROBE(fail2);
 fail1:
 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
 
 	return (rc);
 }
 
 /*
  * The filter hash function is LFSR polynomial x^16 + x^3 + 1 of a 32-bit
  * key derived from the n-tuple.
  */
 static			uint16_t
 siena_filter_tbl_hash(
 	__in		uint32_t key)
 {
 	uint16_t tmp;
 
 	/* First 16 rounds */
 	tmp = 0x1fff ^ (uint16_t)(key >> 16);
 	tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
 	tmp = tmp ^ tmp >> 9;
 
 	/* Last 16 rounds */
 	tmp = tmp ^ tmp << 13 ^ (uint16_t)(key & 0xffff);
 	tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
 	tmp = tmp ^ tmp >> 9;
 
 	return (tmp);
 }
 
 /*
  * To allow for hash collisions, filter search continues at these
  * increments from the first possible entry selected by the hash.
  */
 static			uint16_t
 siena_filter_tbl_increment(
 	__in		uint32_t key)
 {
 	return ((uint16_t)(key * 2 - 1));
 }
 
 static	__checkReturn	boolean_t
 siena_filter_test_used(
 	__in		siena_filter_tbl_t *sftp,
 	__in		unsigned int index)
 {
 	EFSYS_ASSERT3P(sftp->sft_bitmap, !=, NULL);
 	return ((sftp->sft_bitmap[index / 32] & (1 << (index % 32))) != 0);
 }
 
 static			void
 siena_filter_set_used(
 	__in		siena_filter_tbl_t *sftp,
 	__in		unsigned int index)
 {
 	EFSYS_ASSERT3P(sftp->sft_bitmap, !=, NULL);
 	sftp->sft_bitmap[index / 32] |= (1 << (index % 32));
 	++sftp->sft_used;
 }
 
 static			void
 siena_filter_clear_used(
 	__in		siena_filter_tbl_t *sftp,
 	__in		unsigned int index)
 {
 	EFSYS_ASSERT3P(sftp->sft_bitmap, !=, NULL);
 	sftp->sft_bitmap[index / 32] &= ~(1 << (index % 32));
 
 	--sftp->sft_used;
 	EFSYS_ASSERT3U(sftp->sft_used, >=, 0);
 }
 
 
 static			siena_filter_tbl_id_t
 siena_filter_tbl_id(
 	__in		siena_filter_type_t type)
 {
 	siena_filter_tbl_id_t tbl_id;
 
 	switch (type) {
 	case EFX_SIENA_FILTER_RX_TCP_FULL:
 	case EFX_SIENA_FILTER_RX_TCP_WILD:
 	case EFX_SIENA_FILTER_RX_UDP_FULL:
 	case EFX_SIENA_FILTER_RX_UDP_WILD:
 		tbl_id = EFX_SIENA_FILTER_TBL_RX_IP;
 		break;
 
 	case EFX_SIENA_FILTER_RX_MAC_FULL:
 	case EFX_SIENA_FILTER_RX_MAC_WILD:
 		tbl_id = EFX_SIENA_FILTER_TBL_RX_MAC;
 		break;
 
 	case EFX_SIENA_FILTER_TX_TCP_FULL:
 	case EFX_SIENA_FILTER_TX_TCP_WILD:
 	case EFX_SIENA_FILTER_TX_UDP_FULL:
 	case EFX_SIENA_FILTER_TX_UDP_WILD:
 		tbl_id = EFX_SIENA_FILTER_TBL_TX_IP;
 		break;
 
 	case EFX_SIENA_FILTER_TX_MAC_FULL:
 	case EFX_SIENA_FILTER_TX_MAC_WILD:
 		tbl_id = EFX_SIENA_FILTER_TBL_TX_MAC;
 		break;
 
 	default:
 		EFSYS_ASSERT(B_FALSE);
 		tbl_id = EFX_SIENA_FILTER_NTBLS;
 		break;
 	}
 	return (tbl_id);
 }
 
 static			void
 siena_filter_reset_search_depth(
 	__inout		siena_filter_t *sfp,
 	__in		siena_filter_tbl_id_t tbl_id)
 {
 	switch (tbl_id) {
 	case EFX_SIENA_FILTER_TBL_RX_IP:
 		sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_FULL] = 0;
 		sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_WILD] = 0;
 		sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_FULL] = 0;
 		sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_WILD] = 0;
 		break;
 
 	case EFX_SIENA_FILTER_TBL_RX_MAC:
 		sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_FULL] = 0;
 		sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_WILD] = 0;
 		break;
 
 	case EFX_SIENA_FILTER_TBL_TX_IP:
 		sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_FULL] = 0;
 		sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_WILD] = 0;
 		sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_FULL] = 0;
 		sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_WILD] = 0;
 		break;
 
 	case EFX_SIENA_FILTER_TBL_TX_MAC:
 		sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_FULL] = 0;
 		sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_WILD] = 0;
 		break;
 
 	default:
 		EFSYS_ASSERT(B_FALSE);
 		break;
 	}
 }
 
 static			void
 siena_filter_push_rx_limits(
 	__in		efx_nic_t *enp)
 {
 	siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
 	efx_oword_t oword;
 
 	EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 
 	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TCP_FULL_SRCH_LIMIT,
 	    sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_FULL] +
 	    FILTER_CTL_SRCH_FUDGE_FULL);
 	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TCP_WILD_SRCH_LIMIT,
 	    sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_WILD] +
 	    FILTER_CTL_SRCH_FUDGE_WILD);
 	EFX_SET_OWORD_FIELD(oword, FRF_AZ_UDP_FULL_SRCH_LIMIT,
 	    sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_FULL] +
 	    FILTER_CTL_SRCH_FUDGE_FULL);
 	EFX_SET_OWORD_FIELD(oword, FRF_AZ_UDP_WILD_SRCH_LIMIT,
 	    sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_WILD] +
 	    FILTER_CTL_SRCH_FUDGE_WILD);
 
 	if (sfp->sf_tbl[EFX_SIENA_FILTER_TBL_RX_MAC].sft_size) {
 		EFX_SET_OWORD_FIELD(oword,
 		    FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT,
 		    sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_FULL] +
 		    FILTER_CTL_SRCH_FUDGE_FULL);
 		EFX_SET_OWORD_FIELD(oword,
 		    FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT,
 		    sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_WILD] +
 		    FILTER_CTL_SRCH_FUDGE_WILD);
 	}
 
 	EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 }
 
 static			void
 siena_filter_push_tx_limits(
 	__in		efx_nic_t *enp)
 {
 	siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
 	efx_oword_t oword;
 
 	EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
 
 	if (sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_IP].sft_size != 0) {
 		EFX_SET_OWORD_FIELD(oword,
 		    FRF_CZ_TX_TCPIP_FILTER_FULL_SEARCH_RANGE,
 		    sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_FULL] +
 		    FILTER_CTL_SRCH_FUDGE_FULL);
 		EFX_SET_OWORD_FIELD(oword,
 		    FRF_CZ_TX_TCPIP_FILTER_WILD_SEARCH_RANGE,
 		    sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_WILD] +
 		    FILTER_CTL_SRCH_FUDGE_WILD);
 		EFX_SET_OWORD_FIELD(oword,
 		    FRF_CZ_TX_UDPIP_FILTER_FULL_SEARCH_RANGE,
 		    sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_FULL] +
 		    FILTER_CTL_SRCH_FUDGE_FULL);
 		EFX_SET_OWORD_FIELD(oword,
 		    FRF_CZ_TX_UDPIP_FILTER_WILD_SEARCH_RANGE,
 		    sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_WILD] +
 		    FILTER_CTL_SRCH_FUDGE_WILD);
 	}
 
 	if (sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_MAC].sft_size != 0) {
 		EFX_SET_OWORD_FIELD(
 			oword, FRF_CZ_TX_ETH_FILTER_FULL_SEARCH_RANGE,
 			sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_FULL] +
 			FILTER_CTL_SRCH_FUDGE_FULL);
 		EFX_SET_OWORD_FIELD(
 			oword, FRF_CZ_TX_ETH_FILTER_WILD_SEARCH_RANGE,
 			sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_WILD] +
 			FILTER_CTL_SRCH_FUDGE_WILD);
 	}
 
 	EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
 }
 
 /* Build a filter entry and return its n-tuple key. */
 static	__checkReturn	uint32_t
 siena_filter_build(
 	__out		efx_oword_t *filter,
 	__in		siena_filter_spec_t *spec)
 {
 	uint32_t dword3;
 	uint32_t key;
 	uint8_t  type  = spec->sfs_type;
 	uint32_t flags = spec->sfs_flags;
 
 	switch (siena_filter_tbl_id(type)) {
 	case EFX_SIENA_FILTER_TBL_RX_IP: {
 		boolean_t is_udp = (type == EFX_SIENA_FILTER_RX_UDP_FULL ||
 		    type == EFX_SIENA_FILTER_RX_UDP_WILD);
 		EFX_POPULATE_OWORD_7(*filter,
 		    FRF_BZ_RSS_EN,
 		    (flags & EFX_FILTER_FLAG_RX_RSS) ? 1 : 0,
 		    FRF_BZ_SCATTER_EN,
 		    (flags & EFX_FILTER_FLAG_RX_SCATTER) ? 1 : 0,
 		    FRF_AZ_TCP_UDP, is_udp,
 		    FRF_AZ_RXQ_ID, spec->sfs_dmaq_id,
 		    EFX_DWORD_2, spec->sfs_dword[2],
 		    EFX_DWORD_1, spec->sfs_dword[1],
 		    EFX_DWORD_0, spec->sfs_dword[0]);
 		dword3 = is_udp;
 		break;
 	}
 
 	case EFX_SIENA_FILTER_TBL_RX_MAC: {
 		boolean_t is_wild = (type == EFX_SIENA_FILTER_RX_MAC_WILD);
 		EFX_POPULATE_OWORD_7(*filter,
 		    FRF_CZ_RMFT_RSS_EN,
 		    (flags & EFX_FILTER_FLAG_RX_RSS) ? 1 : 0,
 		    FRF_CZ_RMFT_SCATTER_EN,
 		    (flags & EFX_FILTER_FLAG_RX_SCATTER) ? 1 : 0,
 		    FRF_CZ_RMFT_RXQ_ID, spec->sfs_dmaq_id,
 		    FRF_CZ_RMFT_WILDCARD_MATCH, is_wild,
 		    FRF_CZ_RMFT_DEST_MAC_DW1, spec->sfs_dword[2],
 		    FRF_CZ_RMFT_DEST_MAC_DW0, spec->sfs_dword[1],
 		    FRF_CZ_RMFT_VLAN_ID, spec->sfs_dword[0]);
 		dword3 = is_wild;
 		break;
 	}
 
 	case EFX_SIENA_FILTER_TBL_TX_IP: {
 		boolean_t is_udp = (type == EFX_SIENA_FILTER_TX_UDP_FULL ||
 		    type == EFX_SIENA_FILTER_TX_UDP_WILD);
 		EFX_POPULATE_OWORD_5(*filter,
 		    FRF_CZ_TIFT_TCP_UDP, is_udp,
 		    FRF_CZ_TIFT_TXQ_ID, spec->sfs_dmaq_id,
 		    EFX_DWORD_2, spec->sfs_dword[2],
 		    EFX_DWORD_1, spec->sfs_dword[1],
 		    EFX_DWORD_0, spec->sfs_dword[0]);
 		dword3 = is_udp | spec->sfs_dmaq_id << 1;
 		break;
 	}
 
 	case EFX_SIENA_FILTER_TBL_TX_MAC: {
 		boolean_t is_wild = (type == EFX_SIENA_FILTER_TX_MAC_WILD);
 		EFX_POPULATE_OWORD_5(*filter,
 		    FRF_CZ_TMFT_TXQ_ID, spec->sfs_dmaq_id,
 		    FRF_CZ_TMFT_WILDCARD_MATCH, is_wild,
 		    FRF_CZ_TMFT_SRC_MAC_DW1, spec->sfs_dword[2],
 		    FRF_CZ_TMFT_SRC_MAC_DW0, spec->sfs_dword[1],
 		    FRF_CZ_TMFT_VLAN_ID, spec->sfs_dword[0]);
 		dword3 = is_wild | spec->sfs_dmaq_id << 1;
 		break;
 	}
 
 	default:
 		EFSYS_ASSERT(B_FALSE);
 		return (0);
 	}
 
 	key =
 	    spec->sfs_dword[0] ^
 	    spec->sfs_dword[1] ^
 	    spec->sfs_dword[2] ^
 	    dword3;
 
 	return (key);
 }
 
 static	__checkReturn		efx_rc_t
 siena_filter_push_entry(
 	__inout			efx_nic_t *enp,
 	__in			siena_filter_type_t type,
 	__in			int index,
 	__in			efx_oword_t *eop)
 {
 	efx_rc_t rc;
 
 	switch (type) {
 	case EFX_SIENA_FILTER_RX_TCP_FULL:
 	case EFX_SIENA_FILTER_RX_TCP_WILD:
 	case EFX_SIENA_FILTER_RX_UDP_FULL:
 	case EFX_SIENA_FILTER_RX_UDP_WILD:
 		EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_FILTER_TBL0, index,
 		    eop, B_TRUE);
 		break;
 
 	case EFX_SIENA_FILTER_RX_MAC_FULL:
 	case EFX_SIENA_FILTER_RX_MAC_WILD:
 		EFX_BAR_TBL_WRITEO(enp, FR_CZ_RX_MAC_FILTER_TBL0, index,
 		    eop, B_TRUE);
 		break;
 
 	case EFX_SIENA_FILTER_TX_TCP_FULL:
 	case EFX_SIENA_FILTER_TX_TCP_WILD:
 	case EFX_SIENA_FILTER_TX_UDP_FULL:
 	case EFX_SIENA_FILTER_TX_UDP_WILD:
 		EFX_BAR_TBL_WRITEO(enp, FR_CZ_TX_FILTER_TBL0, index,
 		    eop, B_TRUE);
 		break;
 
 	case EFX_SIENA_FILTER_TX_MAC_FULL:
 	case EFX_SIENA_FILTER_TX_MAC_WILD:
 		EFX_BAR_TBL_WRITEO(enp, FR_CZ_TX_MAC_FILTER_TBL0, index,
 		    eop, B_TRUE);
 		break;
 
 	default:
 		EFSYS_ASSERT(B_FALSE);
 		rc = ENOTSUP;
 		goto fail1;
 	}
 	return (0);
 
 fail1:
 	return (rc);
 }
 
 
 static	__checkReturn	boolean_t
 siena_filter_equal(
 	__in		const siena_filter_spec_t *left,
 	__in		const siena_filter_spec_t *right)
 {
 	siena_filter_tbl_id_t tbl_id;
 
 	tbl_id = siena_filter_tbl_id(left->sfs_type);
 
 
 	if (left->sfs_type != right->sfs_type)
 		return (B_FALSE);
 
 	if (memcmp(left->sfs_dword, right->sfs_dword,
 		sizeof (left->sfs_dword)))
 		return (B_FALSE);
 
 	if ((tbl_id == EFX_SIENA_FILTER_TBL_TX_IP ||
 		tbl_id == EFX_SIENA_FILTER_TBL_TX_MAC) &&
 	    left->sfs_dmaq_id != right->sfs_dmaq_id)
 		return (B_FALSE);
 
 	return (B_TRUE);
 }
 
 static	__checkReturn	efx_rc_t
 siena_filter_search(
 	__in		siena_filter_tbl_t *sftp,
 	__in		siena_filter_spec_t *spec,
 	__in		uint32_t key,
 	__in		boolean_t for_insert,
 	__out		int *filter_index,
 	__out		unsigned int *depth_required)
 {
 	unsigned int hash, incr, filter_idx, depth;
 
 	hash = siena_filter_tbl_hash(key);
 	incr = siena_filter_tbl_increment(key);
 
 	filter_idx = hash & (sftp->sft_size - 1);
 	depth = 1;
 
 	for (;;) {
 		/*
 		 * Return success if entry is used and matches this spec
 		 * or entry is unused and we are trying to insert.
 		 */
 		if (siena_filter_test_used(sftp, filter_idx) ?
 		    siena_filter_equal(spec,
 		    &sftp->sft_spec[filter_idx]) :
 		    for_insert) {
 			*filter_index = filter_idx;
 			*depth_required = depth;
 			return (0);
 		}
 
 		/* Return failure if we reached the maximum search depth */
 		if (depth == FILTER_CTL_SRCH_MAX)
 			return (for_insert ? EBUSY : ENOENT);
 
 		filter_idx = (filter_idx + incr) & (sftp->sft_size - 1);
 		++depth;
 	}
 }
 
 static			void
 siena_filter_clear_entry(
 	__in		efx_nic_t *enp,
 	__in		siena_filter_tbl_t *sftp,
 	__in		int index)
 {
 	efx_oword_t filter;
 
 	if (siena_filter_test_used(sftp, index)) {
 		siena_filter_clear_used(sftp, index);
 
 		EFX_ZERO_OWORD(filter);
 		siena_filter_push_entry(enp,
 		    sftp->sft_spec[index].sfs_type,
 		    index, &filter);
 
 		memset(&sftp->sft_spec[index],
 		    0, sizeof (sftp->sft_spec[0]));
 	}
 }
 
 			void
 siena_filter_tbl_clear(
 	__in		efx_nic_t *enp,
 	__in		siena_filter_tbl_id_t tbl_id)
 {
 	siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
 	siena_filter_tbl_t *sftp = &sfp->sf_tbl[tbl_id];
 	int index;
 	efsys_lock_state_t state;
 
 	EFSYS_LOCK(enp->en_eslp, state);
 
 	for (index = 0; index < sftp->sft_size; ++index) {
 		siena_filter_clear_entry(enp, sftp, index);
 	}
 
 	if (sftp->sft_used == 0)
 		siena_filter_reset_search_depth(sfp, tbl_id);
 
 	EFSYS_UNLOCK(enp->en_eslp, state);
 }
 
 static	__checkReturn	efx_rc_t
 siena_filter_init(
 	__in		efx_nic_t *enp)
 {
 	siena_filter_t *sfp;
 	siena_filter_tbl_t *sftp;
 	int tbl_id;
 	efx_rc_t rc;
 
 	EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (siena_filter_t), sfp);
 
 	if (!sfp) {
 		rc = ENOMEM;
 		goto fail1;
 	}
 
 	enp->en_filter.ef_siena_filter = sfp;
 
 	switch (enp->en_family) {
 	case EFX_FAMILY_SIENA:
 		sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_RX_IP];
 		sftp->sft_size = FR_AZ_RX_FILTER_TBL0_ROWS;
 
 		sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_RX_MAC];
 		sftp->sft_size = FR_CZ_RX_MAC_FILTER_TBL0_ROWS;
 
 		sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_IP];
 		sftp->sft_size = FR_CZ_TX_FILTER_TBL0_ROWS;
 
 		sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_MAC];
 		sftp->sft_size = FR_CZ_TX_MAC_FILTER_TBL0_ROWS;
 		break;
 
 	default:
 		rc = ENOTSUP;
 		goto fail2;
 	}
 
 	for (tbl_id = 0; tbl_id < EFX_SIENA_FILTER_NTBLS; tbl_id++) {
 		unsigned int bitmap_size;
 
 		sftp = &sfp->sf_tbl[tbl_id];
 		if (sftp->sft_size == 0)
 			continue;
 
 		EFX_STATIC_ASSERT(sizeof (sftp->sft_bitmap[0]) ==
 		    sizeof (uint32_t));
 		bitmap_size =
 		    (sftp->sft_size + (sizeof (uint32_t) * 8) - 1) / 8;
 
 		EFSYS_KMEM_ALLOC(enp->en_esip, bitmap_size, sftp->sft_bitmap);
 		if (!sftp->sft_bitmap) {
 			rc = ENOMEM;
 			goto fail3;
 		}
 
 		EFSYS_KMEM_ALLOC(enp->en_esip,
 		    sftp->sft_size * sizeof (*sftp->sft_spec),
 		    sftp->sft_spec);
 		if (!sftp->sft_spec) {
 			rc = ENOMEM;
 			goto fail4;
 		}
 		memset(sftp->sft_spec, 0,
 		    sftp->sft_size * sizeof (*sftp->sft_spec));
 	}
 
 	return (0);
 
 fail4:
 	EFSYS_PROBE(fail4);
 
 fail3:
 	EFSYS_PROBE(fail3);
 
 fail2:
 	EFSYS_PROBE(fail2);
 	siena_filter_fini(enp);
 
 fail1:
 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
 	return (rc);
 }
 
 static			void
 siena_filter_fini(
 	__in		efx_nic_t *enp)
 {
 	siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
 	siena_filter_tbl_id_t tbl_id;
 
 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 
 	if (sfp == NULL)
 		return;
 
 	for (tbl_id = 0; tbl_id < EFX_SIENA_FILTER_NTBLS; tbl_id++) {
 		siena_filter_tbl_t *sftp = &sfp->sf_tbl[tbl_id];
 		unsigned int bitmap_size;
 
 		EFX_STATIC_ASSERT(sizeof (sftp->sft_bitmap[0]) ==
 		    sizeof (uint32_t));
 		bitmap_size =
 		    (sftp->sft_size + (sizeof (uint32_t) * 8) - 1) / 8;
 
 		if (sftp->sft_bitmap != NULL) {
 			EFSYS_KMEM_FREE(enp->en_esip, bitmap_size,
 			    sftp->sft_bitmap);
 			sftp->sft_bitmap = NULL;
 		}
 
 		if (sftp->sft_spec != NULL) {
 			EFSYS_KMEM_FREE(enp->en_esip, sftp->sft_size *
 			    sizeof (*sftp->sft_spec), sftp->sft_spec);
 			sftp->sft_spec = NULL;
 		}
 	}
 
 	EFSYS_KMEM_FREE(enp->en_esip, sizeof (siena_filter_t),
 	    enp->en_filter.ef_siena_filter);
 }
 
 /* Restore filter state after a reset */
 static	__checkReturn	efx_rc_t
 siena_filter_restore(
 	__in		efx_nic_t *enp)
 {
 	siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
 	siena_filter_tbl_id_t tbl_id;
 	siena_filter_tbl_t *sftp;
 	siena_filter_spec_t *spec;
 	efx_oword_t filter;
 	int filter_idx;
 	efsys_lock_state_t state;
 	uint32_t key;
 	efx_rc_t rc;
 
 	EFSYS_LOCK(enp->en_eslp, state);
 
 	for (tbl_id = 0; tbl_id < EFX_SIENA_FILTER_NTBLS; tbl_id++) {
 		sftp = &sfp->sf_tbl[tbl_id];
 		for (filter_idx = 0;
 			filter_idx < sftp->sft_size;
 			filter_idx++) {
 			if (!siena_filter_test_used(sftp, filter_idx))
 				continue;
 
 			spec = &sftp->sft_spec[filter_idx];
 			if ((key = siena_filter_build(&filter, spec)) == 0) {
 				rc = EINVAL;
 				goto fail1;
 			}
 			if ((rc = siena_filter_push_entry(enp,
 				    spec->sfs_type, filter_idx, &filter)) != 0)
 				goto fail2;
 		}
 	}
 
 	siena_filter_push_rx_limits(enp);
 	siena_filter_push_tx_limits(enp);
 
 	EFSYS_UNLOCK(enp->en_eslp, state);
 
 	return (0);
 
 fail2:
 	EFSYS_PROBE(fail2);
 
 fail1:
 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
 
 	EFSYS_UNLOCK(enp->en_eslp, state);
 
 	return (rc);
 }
 
 static	 __checkReturn	efx_rc_t
 siena_filter_add(
 	__in		efx_nic_t *enp,
 	__inout		efx_filter_spec_t *spec,
 	__in		boolean_t may_replace)
 {
 	efx_rc_t rc;
 	siena_filter_spec_t sf_spec;
 	siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
 	siena_filter_tbl_id_t tbl_id;
 	siena_filter_tbl_t *sftp;
 	siena_filter_spec_t *saved_sf_spec;
 	efx_oword_t filter;
 	int filter_idx;
 	unsigned int depth;
 	efsys_lock_state_t state;
 	uint32_t key;
 
 
 	EFSYS_ASSERT3P(spec, !=, NULL);
 
 	if ((rc = siena_filter_spec_from_gen_spec(&sf_spec, spec)) != 0)
 		goto fail1;
 
 	tbl_id = siena_filter_tbl_id(sf_spec.sfs_type);
 	sftp = &sfp->sf_tbl[tbl_id];
 
 	if (sftp->sft_size == 0) {
 		rc = EINVAL;
 		goto fail2;
 	}
 
 	key = siena_filter_build(&filter, &sf_spec);
 
 	EFSYS_LOCK(enp->en_eslp, state);
 
 	rc = siena_filter_search(sftp, &sf_spec, key, B_TRUE,
 	    &filter_idx, &depth);
 	if (rc != 0)
 		goto fail3;
 
 	EFSYS_ASSERT3U(filter_idx, <, sftp->sft_size);
 	saved_sf_spec = &sftp->sft_spec[filter_idx];
 
 	if (siena_filter_test_used(sftp, filter_idx)) {
 		if (may_replace == B_FALSE) {
 			rc = EEXIST;
 			goto fail4;
 		}
 	}
 	siena_filter_set_used(sftp, filter_idx);
 	*saved_sf_spec = sf_spec;
 
 	if (sfp->sf_depth[sf_spec.sfs_type] < depth) {
 		sfp->sf_depth[sf_spec.sfs_type] = depth;
 		if (tbl_id == EFX_SIENA_FILTER_TBL_TX_IP ||
 		    tbl_id == EFX_SIENA_FILTER_TBL_TX_MAC)
 			siena_filter_push_tx_limits(enp);
 		else
 			siena_filter_push_rx_limits(enp);
 	}
 
 	siena_filter_push_entry(enp, sf_spec.sfs_type,
 	    filter_idx, &filter);
 
 	EFSYS_UNLOCK(enp->en_eslp, state);
 	return (0);
 
 fail4:
 	EFSYS_PROBE(fail4);
 
 fail3:
 	EFSYS_UNLOCK(enp->en_eslp, state);
 	EFSYS_PROBE(fail3);
 
 fail2:
 	EFSYS_PROBE(fail2);
 
 fail1:
 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
 	return (rc);
 }
 
 static	 __checkReturn	efx_rc_t
 siena_filter_delete(
 	__in		efx_nic_t *enp,
 	__inout		efx_filter_spec_t *spec)
 {
 	efx_rc_t rc;
 	siena_filter_spec_t sf_spec;
 	siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
 	siena_filter_tbl_id_t tbl_id;
 	siena_filter_tbl_t *sftp;
 	efx_oword_t filter;
 	int filter_idx;
 	unsigned int depth;
 	efsys_lock_state_t state;
 	uint32_t key;
 
 	EFSYS_ASSERT3P(spec, !=, NULL);
 
 	if ((rc = siena_filter_spec_from_gen_spec(&sf_spec, spec)) != 0)
 		goto fail1;
 
 	tbl_id = siena_filter_tbl_id(sf_spec.sfs_type);
 	sftp = &sfp->sf_tbl[tbl_id];
 
 	key = siena_filter_build(&filter, &sf_spec);
 
 	EFSYS_LOCK(enp->en_eslp, state);
 
 	rc = siena_filter_search(sftp, &sf_spec, key, B_FALSE,
 	    &filter_idx, &depth);
 	if (rc != 0)
 		goto fail2;
 
 	siena_filter_clear_entry(enp, sftp, filter_idx);
 	if (sftp->sft_used == 0)
 		siena_filter_reset_search_depth(sfp, tbl_id);
 
 	EFSYS_UNLOCK(enp->en_eslp, state);
 	return (0);
 
 fail2:
 	EFSYS_UNLOCK(enp->en_eslp, state);
 	EFSYS_PROBE(fail2);
 
 fail1:
 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
 	return (rc);
 }
 
 #define	SIENA_MAX_SUPPORTED_MATCHES 4
 
 static	__checkReturn	efx_rc_t
 siena_filter_supported_filters(
 	__in				efx_nic_t *enp,
 	__out_ecount(buffer_length)	uint32_t *buffer,
 	__in				size_t buffer_length,
 	__out				size_t *list_lengthp)
 {
 	uint32_t index = 0;
 	uint32_t rx_matches[SIENA_MAX_SUPPORTED_MATCHES];
 	size_t list_length;
 	efx_rc_t rc;
 
 	rx_matches[index++] =
 	    EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 	    EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |
 	    EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT;
 
 	rx_matches[index++] =
 	    EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 	    EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT;
 
 	if (enp->en_features & EFX_FEATURE_MAC_HEADER_FILTERS) {
 		rx_matches[index++] =
 		    EFX_FILTER_MATCH_OUTER_VID | EFX_FILTER_MATCH_LOC_MAC;
 
 		rx_matches[index++] = EFX_FILTER_MATCH_LOC_MAC;
 	}
 
 	EFSYS_ASSERT3U(index, <=, SIENA_MAX_SUPPORTED_MATCHES);
 	list_length = index;
 
 	*list_lengthp = list_length;
 
 	if (buffer_length < list_length) {
 		rc = ENOSPC;
 		goto fail1;
 	}
 
 	memcpy(buffer, rx_matches, list_length * sizeof (rx_matches[0]));
 
 	return (0);
 
 fail1:
 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
 
 	return (rc);
 }
 
 #undef MAX_SUPPORTED
 
 #endif /* EFSYS_OPT_SIENA */
 
 #endif /* EFSYS_OPT_FILTER */
Index: stable/12
===================================================================
--- stable/12	(revision 342320)
+++ stable/12	(revision 342321)

Property changes on: stable/12
___________________________________________________________________
Modified: svn:mergeinfo
## -0,0 +0,1 ##
   Merged /head:r340803