Differential D2618 Diff 5671 head/sys/dev/sfxge/common/efx_filter.c

Changeset View

Standalone View

head/sys/dev/sfxge/common/efx_filter.c

Property	Old Value	New Value
svn:eol-style	null	native \ No newline at end of property
svn:mime-type	null	text/plain \ No newline at end of property

	/*-			/*-
	* Copyright 2007-2009 Solarflare Communications Inc. All rights reserved.			* Copyright (c) 2007-2015 Solarflare Communications Inc.
				* All rights reserved.
	*			*
	* Redistribution and use in source and binary forms, with or without			* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions			* modification, are permitted provided that the following conditions are met:
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*			*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND			* 1. Redistributions of source code must retain the above copyright notice,
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE			* this list of conditions and the following disclaimer.
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE			* 2. Redistributions in binary form must reproduce the above copyright notice,
	* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE			* this list of conditions and the following disclaimer in the documentation
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL			* and/or other materials provided with the distribution.
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS			*
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)			* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT			* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY			* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF			* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
	* SUCH DAMAGE.			* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
				* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
				* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
				* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
				* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
				* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
				* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
				*
				* The views and conclusions contained in the software and documentation are
				* those of the authors and should not be interpreted as representing official
				* policies, either expressed or implied, of the FreeBSD Project.
	*/			*/

	#include <sys/cdefs.h>			#include <sys/cdefs.h>
	__FBSDID("$FreeBSD$");			__FBSDID("$FreeBSD$");

	#include "efsys.h"			#include "efsys.h"
	#include "efx.h"			#include "efx.h"
	#include "efx_types.h"			#include "efx_types.h"
	#include "efx_regs.h"			#include "efx_regs.h"
	#include "efx_impl.h"			#include "efx_impl.h"


	#if EFSYS_OPT_FILTER			#if EFSYS_OPT_FILTER

	/* "Fudge factors" - difference between programmed value and actual depth.			#if EFSYS_OPT_FALCON \|\| EFSYS_OPT_SIENA

				static __checkReturn int
				falconsiena_filter_init(
				__in efx_nic_t *enp);

				static void
				falconsiena_filter_fini(
				__in efx_nic_t *enp);

				static __checkReturn int
				falconsiena_filter_restore(
				__in efx_nic_t *enp);

				static __checkReturn int
				falconsiena_filter_add(
				__in efx_nic_t *enp,
				__inout efx_filter_spec_t *spec,
				__in boolean_t may_replace);

				static __checkReturn int
				falconsiena_filter_delete(
				__in efx_nic_t *enp,
				__inout efx_filter_spec_t *spec);

				static __checkReturn int
				falconsiena_filter_supported_filters(
				__in efx_nic_t *enp,
				__out uint32_t *list,
				__out size_t *length);

				#endif /* EFSYS_OPT_FALCON \|\| EFSYS_OPT_SIENA */

				#if EFSYS_OPT_FALCON
				static efx_filter_ops_t __efx_filter_falcon_ops = {
				falconsiena_filter_init, /* efo_init */
				falconsiena_filter_fini, /* efo_fini */
				falconsiena_filter_restore, /* efo_restore */
				falconsiena_filter_add, /* efo_add */
				falconsiena_filter_delete, /* efo_delete */
				falconsiena_filter_supported_filters, /* efo_supported_filters */
				NULL, /* efo_reconfigure */
				};
				#endif /* EFSYS_OPT_FALCON */

				#if EFSYS_OPT_SIENA
				static efx_filter_ops_t __efx_filter_siena_ops = {
				falconsiena_filter_init, /* efo_init */
				falconsiena_filter_fini, /* efo_fini */
				falconsiena_filter_restore, /* efo_restore */
				falconsiena_filter_add, /* efo_add */
				falconsiena_filter_delete, /* efo_delete */
				falconsiena_filter_supported_filters, /* efo_supported_filters */
				NULL, /* efo_reconfigure */
				};
				#endif /* EFSYS_OPT_SIENA */

				#if EFSYS_OPT_HUNTINGTON
				static efx_filter_ops_t __efx_filter_hunt_ops = {
				hunt_filter_init, /* efo_init */
				hunt_filter_fini, /* efo_fini */
				hunt_filter_restore, /* efo_restore */
				hunt_filter_add, /* efo_add */
				hunt_filter_delete, /* efo_delete */
				hunt_filter_supported_filters, /* efo_supported_filters */
				hunt_filter_reconfigure, /* efo_reconfigure */
				};
				#endif /* EFSYS_OPT_HUNTINGTON */

				__checkReturn int
				efx_filter_insert(
				__in efx_nic_t *enp,
				__inout efx_filter_spec_t *spec)
				{
				efx_filter_ops_t *efop = enp->en_efop;

				EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
				EFSYS_ASSERT3P(spec, !=, NULL);
				EFSYS_ASSERT3U(spec->efs_flags, &, EFX_FILTER_FLAG_RX);

				return (efop->efo_add(enp, spec, B_FALSE));
				}

				__checkReturn int
				efx_filter_remove(
				__in efx_nic_t *enp,
				__inout efx_filter_spec_t *spec)
				{
				efx_filter_ops_t *efop = enp->en_efop;

				EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
				EFSYS_ASSERT3P(spec, !=, NULL);
				EFSYS_ASSERT3U(spec->efs_flags, &, EFX_FILTER_FLAG_RX);

				#if EFSYS_OPT_RX_SCALE
				spec->efs_rss_context = enp->en_rss_context;
				#endif

				return (efop->efo_delete(enp, spec));
				}

				__checkReturn int
				efx_filter_restore(
				__in efx_nic_t *enp)
				{
				int rc;

				EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);

				if ((rc = enp->en_efop->efo_restore(enp)) != 0)
				goto fail1;

				return (0);

				fail1:
				EFSYS_PROBE1(fail1, int, rc);

				return (rc);
				}

				__checkReturn int
				efx_filter_init(
				__in efx_nic_t *enp)
				{
				efx_filter_ops_t *efop;
				int rc;

				/* Check that efx_filter_spec_t is 64 bytes. */
				EFX_STATIC_ASSERT(sizeof (efx_filter_spec_t) == 64);

				EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
				EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
				EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_FILTER));

				switch (enp->en_family) {
				#if EFSYS_OPT_FALCON
				case EFX_FAMILY_FALCON:
				efop = (efx_filter_ops_t *)&__efx_filter_falcon_ops;
				break;
				#endif /* EFSYS_OPT_FALCON */

				#if EFSYS_OPT_SIENA
				case EFX_FAMILY_SIENA:
				efop = (efx_filter_ops_t *)&__efx_filter_siena_ops;
				break;
				#endif /* EFSYS_OPT_SIENA */

				#if EFSYS_OPT_HUNTINGTON
				case EFX_FAMILY_HUNTINGTON:
				efop = (efx_filter_ops_t *)&__efx_filter_hunt_ops;
				break;
				#endif /* EFSYS_OPT_HUNTINGTON */

				default:
				EFSYS_ASSERT(0);
				rc = ENOTSUP;
				goto fail1;
				}

				if ((rc = efop->efo_init(enp)) != 0)
				goto fail2;

				enp->en_efop = efop;
				enp->en_mod_flags \|= EFX_MOD_FILTER;
				return (0);

				fail2:
				EFSYS_PROBE(fail2);
				fail1:
				EFSYS_PROBE1(fail1, int, rc);

				enp->en_efop = NULL;
				enp->en_mod_flags &= ~EFX_MOD_FILTER;
				return (rc);
				}

				void
				efx_filter_fini(
				__in efx_nic_t *enp)
				{
				EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
				EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
				EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);

				enp->en_efop->efo_fini(enp);

				enp->en_efop = NULL;
				enp->en_mod_flags &= ~EFX_MOD_FILTER;
				}

				__checkReturn int
				efx_filter_supported_filters(
				__in efx_nic_t *enp,
				__out uint32_t *list,
				__out size_t *length)
				{
				int rc;

				EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
				EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
				EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
				EFSYS_ASSERT(enp->en_efop->efo_supported_filters != NULL);

				if ((rc = enp->en_efop->efo_supported_filters(enp, list, length)) != 0)
				goto fail1;

				return (0);

				fail1:
				EFSYS_PROBE1(fail1, int, rc);

				return (rc);
				}

				__checkReturn int
				efx_filter_reconfigure(
				__in efx_nic_t *enp,
				__in_ecount(6) uint8_t const *mac_addr,
				__in boolean_t all_unicst,
				__in boolean_t mulcst,
				__in boolean_t all_mulcst,
				__in boolean_t brdcst,
				__in_ecount(6count) uint8_t const addrs,
				__in int count)
				{
				int rc;

				EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
				EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
				EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);

				if (enp->en_efop->efo_reconfigure != NULL) {
				if ((rc = enp->en_efop->efo_reconfigure(enp, mac_addr,
				all_unicst, mulcst,
				all_mulcst, brdcst,
				addrs, count)) != 0)
				goto fail1;
				}

				return (0);

				fail1:
				EFSYS_PROBE1(fail1, int, rc);

				return (rc);
				}

				void
				efx_filter_spec_init_rx(
				__inout efx_filter_spec_t *spec,
				__in efx_filter_priority_t priority,
				__in efx_filter_flag_t flags,
				__in efx_rxq_t *erp)
				{
				EFSYS_ASSERT3P(spec, !=, NULL);
				EFSYS_ASSERT3P(erp, !=, NULL);
				EFSYS_ASSERT((flags & ~(EFX_FILTER_FLAG_RX_RSS \|
				EFX_FILTER_FLAG_RX_SCATTER)) == 0);

				memset(spec, 0, sizeof (*spec));
				spec->efs_priority = priority;
				spec->efs_flags = EFX_FILTER_FLAG_RX \| flags;
				spec->efs_rss_context = EFX_FILTER_SPEC_RSS_CONTEXT_DEFAULT;
				spec->efs_dmaq_id = (uint16_t)erp->er_index;
				}

				void
				efx_filter_spec_init_tx(
				__inout efx_filter_spec_t *spec,
				__in efx_txq_t *etp)
				{
				EFSYS_ASSERT3P(spec, !=, NULL);
				EFSYS_ASSERT3P(etp, !=, NULL);

				memset(spec, 0, sizeof (*spec));
				spec->efs_priority = EFX_FILTER_PRI_REQUIRED;
				spec->efs_flags = EFX_FILTER_FLAG_TX;
				spec->efs_dmaq_id = (uint16_t)etp->et_index;
				}


				/*
				* Specify IPv4 host, transport protocol and port in a filter specification
				*/
				__checkReturn int
				efx_filter_spec_set_ipv4_local(
				__inout efx_filter_spec_t *spec,
				__in uint8_t proto,
				__in uint32_t host,
				__in uint16_t port)
				{
				EFSYS_ASSERT3P(spec, !=, NULL);

				spec->efs_match_flags \|=
				EFX_FILTER_MATCH_ETHER_TYPE \| EFX_FILTER_MATCH_IP_PROTO \|
				EFX_FILTER_MATCH_LOC_HOST \| EFX_FILTER_MATCH_LOC_PORT;
				spec->efs_ether_type = EFX_ETHER_TYPE_IPV4;
				spec->efs_ip_proto = proto;
				spec->efs_loc_host.eo_u32[0] = host;
				spec->efs_loc_port = port;
				return (0);
				}

				/*
				* Specify IPv4 hosts, transport protocol and ports in a filter specification
				*/
				__checkReturn int
				efx_filter_spec_set_ipv4_full(
				__inout efx_filter_spec_t *spec,
				__in uint8_t proto,
				__in uint32_t lhost,
				__in uint16_t lport,
				__in uint32_t rhost,
				__in uint16_t rport)
				{
				EFSYS_ASSERT3P(spec, !=, NULL);

				spec->efs_match_flags \|=
				EFX_FILTER_MATCH_ETHER_TYPE \| EFX_FILTER_MATCH_IP_PROTO \|
				EFX_FILTER_MATCH_LOC_HOST \| EFX_FILTER_MATCH_LOC_PORT \|
				EFX_FILTER_MATCH_REM_HOST \| EFX_FILTER_MATCH_REM_PORT;
				spec->efs_ether_type = EFX_ETHER_TYPE_IPV4;
				spec->efs_ip_proto = proto;
				spec->efs_loc_host.eo_u32[0] = lhost;
				spec->efs_loc_port = lport;
				spec->efs_rem_host.eo_u32[0] = rhost;
				spec->efs_rem_port = rport;
				return (0);
				}

				/*
				* Specify local Ethernet address and/or VID in filter specification
				*/
				__checkReturn int
				efx_filter_spec_set_eth_local(
				__inout efx_filter_spec_t *spec,
				__in uint16_t vid,
				__in const uint8_t *addr)
				{
				EFSYS_ASSERT3P(spec, !=, NULL);
				EFSYS_ASSERT3P(addr, !=, NULL);

				if (vid == EFX_FILTER_SPEC_VID_UNSPEC && addr == NULL)
				return (EINVAL);

				if (vid != EFX_FILTER_SPEC_VID_UNSPEC) {
				spec->efs_match_flags \|= EFX_FILTER_MATCH_OUTER_VID;
				spec->efs_outer_vid = vid;
				}
				if (addr != NULL) {
				spec->efs_match_flags \|= EFX_FILTER_MATCH_LOC_MAC;
				memcpy(spec->efs_loc_mac, addr, EFX_MAC_ADDR_LEN);
				}
				return (0);
				}

				/*
				* Specify matching otherwise-unmatched unicast in a filter specification
				*/
				__checkReturn int
				efx_filter_spec_set_uc_def(
				__inout efx_filter_spec_t *spec)
				{
				EFSYS_ASSERT3P(spec, !=, NULL);

				spec->efs_match_flags \|= EFX_FILTER_MATCH_LOC_MAC_IG;
				return (0);
				}

				/*
				* Specify matching otherwise-unmatched multicast in a filter specification
				*/
				__checkReturn int
				efx_filter_spec_set_mc_def(
				__inout efx_filter_spec_t *spec)
				{
				EFSYS_ASSERT3P(spec, !=, NULL);

				spec->efs_match_flags \|= EFX_FILTER_MATCH_LOC_MAC_IG;
				spec->efs_loc_mac[0] = 1;
				return (0);
				}



				#if EFSYS_OPT_FALCON \|\| EFSYS_OPT_SIENA

				/*
				* "Fudge factors" - difference between programmed value and actual depth.
	* Due to pipelined implementation we need to program H/W with a value that			* Due to pipelined implementation we need to program H/W with a value that
	* is larger than the hop limit we want.			* is larger than the hop limit we want.
	*/			*/
	#define FILTER_CTL_SRCH_FUDGE_WILD 3			#define FILTER_CTL_SRCH_FUDGE_WILD 3
	#define FILTER_CTL_SRCH_FUDGE_FULL 1			#define FILTER_CTL_SRCH_FUDGE_FULL 1

	/* Hard maximum hop limit. Hardware will time-out beyond 200-something.			/*
				* Hard maximum hop limit. Hardware will time-out beyond 200-something.
	* We also need to avoid infinite loops in efx_filter_search() when the			* We also need to avoid infinite loops in efx_filter_search() when the
	* table is full.			* table is full.
	*/			*/
	#define FILTER_CTL_SRCH_MAX 200			#define FILTER_CTL_SRCH_MAX 200

	/* The filter hash function is LFSR polynomial x^16 + x^3 + 1 of a 32-bit			static __checkReturn int
	* key derived from the n-tuple. */			falconsiena_filter_spec_from_gen_spec(
				__out falconsiena_filter_spec_t *fs_spec,
				__in efx_filter_spec_t *gen_spec)
				{
				int rc;
				boolean_t is_full = B_FALSE;

				if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX)
				EFSYS_ASSERT3U(gen_spec->efs_flags, ==, EFX_FILTER_FLAG_TX);
				else
				EFSYS_ASSERT3U(gen_spec->efs_flags, &, EFX_FILTER_FLAG_RX);

				/* Falconsiena only has one RSS context */
				if ((gen_spec->efs_flags & EFX_FILTER_FLAG_RX_RSS) &&
				gen_spec->efs_rss_context != 0) {
				rc = EINVAL;
				goto fail1;
				}

				fs_spec->fsfs_flags = gen_spec->efs_flags;
				fs_spec->fsfs_dmaq_id = gen_spec->efs_dmaq_id;

				switch (gen_spec->efs_match_flags) {
				case EFX_FILTER_MATCH_ETHER_TYPE \| EFX_FILTER_MATCH_IP_PROTO \|
				EFX_FILTER_MATCH_LOC_HOST \| EFX_FILTER_MATCH_LOC_PORT \|
				EFX_FILTER_MATCH_REM_HOST \| EFX_FILTER_MATCH_REM_PORT:
				is_full = B_TRUE;
				/* Fall through */
				case EFX_FILTER_MATCH_ETHER_TYPE \| EFX_FILTER_MATCH_IP_PROTO \|
				EFX_FILTER_MATCH_LOC_HOST \| EFX_FILTER_MATCH_LOC_PORT: {
				uint32_t rhost, host1, host2;
				uint16_t rport, port1, port2;

				if (gen_spec->efs_ether_type != EFX_ETHER_TYPE_IPV4) {
				rc = ENOTSUP;
				goto fail2;
				}
				if (gen_spec->efs_loc_port == 0 \|\|
				(is_full && gen_spec->efs_rem_port == 0)) {
				rc = EINVAL;
				goto fail3;
				}
				switch (gen_spec->efs_ip_proto) {
				case EFX_IPPROTO_TCP:
				if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
				fs_spec->fsfs_type = (is_full ?
				EFX_FS_FILTER_TX_TCP_FULL :
				EFX_FS_FILTER_TX_TCP_WILD);
				} else {
				fs_spec->fsfs_type = (is_full ?
				EFX_FS_FILTER_RX_TCP_FULL :
				EFX_FS_FILTER_RX_TCP_WILD);
				}
				break;
				case EFX_IPPROTO_UDP:
				if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
				fs_spec->fsfs_type = (is_full ?
				EFX_FS_FILTER_TX_UDP_FULL :
				EFX_FS_FILTER_TX_UDP_WILD);
				} else {
				fs_spec->fsfs_type = (is_full ?
				EFX_FS_FILTER_RX_UDP_FULL :
				EFX_FS_FILTER_RX_UDP_WILD);
				}
				break;
				default:
				rc = ENOTSUP;
				goto fail4;
				}
				/*
				* The filter is constructed in terms of source and destination,
				* with the odd wrinkle that the ports are swapped in a UDP
				* wildcard filter. We need to convert from local and remote
				* addresses (zero for a wildcard).
				*/
				rhost = is_full ? gen_spec->efs_rem_host.eo_u32[0] : 0;
				rport = is_full ? gen_spec->efs_rem_port : 0;
				if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
				host1 = gen_spec->efs_loc_host.eo_u32[0];
				host2 = rhost;
				} else {
				host1 = rhost;
				host2 = gen_spec->efs_loc_host.eo_u32[0];
				}
				if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
				if (fs_spec->fsfs_type == EFX_FS_FILTER_TX_UDP_WILD) {
				port1 = rport;
				port2 = gen_spec->efs_loc_port;
				} else {
				port1 = gen_spec->efs_loc_port;
				port2 = rport;
				}
				} else {
				if (fs_spec->fsfs_type == EFX_FS_FILTER_RX_UDP_WILD) {
				port1 = gen_spec->efs_loc_port;
				port2 = rport;
				} else {
				port1 = rport;
				port2 = gen_spec->efs_loc_port;
				}
				}
				fs_spec->fsfs_dword[0] = (host1 << 16) \| port1;
				fs_spec->fsfs_dword[1] = (port2 << 16) \| (host1 >> 16);
				fs_spec->fsfs_dword[2] = host2;
				break;
				}

				case EFX_FILTER_MATCH_LOC_MAC \| EFX_FILTER_MATCH_OUTER_VID:
				is_full = B_TRUE;
				/* Fall through */
				case EFX_FILTER_MATCH_LOC_MAC:
				if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
				fs_spec->fsfs_type = (is_full ?
				EFX_FS_FILTER_TX_MAC_FULL :
				EFX_FS_FILTER_TX_MAC_WILD);
				} else {
				fs_spec->fsfs_type = (is_full ?
				EFX_FS_FILTER_RX_MAC_FULL :
				EFX_FS_FILTER_RX_MAC_WILD);
				}
				fs_spec->fsfs_dword[0] = is_full ? gen_spec->efs_outer_vid : 0;
				fs_spec->fsfs_dword[1] =
				gen_spec->efs_loc_mac[2] << 24 \|
				gen_spec->efs_loc_mac[3] << 16 \|
				gen_spec->efs_loc_mac[4] << 8 \|
				gen_spec->efs_loc_mac[5];
				fs_spec->fsfs_dword[2] =
				gen_spec->efs_loc_mac[0] << 8 \|
				gen_spec->efs_loc_mac[1];
				break;

				default:
				EFSYS_ASSERT(B_FALSE);
				rc = ENOTSUP;
				goto fail5;
				}

				return (0);

				fail5:
				EFSYS_PROBE(fail5);
				fail4:
				EFSYS_PROBE(fail4);
				fail3:
				EFSYS_PROBE(fail3);
				fail2:
				EFSYS_PROBE(fail2);
				fail1:
				EFSYS_PROBE1(fail1, int, rc);

				return (rc);
				}

				/*
				* The filter hash function is LFSR polynomial x^16 + x^3 + 1 of a 32-bit
				* key derived from the n-tuple.
				*/
	static uint16_t			static uint16_t
	efx_filter_tbl_hash(			falconsiena_filter_tbl_hash(
	__in uint32_t key)			__in uint32_t key)
	{			{
	uint16_t tmp;			uint16_t tmp;

	/* First 16 rounds */			/* First 16 rounds */
	tmp = 0x1fff ^ (uint16_t)(key >> 16);			tmp = 0x1fff ^ (uint16_t)(key >> 16);
	tmp = tmp ^ tmp >> 3 ^ tmp >> 6;			tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
	tmp = tmp ^ tmp >> 9;			tmp = tmp ^ tmp >> 9;

	/* Last 16 rounds */			/* Last 16 rounds */
	tmp = tmp ^ tmp << 13 ^ (uint16_t)(key & 0xffff);			tmp = tmp ^ tmp << 13 ^ (uint16_t)(key & 0xffff);
	tmp = tmp ^ tmp >> 3 ^ tmp >> 6;			tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
	tmp = tmp ^ tmp >> 9;			tmp = tmp ^ tmp >> 9;

	return (tmp);			return (tmp);
	}			}

				/*
	/* To allow for hash collisions, filter search continues at these			* To allow for hash collisions, filter search continues at these
	* increments from the first possible entry selected by the hash. */			* increments from the first possible entry selected by the hash.
				*/
	static uint16_t			static uint16_t
	efx_filter_tbl_increment(			falconsiena_filter_tbl_increment(
	__in uint32_t key)			__in uint32_t key)
	{			{
	return ((uint16_t)(key * 2 - 1));			return ((uint16_t)(key * 2 - 1));
	}			}

	static __checkReturn boolean_t			static __checkReturn boolean_t
	efx_filter_test_used(			falconsiena_filter_test_used(
	__in efx_filter_tbl_t *eftp,			__in falconsiena_filter_tbl_t *fsftp,
	__in unsigned int index)			__in unsigned int index)
	{			{
	EFSYS_ASSERT3P(eftp->eft_bitmap, !=, NULL);			EFSYS_ASSERT3P(fsftp->fsft_bitmap, !=, NULL);
	return ((eftp->eft_bitmap[index / 32] & (1 << (index % 32))) != 0);			return ((fsftp->fsft_bitmap[index / 32] & (1 << (index % 32))) != 0);
	}			}

	static void			static void
	efx_filter_set_used(			falconsiena_filter_set_used(
	__in efx_filter_tbl_t *eftp,			__in falconsiena_filter_tbl_t *fsftp,
	__in unsigned int index)			__in unsigned int index)
	{			{
	EFSYS_ASSERT3P(eftp->eft_bitmap, !=, NULL);			EFSYS_ASSERT3P(fsftp->fsft_bitmap, !=, NULL);
	eftp->eft_bitmap[index / 32] \|= (1 << (index % 32));			fsftp->fsft_bitmap[index / 32] \|= (1 << (index % 32));
	++eftp->eft_used;			++fsftp->fsft_used;
	}			}

	static void			static void
	efx_filter_clear_used(			falconsiena_filter_clear_used(
	__in efx_filter_tbl_t *eftp,			__in falconsiena_filter_tbl_t *fsftp,
	__in unsigned int index)			__in unsigned int index)
	{			{
	EFSYS_ASSERT3P(eftp->eft_bitmap, !=, NULL);			EFSYS_ASSERT3P(fsftp->fsft_bitmap, !=, NULL);
	eftp->eft_bitmap[index / 32] &= ~(1 << (index % 32));			fsftp->fsft_bitmap[index / 32] &= ~(1 << (index % 32));

	--eftp->eft_used;			--fsftp->fsft_used;
	EFSYS_ASSERT3U(eftp->eft_used, >=, 0);			EFSYS_ASSERT3U(fsftp->fsft_used, >=, 0);
	}			}


	static efx_filter_tbl_id_t			static falconsiena_filter_tbl_id_t
	efx_filter_tbl_id(			falconsiena_filter_tbl_id(
	__in efx_filter_type_t type)			__in falconsiena_filter_type_t type)
	{			{
	efx_filter_tbl_id_t tbl_id;			falconsiena_filter_tbl_id_t tbl_id;

	switch (type)			switch (type) {
	{			case EFX_FS_FILTER_RX_TCP_FULL:
	case EFX_FILTER_RX_TCP_FULL:			case EFX_FS_FILTER_RX_TCP_WILD:
	case EFX_FILTER_RX_TCP_WILD:			case EFX_FS_FILTER_RX_UDP_FULL:
	case EFX_FILTER_RX_UDP_FULL:			case EFX_FS_FILTER_RX_UDP_WILD:
	case EFX_FILTER_RX_UDP_WILD:			tbl_id = EFX_FS_FILTER_TBL_RX_IP;
	tbl_id = EFX_FILTER_TBL_RX_IP;
	break;			break;

	#if EFSYS_OPT_SIENA			#if EFSYS_OPT_SIENA
	case EFX_FILTER_RX_MAC_FULL:			case EFX_FS_FILTER_RX_MAC_FULL:
	case EFX_FILTER_RX_MAC_WILD:			case EFX_FS_FILTER_RX_MAC_WILD:
	tbl_id = EFX_FILTER_TBL_RX_MAC;			tbl_id = EFX_FS_FILTER_TBL_RX_MAC;
	break;			break;

	case EFX_FILTER_TX_TCP_FULL:			case EFX_FS_FILTER_TX_TCP_FULL:
	case EFX_FILTER_TX_TCP_WILD:			case EFX_FS_FILTER_TX_TCP_WILD:
	case EFX_FILTER_TX_UDP_FULL:			case EFX_FS_FILTER_TX_UDP_FULL:
	case EFX_FILTER_TX_UDP_WILD:			case EFX_FS_FILTER_TX_UDP_WILD:
	tbl_id = EFX_FILTER_TBL_TX_IP;			tbl_id = EFX_FS_FILTER_TBL_TX_IP;
	break;			break;

	case EFX_FILTER_TX_MAC_FULL:			case EFX_FS_FILTER_TX_MAC_FULL:
	case EFX_FILTER_TX_MAC_WILD:			case EFX_FS_FILTER_TX_MAC_WILD:
	tbl_id = EFX_FILTER_TBL_RX_MAC;			tbl_id = EFX_FS_FILTER_TBL_TX_MAC;
	break;			break;
	#endif /* EFSYS_OPT_SIENA */			#endif /* EFSYS_OPT_SIENA */

	default:			default:
	EFSYS_ASSERT(B_FALSE);			EFSYS_ASSERT(B_FALSE);
				tbl_id = EFX_FS_FILTER_NTBLS;
	break;			break;
	}			}
	return (tbl_id);			return (tbl_id);
	}			}

	static void			static void
	efx_filter_reset_search_depth(			falconsiena_filter_reset_search_depth(
	__inout efx_filter_t *efp,			__inout falconsiena_filter_t *fsfp,
	__in efx_filter_tbl_id_t tbl_id)			__in falconsiena_filter_tbl_id_t tbl_id)
	{			{
	switch (tbl_id)			switch (tbl_id) {
	{			case EFX_FS_FILTER_TBL_RX_IP:
	case EFX_FILTER_TBL_RX_IP:			fsfp->fsf_depth[EFX_FS_FILTER_RX_TCP_FULL] = 0;
	efp->ef_depth[EFX_FILTER_RX_TCP_FULL] = 0;			fsfp->fsf_depth[EFX_FS_FILTER_RX_TCP_WILD] = 0;
	efp->ef_depth[EFX_FILTER_RX_TCP_WILD] = 0;			fsfp->fsf_depth[EFX_FS_FILTER_RX_UDP_FULL] = 0;
	efp->ef_depth[EFX_FILTER_RX_UDP_FULL] = 0;			fsfp->fsf_depth[EFX_FS_FILTER_RX_UDP_WILD] = 0;
	efp->ef_depth[EFX_FILTER_RX_UDP_WILD] = 0;
	break;			break;

	#if EFSYS_OPT_SIENA			#if EFSYS_OPT_SIENA
	case EFX_FILTER_TBL_RX_MAC:			case EFX_FS_FILTER_TBL_RX_MAC:
	efp->ef_depth[EFX_FILTER_RX_MAC_FULL] = 0;			fsfp->fsf_depth[EFX_FS_FILTER_RX_MAC_FULL] = 0;
	efp->ef_depth[EFX_FILTER_RX_MAC_WILD] = 0;			fsfp->fsf_depth[EFX_FS_FILTER_RX_MAC_WILD] = 0;
	break;			break;

	case EFX_FILTER_TBL_TX_IP:			case EFX_FS_FILTER_TBL_TX_IP:
	efp->ef_depth[EFX_FILTER_TX_TCP_FULL] = 0;			fsfp->fsf_depth[EFX_FS_FILTER_TX_TCP_FULL] = 0;
	efp->ef_depth[EFX_FILTER_TX_TCP_WILD] = 0;			fsfp->fsf_depth[EFX_FS_FILTER_TX_TCP_WILD] = 0;
	efp->ef_depth[EFX_FILTER_TX_UDP_FULL] = 0;			fsfp->fsf_depth[EFX_FS_FILTER_TX_UDP_FULL] = 0;
	efp->ef_depth[EFX_FILTER_TX_UDP_WILD] = 0;			fsfp->fsf_depth[EFX_FS_FILTER_TX_UDP_WILD] = 0;
	break;			break;

	case EFX_FILTER_TBL_TX_MAC:			case EFX_FS_FILTER_TBL_TX_MAC:
	efp->ef_depth[EFX_FILTER_TX_MAC_FULL] = 0;			fsfp->fsf_depth[EFX_FS_FILTER_TX_MAC_FULL] = 0;
	efp->ef_depth[EFX_FILTER_TX_MAC_WILD] = 0;			fsfp->fsf_depth[EFX_FS_FILTER_TX_MAC_WILD] = 0;
	break;			break;
	#endif /* EFSYS_OPT_SIENA */			#endif /* EFSYS_OPT_SIENA */

	default:			default:
	EFSYS_ASSERT(B_FALSE);			EFSYS_ASSERT(B_FALSE);
	break;			break;
	}			}
	}			}

	static void			static void
	efx_filter_push_rx_limits(			falconsiena_filter_push_rx_limits(
	__in efx_nic_t *enp)			__in efx_nic_t *enp)
	{			{
	efx_filter_t *efp = &enp->en_filter;			falconsiena_filter_t *fsfp = enp->en_filter.ef_falconsiena_filter;
	efx_oword_t oword;			efx_oword_t oword;

	EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);			EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);

	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TCP_FULL_SRCH_LIMIT,			EFX_SET_OWORD_FIELD(oword, FRF_AZ_TCP_FULL_SRCH_LIMIT,
	efp->ef_depth[EFX_FILTER_RX_TCP_FULL] +			fsfp->fsf_depth[EFX_FS_FILTER_RX_TCP_FULL] +
	FILTER_CTL_SRCH_FUDGE_FULL);			FILTER_CTL_SRCH_FUDGE_FULL);
	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TCP_WILD_SRCH_LIMIT,			EFX_SET_OWORD_FIELD(oword, FRF_AZ_TCP_WILD_SRCH_LIMIT,
	efp->ef_depth[EFX_FILTER_RX_TCP_WILD] +			fsfp->fsf_depth[EFX_FS_FILTER_RX_TCP_WILD] +
	FILTER_CTL_SRCH_FUDGE_WILD);			FILTER_CTL_SRCH_FUDGE_WILD);
	EFX_SET_OWORD_FIELD(oword, FRF_AZ_UDP_FULL_SRCH_LIMIT,			EFX_SET_OWORD_FIELD(oword, FRF_AZ_UDP_FULL_SRCH_LIMIT,
	efp->ef_depth[EFX_FILTER_RX_UDP_FULL] +			fsfp->fsf_depth[EFX_FS_FILTER_RX_UDP_FULL] +
	FILTER_CTL_SRCH_FUDGE_FULL);			FILTER_CTL_SRCH_FUDGE_FULL);
	EFX_SET_OWORD_FIELD(oword, FRF_AZ_UDP_WILD_SRCH_LIMIT,			EFX_SET_OWORD_FIELD(oword, FRF_AZ_UDP_WILD_SRCH_LIMIT,
	efp->ef_depth[EFX_FILTER_RX_UDP_WILD] +			fsfp->fsf_depth[EFX_FS_FILTER_RX_UDP_WILD] +
	FILTER_CTL_SRCH_FUDGE_WILD);			FILTER_CTL_SRCH_FUDGE_WILD);

	#if EFSYS_OPT_SIENA			#if EFSYS_OPT_SIENA
	if (efp->ef_tbl[EFX_FILTER_TBL_RX_MAC].eft_size) {			if (fsfp->fsf_tbl[EFX_FS_FILTER_TBL_RX_MAC].fsft_size) {
	EFX_SET_OWORD_FIELD(oword,			EFX_SET_OWORD_FIELD(oword,
	FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT,			FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT,
	efp->ef_depth[EFX_FILTER_RX_MAC_FULL] +			fsfp->fsf_depth[EFX_FS_FILTER_RX_MAC_FULL] +
	FILTER_CTL_SRCH_FUDGE_FULL);			FILTER_CTL_SRCH_FUDGE_FULL);
	EFX_SET_OWORD_FIELD(oword,			EFX_SET_OWORD_FIELD(oword,
	FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT,			FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT,
	efp->ef_depth[EFX_FILTER_RX_MAC_WILD] +			fsfp->fsf_depth[EFX_FS_FILTER_RX_MAC_WILD] +
	FILTER_CTL_SRCH_FUDGE_WILD);			FILTER_CTL_SRCH_FUDGE_WILD);
	}			}
	#endif /* EFSYS_OPT_SIENA */			#endif /* EFSYS_OPT_SIENA */

	EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);			EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
	}			}

	static void			static void
	efx_filter_push_tx_limits(			falconsiena_filter_push_tx_limits(
	__in efx_nic_t *enp)			__in efx_nic_t *enp)
	{			{
	efx_filter_t *efp = &enp->en_filter;			falconsiena_filter_t *fsfp = enp->en_filter.ef_falconsiena_filter;
	efx_oword_t oword;			efx_oword_t oword;

	if (efp->ef_tbl[EFX_FILTER_TBL_TX_IP].eft_size == 0)
	return;

	EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);			EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);

	EFX_SET_OWORD_FIELD(oword, FRF_CZ_TX_TCPIP_FILTER_FULL_SEARCH_RANGE,			if (fsfp->fsf_tbl[EFX_FS_FILTER_TBL_TX_IP].fsft_size != 0) {
	efp->ef_depth[EFX_FILTER_TX_TCP_FULL] +			EFX_SET_OWORD_FIELD(oword,
				FRF_CZ_TX_TCPIP_FILTER_FULL_SEARCH_RANGE,
				fsfp->fsf_depth[EFX_FS_FILTER_TX_TCP_FULL] +
	FILTER_CTL_SRCH_FUDGE_FULL);			FILTER_CTL_SRCH_FUDGE_FULL);
	EFX_SET_OWORD_FIELD(oword, FRF_CZ_TX_TCPIP_FILTER_WILD_SEARCH_RANGE,			EFX_SET_OWORD_FIELD(oword,
	efp->ef_depth[EFX_FILTER_TX_TCP_WILD] +			FRF_CZ_TX_TCPIP_FILTER_WILD_SEARCH_RANGE,
				fsfp->fsf_depth[EFX_FS_FILTER_TX_TCP_WILD] +
	FILTER_CTL_SRCH_FUDGE_WILD);			FILTER_CTL_SRCH_FUDGE_WILD);
	EFX_SET_OWORD_FIELD(oword, FRF_CZ_TX_UDPIP_FILTER_FULL_SEARCH_RANGE,			EFX_SET_OWORD_FIELD(oword,
	efp->ef_depth[EFX_FILTER_TX_UDP_FULL] +			FRF_CZ_TX_UDPIP_FILTER_FULL_SEARCH_RANGE,
				fsfp->fsf_depth[EFX_FS_FILTER_TX_UDP_FULL] +
	FILTER_CTL_SRCH_FUDGE_FULL);			FILTER_CTL_SRCH_FUDGE_FULL);
	EFX_SET_OWORD_FIELD(oword, FRF_CZ_TX_UDPIP_FILTER_WILD_SEARCH_RANGE,			EFX_SET_OWORD_FIELD(oword,
	efp->ef_depth[EFX_FILTER_TX_UDP_WILD] +			FRF_CZ_TX_UDPIP_FILTER_WILD_SEARCH_RANGE,
				fsfp->fsf_depth[EFX_FS_FILTER_TX_UDP_WILD] +
	FILTER_CTL_SRCH_FUDGE_WILD);			FILTER_CTL_SRCH_FUDGE_WILD);
				}

				if (fsfp->fsf_tbl[EFX_FS_FILTER_TBL_TX_MAC].fsft_size != 0) {
				EFX_SET_OWORD_FIELD(
				oword, FRF_CZ_TX_ETH_FILTER_FULL_SEARCH_RANGE,
				fsfp->fsf_depth[EFX_FS_FILTER_TX_MAC_FULL] +
				FILTER_CTL_SRCH_FUDGE_FULL);
				EFX_SET_OWORD_FIELD(
				oword, FRF_CZ_TX_ETH_FILTER_WILD_SEARCH_RANGE,
				fsfp->fsf_depth[EFX_FS_FILTER_TX_MAC_WILD] +
				FILTER_CTL_SRCH_FUDGE_WILD);
				}

	EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);			EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
	}			}

	/* Build a filter entry and return its n-tuple key. */			/* Build a filter entry and return its n-tuple key. */
	static __checkReturn uint32_t			static __checkReturn uint32_t
	efx_filter_build(			falconsiena_filter_build(
	__out efx_oword_t *filter,			__out efx_oword_t *filter,
	__in efx_filter_spec_t *spec)			__in falconsiena_filter_spec_t *spec)
	{			{
	uint32_t dword3;			uint32_t dword3;
	uint32_t key;			uint32_t key;
	uint8_t type = spec->efs_type;			uint8_t type = spec->fsfs_type;
	uint8_t flags = spec->efs_flags;			uint32_t flags = spec->fsfs_flags;

	switch (efx_filter_tbl_id(type)) {			switch (falconsiena_filter_tbl_id(type)) {
	case EFX_FILTER_TBL_RX_IP: {			case EFX_FS_FILTER_TBL_RX_IP: {
	boolean_t is_udp = (type == EFX_FILTER_RX_UDP_FULL \|\|			boolean_t is_udp = (type == EFX_FS_FILTER_RX_UDP_FULL \|\|
	type == EFX_FILTER_RX_UDP_WILD);			type == EFX_FS_FILTER_RX_UDP_WILD);
	EFX_POPULATE_OWORD_7(*filter,			EFX_POPULATE_OWORD_7(*filter,
	FRF_BZ_RSS_EN, (flags & EFX_FILTER_FLAG_RX_RSS) ? 1 : 0,			FRF_BZ_RSS_EN,
	FRF_BZ_SCATTER_EN, (flags & EFX_FILTER_FLAG_RX_SCATTER) ? 1 : 0,			(flags & EFX_FILTER_FLAG_RX_RSS) ? 1 : 0,
				FRF_BZ_SCATTER_EN,
				(flags & EFX_FILTER_FLAG_RX_SCATTER) ? 1 : 0,
	FRF_AZ_TCP_UDP, is_udp,			FRF_AZ_TCP_UDP, is_udp,
	FRF_AZ_RXQ_ID, spec->efs_dmaq_id,			FRF_AZ_RXQ_ID, spec->fsfs_dmaq_id,
	EFX_DWORD_2, spec->efs_dword[2],			EFX_DWORD_2, spec->fsfs_dword[2],
	EFX_DWORD_1, spec->efs_dword[1],			EFX_DWORD_1, spec->fsfs_dword[1],
	EFX_DWORD_0, spec->efs_dword[0]);			EFX_DWORD_0, spec->fsfs_dword[0]);
	dword3 = is_udp;			dword3 = is_udp;
	break;			break;
	}			}

	#if EFSYS_OPT_SIENA			#if EFSYS_OPT_SIENA
	case EFX_FILTER_TBL_RX_MAC: {			case EFX_FS_FILTER_TBL_RX_MAC: {
	boolean_t is_wild = (type == EFX_FILTER_RX_MAC_WILD);			boolean_t is_wild = (type == EFX_FS_FILTER_RX_MAC_WILD);
	EFX_POPULATE_OWORD_8(*filter,			EFX_POPULATE_OWORD_7(*filter,
	FRF_CZ_RMFT_RSS_EN, (flags & EFX_FILTER_FLAG_RX_RSS) ? 1 : 0,			FRF_CZ_RMFT_RSS_EN,
	FRF_CZ_RMFT_SCATTER_EN, (flags & EFX_FILTER_FLAG_RX_SCATTER) ? 1 : 0,			(flags & EFX_FILTER_FLAG_RX_RSS) ? 1 : 0,
	FRF_CZ_RMFT_IP_OVERRIDE, (flags & EFX_FILTER_FLAG_RX_OVERRIDE_IP) ? 1 : 0,			FRF_CZ_RMFT_SCATTER_EN,
	FRF_CZ_RMFT_RXQ_ID, spec->efs_dmaq_id,			(flags & EFX_FILTER_FLAG_RX_SCATTER) ? 1 : 0,
				FRF_CZ_RMFT_RXQ_ID, spec->fsfs_dmaq_id,
	FRF_CZ_RMFT_WILDCARD_MATCH, is_wild,			FRF_CZ_RMFT_WILDCARD_MATCH, is_wild,
	FRF_CZ_RMFT_DEST_MAC_DW1, spec->efs_dword[2],			FRF_CZ_RMFT_DEST_MAC_DW1, spec->fsfs_dword[2],
	FRF_CZ_RMFT_DEST_MAC_DW0, spec->efs_dword[1],			FRF_CZ_RMFT_DEST_MAC_DW0, spec->fsfs_dword[1],
	FRF_CZ_RMFT_VLAN_ID, spec->efs_dword[0]);			FRF_CZ_RMFT_VLAN_ID, spec->fsfs_dword[0]);
	dword3 = is_wild;			dword3 = is_wild;
	break;			break;
	}			}
	#endif /* EFSYS_OPT_SIENA */			#endif /* EFSYS_OPT_SIENA */

	case EFX_FILTER_TBL_TX_IP: {			case EFX_FS_FILTER_TBL_TX_IP: {
	boolean_t is_udp = (type == EFX_FILTER_TX_UDP_FULL \|\|			boolean_t is_udp = (type == EFX_FS_FILTER_TX_UDP_FULL \|\|
	type == EFX_FILTER_TX_UDP_WILD);			type == EFX_FS_FILTER_TX_UDP_WILD);
	EFX_POPULATE_OWORD_5(*filter,			EFX_POPULATE_OWORD_5(*filter,
	FRF_CZ_TIFT_TCP_UDP, is_udp,			FRF_CZ_TIFT_TCP_UDP, is_udp,
	FRF_CZ_TIFT_TXQ_ID, spec->efs_dmaq_id,			FRF_CZ_TIFT_TXQ_ID, spec->fsfs_dmaq_id,
	EFX_DWORD_2, spec->efs_dword[2],			EFX_DWORD_2, spec->fsfs_dword[2],
	EFX_DWORD_1, spec->efs_dword[1],			EFX_DWORD_1, spec->fsfs_dword[1],
	EFX_DWORD_0, spec->efs_dword[0]);			EFX_DWORD_0, spec->fsfs_dword[0]);
	dword3 = is_udp \| spec->efs_dmaq_id << 1;			dword3 = is_udp \| spec->fsfs_dmaq_id << 1;
	break;			break;
	}			}

	#if EFSYS_OPT_SIENA			#if EFSYS_OPT_SIENA
	case EFX_FILTER_TBL_TX_MAC: {			case EFX_FS_FILTER_TBL_TX_MAC: {
	boolean_t is_wild = (type == EFX_FILTER_TX_MAC_WILD);			boolean_t is_wild = (type == EFX_FS_FILTER_TX_MAC_WILD);
	EFX_POPULATE_OWORD_5(*filter,			EFX_POPULATE_OWORD_5(*filter,
	FRF_CZ_TMFT_TXQ_ID, spec->efs_dmaq_id,			FRF_CZ_TMFT_TXQ_ID, spec->fsfs_dmaq_id,
	FRF_CZ_TMFT_WILDCARD_MATCH, is_wild,			FRF_CZ_TMFT_WILDCARD_MATCH, is_wild,
	FRF_CZ_TMFT_SRC_MAC_DW1, spec->efs_dword[2],			FRF_CZ_TMFT_SRC_MAC_DW1, spec->fsfs_dword[2],
	FRF_CZ_TMFT_SRC_MAC_DW0, spec->efs_dword[1],			FRF_CZ_TMFT_SRC_MAC_DW0, spec->fsfs_dword[1],
	FRF_CZ_TMFT_VLAN_ID, spec->efs_dword[0]);			FRF_CZ_TMFT_VLAN_ID, spec->fsfs_dword[0]);
	dword3 = is_wild \| spec->efs_dmaq_id << 1;			dword3 = is_wild \| spec->fsfs_dmaq_id << 1;
	break;			break;
	}			}
	#endif /* EFSYS_OPT_SIENA */			#endif /* EFSYS_OPT_SIENA */

	default:			default:
	EFSYS_ASSERT(B_FALSE);			EFSYS_ASSERT(B_FALSE);
				return (0);
	}			}

	key = spec->efs_dword[0] ^ spec->efs_dword[1] ^ spec->efs_dword[2] ^ dword3;			key =
				spec->fsfs_dword[0] ^
				spec->fsfs_dword[1] ^
				spec->fsfs_dword[2] ^
				dword3;

	return (key);			return (key);
	}			}

	static __checkReturn int			static __checkReturn int
	efx_filter_push_entry(			falconsiena_filter_push_entry(
	__inout efx_nic_t *enp,			__inout efx_nic_t *enp,
	__in efx_filter_type_t type,			__in falconsiena_filter_type_t type,
	__in int index,			__in int index,
	__in efx_oword_t *eop)			__in efx_oword_t *eop)
	{			{
	int rc;			int rc;

	switch (type)			switch (type) {
	{			case EFX_FS_FILTER_RX_TCP_FULL:
	case EFX_FILTER_RX_TCP_FULL:			case EFX_FS_FILTER_RX_TCP_WILD:
	case EFX_FILTER_RX_TCP_WILD:			case EFX_FS_FILTER_RX_UDP_FULL:
	case EFX_FILTER_RX_UDP_FULL:			case EFX_FS_FILTER_RX_UDP_WILD:
	case EFX_FILTER_RX_UDP_WILD:			EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_FILTER_TBL0, index,
	EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_FILTER_TBL0, index, eop);			eop, B_TRUE);
	break;			break;

	#if EFSYS_OPT_SIENA			#if EFSYS_OPT_SIENA
	case EFX_FILTER_RX_MAC_FULL:			case EFX_FS_FILTER_RX_MAC_FULL:
	case EFX_FILTER_RX_MAC_WILD:			case EFX_FS_FILTER_RX_MAC_WILD:
	EFX_BAR_TBL_WRITEO(enp, FR_CZ_RX_MAC_FILTER_TBL0, index, eop);			EFX_BAR_TBL_WRITEO(enp, FR_CZ_RX_MAC_FILTER_TBL0, index,
				eop, B_TRUE);
	break;			break;

	case EFX_FILTER_TX_TCP_FULL:			case EFX_FS_FILTER_TX_TCP_FULL:
	case EFX_FILTER_TX_TCP_WILD:			case EFX_FS_FILTER_TX_TCP_WILD:
	case EFX_FILTER_TX_UDP_FULL:			case EFX_FS_FILTER_TX_UDP_FULL:
	case EFX_FILTER_TX_UDP_WILD:			case EFX_FS_FILTER_TX_UDP_WILD:
	EFX_BAR_TBL_WRITEO(enp, FR_CZ_TX_FILTER_TBL0, index, eop);			EFX_BAR_TBL_WRITEO(enp, FR_CZ_TX_FILTER_TBL0, index,
				eop, B_TRUE);
	break;			break;

	case EFX_FILTER_TX_MAC_FULL:			case EFX_FS_FILTER_TX_MAC_FULL:
	case EFX_FILTER_TX_MAC_WILD:			case EFX_FS_FILTER_TX_MAC_WILD:
	EFX_BAR_TBL_WRITEO(enp, FR_CZ_TX_MAC_FILTER_TBL0, index, eop);			EFX_BAR_TBL_WRITEO(enp, FR_CZ_TX_MAC_FILTER_TBL0, index,
				eop, B_TRUE);
	break;			break;
	#endif /* EFSYS_OPT_SIENA */			#endif /* EFSYS_OPT_SIENA */

	default:			default:
				EFSYS_ASSERT(B_FALSE);
	rc = ENOTSUP;			rc = ENOTSUP;
	goto fail1;			goto fail1;
	}			}
	return (0);			return (0);

	fail1:			fail1:
	return (rc);			return (rc);
	}			}


	static __checkReturn boolean_t			static __checkReturn boolean_t
	efx_filter_equal(			falconsiena_filter_equal(
	__in const efx_filter_spec_t *left,			__in const falconsiena_filter_spec_t *left,
	__in const efx_filter_spec_t *right)			__in const falconsiena_filter_spec_t *right)
	{			{
	efx_filter_tbl_id_t tbl_id = efx_filter_tbl_id(left->efs_type);			falconsiena_filter_tbl_id_t tbl_id;

	if (left->efs_type != right->efs_type)			tbl_id = falconsiena_filter_tbl_id(left->fsfs_type);


				if (left->fsfs_type != right->fsfs_type)
	return (B_FALSE);			return (B_FALSE);

	if (memcmp(left->efs_dword, right->efs_dword, sizeof(left->efs_dword)))			if (memcmp(left->fsfs_dword, right->fsfs_dword,
				sizeof (left->fsfs_dword)))
	return (B_FALSE);			return (B_FALSE);

	if ((tbl_id == EFX_FILTER_TBL_TX_IP \|\|			if ((tbl_id == EFX_FS_FILTER_TBL_TX_IP \|\|
	tbl_id == EFX_FILTER_TBL_TX_MAC) &&			tbl_id == EFX_FS_FILTER_TBL_TX_MAC) &&
	left->efs_dmaq_id != right->efs_dmaq_id)			left->fsfs_dmaq_id != right->fsfs_dmaq_id)
	return (B_FALSE);			return (B_FALSE);

	return (B_TRUE);			return (B_TRUE);
	}			}

	static __checkReturn int			static __checkReturn int
	efx_filter_search(			falconsiena_filter_search(
	__in efx_filter_tbl_t *eftp,			__in falconsiena_filter_tbl_t *fsftp,
	__in efx_filter_spec_t *spec,			__in falconsiena_filter_spec_t *spec,
	__in uint32_t key,			__in uint32_t key,
	__in boolean_t for_insert,			__in boolean_t for_insert,
	__out int *filter_index,			__out int *filter_index,
	__out unsigned int *depth_required)			__out unsigned int *depth_required)
	{			{
	unsigned hash, incr, filter_idx, depth;			unsigned hash, incr, filter_idx, depth;

	hash = efx_filter_tbl_hash(key);			hash = falconsiena_filter_tbl_hash(key);
	incr = efx_filter_tbl_increment(key);			incr = falconsiena_filter_tbl_increment(key);

	filter_idx = hash & (eftp->eft_size - 1);			filter_idx = hash & (fsftp->fsft_size - 1);
	depth = 1;			depth = 1;

	for (;;) {			for (;;) {
	/* Return success if entry is used and matches this spec			/*
				* Return success if entry is used and matches this spec
	* or entry is unused and we are trying to insert.			* or entry is unused and we are trying to insert.
	*/			*/
	if (efx_filter_test_used(eftp, filter_idx) ?			if (falconsiena_filter_test_used(fsftp, filter_idx) ?
	efx_filter_equal(spec, &eftp->eft_spec[filter_idx]) :			falconsiena_filter_equal(spec,
				&fsftp->fsft_spec[filter_idx]) :
	for_insert) {			for_insert) {
	*filter_index = filter_idx;			*filter_index = filter_idx;
	*depth_required = depth;			*depth_required = depth;
	return (0);			return (0);
	}			}

	/* Return failure if we reached the maximum search depth */			/* Return failure if we reached the maximum search depth */
	if (depth == FILTER_CTL_SRCH_MAX)			if (depth == FILTER_CTL_SRCH_MAX)
	return for_insert ? EBUSY : ENOENT;			return (for_insert ? EBUSY : ENOENT);

	filter_idx = (filter_idx + incr) & (eftp->eft_size - 1);			filter_idx = (filter_idx + incr) & (fsftp->fsft_size - 1);
	++depth;			++depth;
	}			}
	}			}

	__checkReturn int
	efx_filter_insert_filter(
	__in efx_nic_t *enp,
	__in efx_filter_spec_t *spec,
	__in boolean_t replace)
	{
	efx_filter_t *efp = &enp->en_filter;
	efx_filter_tbl_id_t tbl_id = efx_filter_tbl_id(spec->efs_type);
	efx_filter_tbl_t *eftp = &efp->ef_tbl[tbl_id];
	efx_filter_spec_t *saved_spec;
	efx_oword_t filter;
	int filter_idx;
	unsigned int depth;
	int state;
	uint32_t key;
	int rc;

	if (eftp->eft_size == 0)
	return (EINVAL);

	key = efx_filter_build(&filter, spec);

	EFSYS_LOCK(enp->en_eslp, state);

	rc = efx_filter_search(eftp, spec, key, B_TRUE, &filter_idx, &depth);
	if (rc != 0)
	goto done;

	EFSYS_ASSERT3U(filter_idx, <, eftp->eft_size);
	saved_spec = &eftp->eft_spec[filter_idx];

	if (efx_filter_test_used(eftp, filter_idx)) {
	if (replace == B_FALSE) {
	rc = EEXIST;
	goto done;
	}
	}
	efx_filter_set_used(eftp, filter_idx);
	saved_spec = spec;

	if (efp->ef_depth[spec->efs_type] < depth) {
	efp->ef_depth[spec->efs_type] = depth;
	if (tbl_id == EFX_FILTER_TBL_TX_IP \|\|
	tbl_id == EFX_FILTER_TBL_TX_MAC)
	efx_filter_push_tx_limits(enp);
	else
	efx_filter_push_rx_limits(enp);
	}

	efx_filter_push_entry(enp, spec->efs_type, filter_idx, &filter);

	done:
	EFSYS_UNLOCK(enp->en_eslp, state);
	return (rc);
	}

	static void			static void
	efx_filter_clear_entry(			falconsiena_filter_clear_entry(
	__in efx_nic_t *enp,			__in efx_nic_t *enp,
	__in efx_filter_tbl_t *eftp,			__in falconsiena_filter_tbl_t *fsftp,
	__in int index)			__in int index)
	{			{
	efx_oword_t filter;			efx_oword_t filter;

	if (efx_filter_test_used(eftp, index)) {			if (falconsiena_filter_test_used(fsftp, index)) {
	efx_filter_clear_used(eftp, index);			falconsiena_filter_clear_used(fsftp, index);

	EFX_ZERO_OWORD(filter);			EFX_ZERO_OWORD(filter);
	efx_filter_push_entry(enp, eftp->eft_spec[index].efs_type,			falconsiena_filter_push_entry(enp,
				fsftp->fsft_spec[index].fsfs_type,
	index, &filter);			index, &filter);

	memset(&eftp->eft_spec[index], 0, sizeof(eftp->eft_spec[0]));			memset(&fsftp->fsft_spec[index],
				0, sizeof (fsftp->fsft_spec[0]));
	}			}
	}			}

	__checkReturn int			void
	efx_filter_remove_filter(			falconsiena_filter_tbl_clear(
	__in efx_nic_t *enp,			__in efx_nic_t *enp,
	__in efx_filter_spec_t *spec)			__in falconsiena_filter_tbl_id_t tbl_id)
	{			{
	efx_filter_t *efp = &enp->en_filter;			falconsiena_filter_t *fsfp = enp->en_filter.ef_falconsiena_filter;
	efx_filter_tbl_id_t tbl_id = efx_filter_tbl_id(spec->efs_type);			falconsiena_filter_tbl_t *fsftp = &fsfp->fsf_tbl[tbl_id];
	efx_filter_tbl_t *eftp = &efp->ef_tbl[tbl_id];
	efx_filter_spec_t *saved_spec;
	efx_oword_t filter;
	int filter_idx;
	unsigned int depth;
	int state;
	uint32_t key;
	int rc;

	key = efx_filter_build(&filter, spec);

	EFSYS_LOCK(enp->en_eslp, state);

	rc = efx_filter_search(eftp, spec, key, B_FALSE, &filter_idx, &depth);
	if (rc != 0)
	goto out;

	saved_spec = &eftp->eft_spec[filter_idx];

	efx_filter_clear_entry(enp, eftp, filter_idx);
	if (eftp->eft_used == 0)
	efx_filter_reset_search_depth(efp, tbl_id);

	rc = 0;

	out:
	EFSYS_UNLOCK(enp->en_eslp, state);
	return (rc);
	}

	void
	efx_filter_remove_index(
	__inout efx_nic_t *enp,
	__in efx_filter_type_t type,
	__in int index)
	{
	efx_filter_t *efp = &enp->en_filter;
	efx_filter_tbl_id_t tbl_id = efx_filter_tbl_id(type);
	efx_filter_tbl_t *eftp = &efp->ef_tbl[tbl_id];
	int state;

	if (index < 0)
	return;

	EFSYS_LOCK(enp->en_eslp, state);

	efx_filter_clear_entry(enp, eftp, index);
	if (eftp->eft_used == 0)
	efx_filter_reset_search_depth(efp, tbl_id);

	EFSYS_UNLOCK(enp->en_eslp, state);
	}

	void
	efx_filter_tbl_clear(
	__inout efx_nic_t *enp,
	__in efx_filter_tbl_id_t tbl_id)
	{
	efx_filter_t *efp = &enp->en_filter;
	efx_filter_tbl_t *eftp = &efp->ef_tbl[tbl_id];
	int index;			int index;
	int state;			int state;

	EFSYS_LOCK(enp->en_eslp, state);			EFSYS_LOCK(enp->en_eslp, state);

	for (index = 0; index < eftp->eft_size; ++index) {			for (index = 0; index < fsftp->fsft_size; ++index) {
	efx_filter_clear_entry(enp, eftp, index);			falconsiena_filter_clear_entry(enp, fsftp, index);
	}			}

	if (eftp->eft_used == 0)			if (fsftp->fsft_used == 0)
	efx_filter_reset_search_depth(efp, tbl_id);			falconsiena_filter_reset_search_depth(fsfp, tbl_id);

	EFSYS_UNLOCK(enp->en_eslp, state);			EFSYS_UNLOCK(enp->en_eslp, state);
	}			}

	/* Restore filter state after a reset */			static __checkReturn int
	void			falconsiena_filter_init(
	efx_filter_restore(
	__in efx_nic_t *enp)			__in efx_nic_t *enp)
	{			{
	efx_filter_t *efp = &enp->en_filter;			falconsiena_filter_t *fsfp;
	efx_filter_tbl_id_t tbl_id;			falconsiena_filter_tbl_t *fsftp;
	efx_filter_tbl_t *eftp;			int tbl_id;
	efx_filter_spec_t *spec;			int rc;
	efx_oword_t filter;
	int filter_idx;
	int state;

	EFSYS_LOCK(enp->en_eslp, state);			EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (falconsiena_filter_t), fsfp);

	for (tbl_id = 0; tbl_id < EFX_FILTER_NTBLS; tbl_id++) {			if (!fsfp) {
	eftp = &efp->ef_tbl[tbl_id];			rc = ENOMEM;
	for (filter_idx = 0; filter_idx < eftp->eft_size; filter_idx++) {			goto fail1;
	if (!efx_filter_test_used(eftp, filter_idx))
	continue;

	spec = &eftp->eft_spec[filter_idx];
	efx_filter_build(&filter, spec);
	efx_filter_push_entry(enp, spec->efs_type,
	filter_idx, &filter);
	}			}
	}

	efx_filter_push_rx_limits(enp);			enp->en_filter.ef_falconsiena_filter = fsfp;
	efx_filter_push_tx_limits(enp);

	EFSYS_UNLOCK(enp->en_eslp, state);			switch (enp->en_family) {
	}

	void
	efx_filter_redirect_index(
	__inout efx_nic_t *enp,
	__in efx_filter_type_t type,
	__in int filter_index,
	__in int rxq_index)
	{
	efx_filter_t *efp = &enp->en_filter;
	efx_filter_tbl_t *eftp =
	&efp->ef_tbl[efx_filter_tbl_id(type)];
	efx_filter_spec_t *spec;
	efx_oword_t filter;
	int state;

	EFSYS_LOCK(enp->en_eslp, state);

	spec = &eftp->eft_spec[filter_index];
	spec->efs_dmaq_id = (uint16_t)rxq_index;

	efx_filter_build(&filter, spec);
	efx_filter_push_entry(enp, spec->efs_type, filter_index, &filter);

	EFSYS_UNLOCK(enp->en_eslp, state);
	}

	__checkReturn int
	efx_filter_init(
	__in efx_nic_t *enp)
	{
	efx_filter_t *efp = &enp->en_filter;
	efx_filter_tbl_t *eftp;
	int tbl_id;
	int rc;

	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_FILTER));

	switch (enp->en_family)
	{
	#if EFSYS_OPT_FALCON			#if EFSYS_OPT_FALCON
	case EFX_FAMILY_FALCON:			case EFX_FAMILY_FALCON:
	eftp = &efp->ef_tbl[EFX_FILTER_TBL_RX_IP];			fsftp = &fsfp->fsf_tbl[EFX_FS_FILTER_TBL_RX_IP];
	eftp->eft_size = FR_AZ_RX_FILTER_TBL0_ROWS;			fsftp->fsft_size = FR_AZ_RX_FILTER_TBL0_ROWS;
	break;			break;
	#endif /* EFSYS_OPT_FALCON */			#endif /* EFSYS_OPT_FALCON */

	#if EFSYS_OPT_SIENA			#if EFSYS_OPT_SIENA
	case EFX_FAMILY_SIENA:			case EFX_FAMILY_SIENA:
	eftp = &efp->ef_tbl[EFX_FILTER_TBL_RX_IP];			fsftp = &fsfp->fsf_tbl[EFX_FS_FILTER_TBL_RX_IP];
	eftp->eft_size = FR_AZ_RX_FILTER_TBL0_ROWS;			fsftp->fsft_size = FR_AZ_RX_FILTER_TBL0_ROWS;

	eftp = &efp->ef_tbl[EFX_FILTER_TBL_RX_MAC];			fsftp = &fsfp->fsf_tbl[EFX_FS_FILTER_TBL_RX_MAC];
	eftp->eft_size = FR_CZ_RX_MAC_FILTER_TBL0_ROWS;			fsftp->fsft_size = FR_CZ_RX_MAC_FILTER_TBL0_ROWS;

	eftp = &efp->ef_tbl[EFX_FILTER_TBL_TX_IP];			fsftp = &fsfp->fsf_tbl[EFX_FS_FILTER_TBL_TX_IP];
	eftp->eft_size = FR_CZ_TX_FILTER_TBL0_ROWS;			fsftp->fsft_size = FR_CZ_TX_FILTER_TBL0_ROWS;

	eftp = &efp->ef_tbl[EFX_FILTER_TBL_TX_MAC];			fsftp = &fsfp->fsf_tbl[EFX_FS_FILTER_TBL_TX_MAC];
	eftp->eft_size = FR_CZ_TX_MAC_FILTER_TBL0_ROWS;			fsftp->fsft_size = FR_CZ_TX_MAC_FILTER_TBL0_ROWS;
	break;			break;
	#endif /* EFSYS_OPT_SIENA */			#endif /* EFSYS_OPT_SIENA */

	default:			default:
	rc = ENOTSUP;			rc = ENOTSUP;
	goto fail1;			goto fail2;
	}			}

	for (tbl_id = 0; tbl_id < EFX_FILTER_NTBLS; tbl_id++) {			for (tbl_id = 0; tbl_id < EFX_FS_FILTER_NTBLS; tbl_id++) {
	unsigned int bitmap_size;			unsigned int bitmap_size;

	eftp = &efp->ef_tbl[tbl_id];			fsftp = &fsfp->fsf_tbl[tbl_id];
	if (eftp->eft_size == 0)			if (fsftp->fsft_size == 0)
	continue;			continue;

	EFX_STATIC_ASSERT(sizeof(eftp->eft_bitmap[0]) == sizeof(uint32_t));			EFX_STATIC_ASSERT(sizeof (fsftp->fsft_bitmap[0]) ==
	bitmap_size = (eftp->eft_size + (sizeof(uint32_t) * 8) - 1) / 8;			sizeof (uint32_t));
				bitmap_size =
				(fsftp->fsft_size + (sizeof (uint32_t) * 8) - 1) / 8;

	EFSYS_KMEM_ALLOC(enp->en_esip, bitmap_size, eftp->eft_bitmap);			EFSYS_KMEM_ALLOC(enp->en_esip, bitmap_size, fsftp->fsft_bitmap);
	if (!eftp->eft_bitmap) {			if (!fsftp->fsft_bitmap) {
	rc = ENOMEM;			rc = ENOMEM;
	goto fail2;			goto fail3;
	}			}

	EFSYS_KMEM_ALLOC(enp->en_esip, eftp->eft_size * sizeof(*eftp->eft_spec),			EFSYS_KMEM_ALLOC(enp->en_esip,
	eftp->eft_spec);			fsftp->fsft_size * sizeof (*fsftp->fsft_spec),
	if (!eftp->eft_spec) {			fsftp->fsft_spec);
				if (!fsftp->fsft_spec) {
	rc = ENOMEM;			rc = ENOMEM;
	goto fail3;			goto fail4;
	}			}
	memset(eftp->eft_spec, 0, eftp->eft_size * sizeof(*eftp->eft_spec));			memset(fsftp->fsft_spec, 0,
				fsftp->fsft_size * sizeof (*fsftp->fsft_spec));
	}			}
	enp->en_mod_flags \|= EFX_MOD_FILTER;

	return (0);			return (0);

				fail4:
				EFSYS_PROBE(fail4);

	fail3:			fail3:
	EFSYS_PROBE(fail3);			EFSYS_PROBE(fail3);

	fail2:			fail2:
	EFSYS_PROBE(fail2);			EFSYS_PROBE(fail2);
	efx_filter_fini(enp);			falconsiena_filter_fini(enp);

	fail1:			fail1:
	EFSYS_PROBE1(fail1, int, rc);			EFSYS_PROBE1(fail1, int, rc);
	return (rc);			return (rc);
	}			}

	void			static void
	efx_filter_fini(			falconsiena_filter_fini(
	__in efx_nic_t *enp)			__in efx_nic_t *enp)
	{			{
	efx_filter_t *efp = &enp->en_filter;			falconsiena_filter_t *fsfp = enp->en_filter.ef_falconsiena_filter;
	efx_filter_tbl_id_t tbl_id;			falconsiena_filter_tbl_id_t tbl_id;

	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);			EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);			EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);

	for (tbl_id = 0; tbl_id < EFX_FILTER_NTBLS; tbl_id++) {			if (fsfp == NULL)
	efx_filter_tbl_t *eftp = &efp->ef_tbl[tbl_id];			return;

				for (tbl_id = 0; tbl_id < EFX_FS_FILTER_NTBLS; tbl_id++) {
				falconsiena_filter_tbl_t *fsftp = &fsfp->fsf_tbl[tbl_id];
	unsigned int bitmap_size;			unsigned int bitmap_size;

	EFX_STATIC_ASSERT(sizeof(eftp->eft_bitmap[0]) == sizeof(uint32_t));			EFX_STATIC_ASSERT(sizeof (fsftp->fsft_bitmap[0]) ==
	bitmap_size = (eftp->eft_size + (sizeof(uint32_t) * 8) - 1) / 8;			sizeof (uint32_t));
				bitmap_size =
				(fsftp->fsft_size + (sizeof (uint32_t) * 8) - 1) / 8;

	if (eftp->eft_bitmap != NULL) {			if (fsftp->fsft_bitmap != NULL) {
	EFSYS_KMEM_FREE(enp->en_esip, bitmap_size,			EFSYS_KMEM_FREE(enp->en_esip, bitmap_size,
	eftp->eft_bitmap);			fsftp->fsft_bitmap);
	eftp->eft_bitmap = NULL;			fsftp->fsft_bitmap = NULL;
	}			}

	if (eftp->eft_spec != NULL) {			if (fsftp->fsft_spec != NULL) {
	EFSYS_KMEM_FREE(enp->en_esip, eftp->eft_size *			EFSYS_KMEM_FREE(enp->en_esip, fsftp->fsft_size *
	sizeof(*eftp->eft_spec), eftp->eft_spec);			sizeof (*fsftp->fsft_spec), fsftp->fsft_spec);
	eftp->eft_spec = NULL;			fsftp->fsft_spec = NULL;
	}			}
	}			}

	enp->en_mod_flags &= ~EFX_MOD_FILTER;			EFSYS_KMEM_FREE(enp->en_esip, sizeof (falconsiena_filter_t),
				enp->en_filter.ef_falconsiena_filter);
	}			}

	extern void			/* Restore filter state after a reset */
	efx_filter_spec_rx_ipv4_tcp_full(			static __checkReturn int
	__inout efx_filter_spec_t *spec,			falconsiena_filter_restore(
	__in unsigned int flags,			__in efx_nic_t *enp)
	__in uint32_t src_ip,
	__in uint16_t src_tcp,
	__in uint32_t dest_ip,
	__in uint16_t dest_tcp)
	{			{
	EFSYS_ASSERT3P(spec, !=, NULL);			falconsiena_filter_t *fsfp = enp->en_filter.ef_falconsiena_filter;
	EFSYS_ASSERT((flags & ~(EFX_FILTER_FLAG_RX_RSS \|			falconsiena_filter_tbl_id_t tbl_id;
	EFX_FILTER_FLAG_RX_SCATTER)) == 0);			falconsiena_filter_tbl_t *fsftp;
				falconsiena_filter_spec_t *spec;
				efx_oword_t filter;
				int filter_idx;
				int state;
				int rc;

	spec->efs_type = EFX_FILTER_RX_TCP_FULL;			EFSYS_LOCK(enp->en_eslp, state);
	spec->efs_flags = (uint8_t)flags;
	spec->efs_dword[0] = src_tcp \| src_ip << 16;
	spec->efs_dword[1] = dest_tcp << 16 \| src_ip >> 16;
	spec->efs_dword[2] = dest_ip;
	}

	extern void			for (tbl_id = 0; tbl_id < EFX_FS_FILTER_NTBLS; tbl_id++) {
	efx_filter_spec_rx_ipv4_tcp_wild(			fsftp = &fsfp->fsf_tbl[tbl_id];
	__inout efx_filter_spec_t *spec,			for (filter_idx = 0;
	__in unsigned int flags,			filter_idx < fsftp->fsft_size;
	__in uint32_t dest_ip,			filter_idx++) {
	__in uint16_t dest_tcp)			if (!falconsiena_filter_test_used(fsftp, filter_idx))
	{			continue;
	EFSYS_ASSERT3P(spec, !=, NULL);
	EFSYS_ASSERT((flags & ~(EFX_FILTER_FLAG_RX_RSS \|
	EFX_FILTER_FLAG_RX_SCATTER)) == 0);

	spec->efs_type = EFX_FILTER_RX_TCP_WILD;			spec = &fsftp->fsft_spec[filter_idx];
	spec->efs_flags = (uint8_t)flags;			if ((rc = falconsiena_filter_build(&filter, spec)) != 0)
	spec->efs_dword[0] = 0;			goto fail1;
	spec->efs_dword[1] = dest_tcp << 16;			if ((rc = falconsiena_filter_push_entry(enp,
	spec->efs_dword[2] = dest_ip;			spec->fsfs_type, filter_idx, &filter)) != 0)
				goto fail2;
	}			}
				}

	extern void			falconsiena_filter_push_rx_limits(enp);
	efx_filter_spec_rx_ipv4_udp_full(			falconsiena_filter_push_tx_limits(enp);
	__inout efx_filter_spec_t *spec,
	__in unsigned int flags,
	__in uint32_t src_ip,
	__in uint16_t src_udp,
	__in uint32_t dest_ip,
	__in uint16_t dest_udp)
	{
	EFSYS_ASSERT3P(spec, !=, NULL);
	EFSYS_ASSERT((flags & ~(EFX_FILTER_FLAG_RX_RSS \|
	EFX_FILTER_FLAG_RX_SCATTER)) == 0);

	spec->efs_type = EFX_FILTER_RX_UDP_FULL;			EFSYS_UNLOCK(enp->en_eslp, state);
	spec->efs_flags = (uint8_t)flags;
	spec->efs_dword[0] = src_udp \| src_ip << 16;
	spec->efs_dword[1] = dest_udp << 16 \| src_ip >> 16;
	spec->efs_dword[2] = dest_ip;
	}

	extern void			return (0);
	efx_filter_spec_rx_ipv4_udp_wild(
	__inout efx_filter_spec_t *spec,
	__in unsigned int flags,
	__in uint32_t dest_ip,
	__in uint16_t dest_udp)
	{
	EFSYS_ASSERT3P(spec, !=, NULL);
	EFSYS_ASSERT((flags & ~(EFX_FILTER_FLAG_RX_RSS \|
	EFX_FILTER_FLAG_RX_SCATTER)) == 0);

	spec->efs_type = EFX_FILTER_RX_UDP_WILD;			fail2:
	spec->efs_flags = (uint8_t)flags;			EFSYS_PROBE(fail2);
	spec->efs_dword[0] = dest_udp;
	spec->efs_dword[1] = 0;
	spec->efs_dword[2] = dest_ip;
	}

	#if EFSYS_OPT_SIENA			fail1:
	extern void			EFSYS_PROBE1(fail1, int, rc);
	efx_filter_spec_rx_mac_full(
	__inout efx_filter_spec_t *spec,
	__in unsigned int flags,
	__in uint16_t vlan_id,
	__in uint8_t *dest_mac)
	{
	EFSYS_ASSERT3P(spec, !=, NULL);
	EFSYS_ASSERT3P(dest_mac, !=, NULL);
	EFSYS_ASSERT((flags & ~(EFX_FILTER_FLAG_RX_RSS \|
	EFX_FILTER_FLAG_RX_SCATTER \|
	EFX_FILTER_FLAG_RX_OVERRIDE_IP)) == 0);

	spec->efs_type = EFX_FILTER_RX_MAC_FULL;			EFSYS_UNLOCK(enp->en_eslp, state);
	spec->efs_flags = (uint8_t)flags;
	spec->efs_dword[0] = vlan_id;			return (rc);
	spec->efs_dword[1] =
	dest_mac[2] << 24 \|
	dest_mac[3] << 16 \|
	dest_mac[4] << 8 \|
	dest_mac[5];
	spec->efs_dword[2] =
	dest_mac[0] << 8 \|
	dest_mac[1];
	}			}
	#endif /* EFSYS_OPT_SIENA */

	#if EFSYS_OPT_SIENA			static __checkReturn int
	extern void			falconsiena_filter_add(
	efx_filter_spec_rx_mac_wild(			__in efx_nic_t *enp,
	__inout efx_filter_spec_t *spec,			__inout efx_filter_spec_t *spec,
	__in unsigned int flags,			__in boolean_t may_replace)
	__in uint8_t *dest_mac)
	{			{
				int rc;
				falconsiena_filter_spec_t fs_spec;
				falconsiena_filter_t *fsfp = enp->en_filter.ef_falconsiena_filter;
				falconsiena_filter_tbl_id_t tbl_id;
				falconsiena_filter_tbl_t *fsftp;
				falconsiena_filter_spec_t *saved_fs_spec;
				efx_oword_t filter;
				int filter_idx;
				unsigned int depth;
				int state;
				uint32_t key;


	EFSYS_ASSERT3P(spec, !=, NULL);			EFSYS_ASSERT3P(spec, !=, NULL);
	EFSYS_ASSERT3P(dest_mac, !=, NULL);
	EFSYS_ASSERT((flags & ~(EFX_FILTER_FLAG_RX_RSS \|
	EFX_FILTER_FLAG_RX_SCATTER \|
	EFX_FILTER_FLAG_RX_OVERRIDE_IP)) == 0);

	spec->efs_type = EFX_FILTER_RX_MAC_WILD;			if ((rc = falconsiena_filter_spec_from_gen_spec(&fs_spec, spec)) != 0)
	spec->efs_flags = (uint8_t)flags;			goto fail1;
	spec->efs_dword[0] = 0;
	spec->efs_dword[1] =			tbl_id = falconsiena_filter_tbl_id(fs_spec.fsfs_type);
	dest_mac[2] << 24 \|			fsftp = &fsfp->fsf_tbl[tbl_id];
	dest_mac[3] << 16 \|
	dest_mac[4] << 8 \|			if (fsftp->fsft_size == 0) {
	dest_mac[5];			rc = EINVAL;
	spec->efs_dword[2] =			goto fail2;
	dest_mac[0] << 8 \|
	dest_mac[1];
	}			}
	#endif /* EFSYS_OPT_SIENA */

	#if EFSYS_OPT_SIENA			key = falconsiena_filter_build(&filter, &fs_spec);
	extern void
	efx_filter_spec_tx_ipv4_tcp_full(
	__inout efx_filter_spec_t *spec,
	__in uint32_t src_ip,
	__in uint16_t src_tcp,
	__in uint32_t dest_ip,
	__in uint16_t dest_tcp)
	{
	EFSYS_ASSERT3P(spec, !=, NULL);

	spec->efs_type = EFX_FILTER_TX_TCP_FULL;			EFSYS_LOCK(enp->en_eslp, state);
	spec->efs_flags = 0;
	spec->efs_dword[0] = src_tcp \| src_ip << 16;			rc = falconsiena_filter_search(fsftp, &fs_spec, key, B_TRUE,
	spec->efs_dword[1] = dest_tcp << 16 \| src_ip >> 16;			&filter_idx, &depth);
	spec->efs_dword[2] = dest_ip;			if (rc != 0)
				goto fail3;

				EFSYS_ASSERT3U(filter_idx, <, fsftp->fsft_size);
				saved_fs_spec = &fsftp->fsft_spec[filter_idx];

				if (falconsiena_filter_test_used(fsftp, filter_idx)) {
				if (may_replace == B_FALSE) {
				rc = EEXIST;
				goto fail4;
	}			}
	#endif /* EFSYS_OPT_SIENA */			}
				falconsiena_filter_set_used(fsftp, filter_idx);
				*saved_fs_spec = fs_spec;

	#if EFSYS_OPT_SIENA			if (fsfp->fsf_depth[fs_spec.fsfs_type] < depth) {
	extern void			fsfp->fsf_depth[fs_spec.fsfs_type] = depth;
	efx_filter_spec_tx_ipv4_tcp_wild(			if (tbl_id == EFX_FS_FILTER_TBL_TX_IP \|\|
	__inout efx_filter_spec_t *spec,			tbl_id == EFX_FS_FILTER_TBL_TX_MAC)
	__in uint32_t src_ip,			falconsiena_filter_push_tx_limits(enp);
	__in uint16_t src_tcp)			else
	{			falconsiena_filter_push_rx_limits(enp);
	EFSYS_ASSERT3P(spec, !=, NULL);			}

	spec->efs_type = EFX_FILTER_TX_TCP_WILD;			falconsiena_filter_push_entry(enp, fs_spec.fsfs_type,
	spec->efs_flags = 0;			filter_idx, &filter);
	spec->efs_dword[0] = 0;
	spec->efs_dword[1] = src_tcp << 16;			EFSYS_UNLOCK(enp->en_eslp, state);
	spec->efs_dword[2] = src_ip;			return (0);

				fail4:
				EFSYS_PROBE(fail4);

				fail3:
				EFSYS_UNLOCK(enp->en_eslp, state);
				EFSYS_PROBE(fail3);

				fail2:
				EFSYS_PROBE(fail2);

				fail1:
				EFSYS_PROBE1(fail1, int, rc);
				return (rc);
	}			}
	#endif /* EFSYS_OPT_SIENA */

	#if EFSYS_OPT_SIENA			static __checkReturn int
	extern void			falconsiena_filter_delete(
	efx_filter_spec_tx_ipv4_udp_full(			__in efx_nic_t *enp,
	__inout efx_filter_spec_t *spec,			__inout efx_filter_spec_t *spec)
	__in uint32_t src_ip,
	__in uint16_t src_udp,
	__in uint32_t dest_ip,
	__in uint16_t dest_udp)
	{			{
				int rc;
				falconsiena_filter_spec_t fs_spec;
				falconsiena_filter_t *fsfp = enp->en_filter.ef_falconsiena_filter;
				falconsiena_filter_tbl_id_t tbl_id;
				falconsiena_filter_tbl_t *fsftp;
				falconsiena_filter_spec_t *saved_spec;
				efx_oword_t filter;
				int filter_idx;
				unsigned int depth;
				int state;
				uint32_t key;

	EFSYS_ASSERT3P(spec, !=, NULL);			EFSYS_ASSERT3P(spec, !=, NULL);

	spec->efs_type = EFX_FILTER_TX_UDP_FULL;			if ((rc = falconsiena_filter_spec_from_gen_spec(&fs_spec, spec)) != 0)
	spec->efs_flags = 0;			goto fail1;
	spec->efs_dword[0] = src_udp \| src_ip << 16;
	spec->efs_dword[1] = dest_udp << 16 \| src_ip >> 16;			tbl_id = falconsiena_filter_tbl_id(fs_spec.fsfs_type);
	spec->efs_dword[2] = dest_ip;			fsftp = &fsfp->fsf_tbl[tbl_id];

				key = falconsiena_filter_build(&filter, &fs_spec);

				EFSYS_LOCK(enp->en_eslp, state);

				rc = falconsiena_filter_search(fsftp, &fs_spec, key, B_FALSE,
				&filter_idx, &depth);
				if (rc != 0)
				goto fail2;

				saved_spec = &fsftp->fsft_spec[filter_idx];

				falconsiena_filter_clear_entry(enp, fsftp, filter_idx);
				if (fsftp->fsft_used == 0)
				falconsiena_filter_reset_search_depth(fsfp, tbl_id);

				EFSYS_UNLOCK(enp->en_eslp, state);
				return (0);

				fail2:
				EFSYS_UNLOCK(enp->en_eslp, state);
				EFSYS_PROBE(fail2);

				fail1:
				EFSYS_PROBE1(fail1, int, rc);
				return (rc);
	}			}
	#endif /* EFSYS_OPT_SIENA */

	#if EFSYS_OPT_SIENA			#define MAX_SUPPORTED 4
	extern void
	efx_filter_spec_tx_ipv4_udp_wild(			static __checkReturn int
	__inout efx_filter_spec_t *spec,			falconsiena_filter_supported_filters(
	__in uint32_t src_ip,			__in efx_nic_t *enp,
	__in uint16_t src_udp)			__out uint32_t *list,
				__out size_t *length)
	{			{
	EFSYS_ASSERT3P(spec, !=, NULL);			int index = 0;
				uint32_t rx_matches[MAX_SUPPORTED];
				int rc;

	spec->efs_type = EFX_FILTER_TX_UDP_WILD;			if (list == NULL) {
	spec->efs_flags = 0;			rc = EINVAL;
	spec->efs_dword[0] = src_udp;			goto fail1;
	spec->efs_dword[1] = 0;
	spec->efs_dword[2] = src_ip;
	}			}
	#endif /* EFSYS_OPT_SIENA */

	#if EFSYS_OPT_SIENA			rx_matches[index++] =
	extern void			EFX_FILTER_MATCH_ETHER_TYPE \| EFX_FILTER_MATCH_IP_PROTO \|
	efx_filter_spec_tx_mac_full(			EFX_FILTER_MATCH_LOC_HOST \| EFX_FILTER_MATCH_LOC_PORT \|
	__inout efx_filter_spec_t *spec,			EFX_FILTER_MATCH_REM_HOST \| EFX_FILTER_MATCH_REM_PORT;
	__in uint16_t vlan_id,
	__in uint8_t *src_mac)
	{
	EFSYS_ASSERT3P(spec, !=, NULL);
	EFSYS_ASSERT3P(src_mac, !=, NULL);

	spec->efs_type = EFX_FILTER_TX_MAC_FULL;			rx_matches[index++] =
	spec->efs_flags = 0;			EFX_FILTER_MATCH_ETHER_TYPE \| EFX_FILTER_MATCH_IP_PROTO \|
	spec->efs_dword[0] = vlan_id;			EFX_FILTER_MATCH_LOC_HOST \| EFX_FILTER_MATCH_LOC_PORT;
	spec->efs_dword[1] =
	src_mac[2] << 24 \|			if (enp->en_features & EFX_FEATURE_MAC_HEADER_FILTERS) {
	src_mac[3] << 16 \|			rx_matches[index++] =
	src_mac[4] << 8 \|			EFX_FILTER_MATCH_OUTER_VID \| EFX_FILTER_MATCH_LOC_MAC;
	src_mac[5];
	spec->efs_dword[2] =			rx_matches[index++] = EFX_FILTER_MATCH_LOC_MAC;
	src_mac[0] << 8 \|
	src_mac[1];
	}			}
	#endif /* EFSYS_OPT_SIENA */

	#if EFSYS_OPT_SIENA			EFSYS_ASSERT3U(index, <=, MAX_SUPPORTED);
	extern void
	efx_filter_spec_tx_mac_wild(
	__inout efx_filter_spec_t *spec,
	__in uint8_t *src_mac)
	{
	EFSYS_ASSERT3P(spec, !=, NULL);
	EFSYS_ASSERT3P(src_mac, !=, NULL);

	spec->efs_type = EFX_FILTER_TX_MAC_WILD;			*length = index;
	spec->efs_flags = 0;			memcpy(list, rx_matches, *length);
	spec->efs_dword[0] = 0;
	spec->efs_dword[1] =			return (0);
	src_mac[2] << 24 \|
	src_mac[3] << 16 \|			fail1:
	src_mac[4] << 8 \|
	src_mac[5];			return (rc);
	spec->efs_dword[2] =
	src_mac[0] << 8 \|
	src_mac[1];
	}			}
	#endif /* EFSYS_OPT_SIENA */

				#undef MAX_SUPPORTED

				#endif /* EFSYS_OPT_FALCON \|\| EFSYS_OPT_SIENA */

	#endif /* EFSYS_OPT_FILTER */			#endif /* EFSYS_OPT_FILTER */