D44220.id.diff
No OneTemporary
Actions

Size

512 KB

Referenced Files

None

Subscribers

None

D44220.id.diff
View Options

This file is larger than 256 KB, so syntax highlighting was skipped.

	diff --git a/sys/dev/mlx5/cq.h b/sys/dev/mlx5/cq.h
	--- a/sys/dev/mlx5/cq.h
	+++ b/sys/dev/mlx5/cq.h
	@@ -113,6 +113,12 @@
	} params;
	};

	+enum {
	+ CQE_STRIDE_64 = 0,
	+ CQE_STRIDE_128 = 1,
	+ CQE_STRIDE_128_PAD = 2,
	+};
	+
	static inline int cqe_sz_to_mlx_sz(u8 size)
	{
	return size == 64 ? CQE_SIZE_64 : CQE_SIZE_128;
	diff --git a/sys/dev/mlx5/crypto.h b/sys/dev/mlx5/crypto.h
	new file mode 100644
	--- /dev/null
	+++ b/sys/dev/mlx5/crypto.h
	@@ -0,0 +1,37 @@
	+/*-
	+ * Copyright (c) 2023, NVIDIA Technologies. All rights reserved.
	+ *
	+ * Redistribution and use in source and binary forms, with or without
	+ * modification, are permitted provided that the following conditions
	+ * are met:
	+ * 1. Redistributions of source code must retain the above copyright
	+ * notice, this list of conditions and the following disclaimer.
	+ * 2. Redistributions in binary form must reproduce the above copyright
	+ * notice, this list of conditions and the following disclaimer in the
	+ * documentation and/or other materials provided with the distribution.
	+ *
	+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
	+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
	+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	+ * SUCH DAMAGE.
	+ *
	+ * $FreeBSD$
	+ */
	+
	+#ifndef __MLX5_CRYPTO_H__
	+#define __MLX5_CRYPTO_H__
	+
	+struct mlx5_core_dev;
	+
	+int mlx5_encryption_key_create(struct mlx5_core_dev *, u32 pdn, u32 key_type,
	+ const void p_key, u32 key_len, u32 p_obj_id);
	+int mlx5_encryption_key_destroy(struct mlx5_core_dev *mdev, u32 oid);
	+
	+#endif /* __MLX5_CRYPTO_H__ */
	diff --git a/sys/dev/mlx5/device.h b/sys/dev/mlx5/device.h
	--- a/sys/dev/mlx5/device.h
	+++ b/sys/dev/mlx5/device.h
	@@ -150,12 +150,14 @@
	tmp; \
	})

	-#define MLX5_BY_PASS_NUM_REGULAR_PRIOS 8
	-#define MLX5_BY_PASS_NUM_DONT_TRAP_PRIOS 8
	+#define MLX5_RDMA_RX_NUM_COUNTERS_PRIOS 2
	+#define MLX5_RDMA_TX_NUM_COUNTERS_PRIOS 1
	+#define MLX5_BY_PASS_NUM_REGULAR_PRIOS 16
	+#define MLX5_BY_PASS_NUM_DONT_TRAP_PRIOS 16
	#define MLX5_BY_PASS_NUM_MULTICAST_PRIOS 1
	#define MLX5_BY_PASS_NUM_PRIOS (MLX5_BY_PASS_NUM_REGULAR_PRIOS +\
	- MLX5_BY_PASS_NUM_DONT_TRAP_PRIOS +\
	- MLX5_BY_PASS_NUM_MULTICAST_PRIOS)
	+ MLX5_BY_PASS_NUM_DONT_TRAP_PRIOS +\
	+ MLX5_BY_PASS_NUM_MULTICAST_PRIOS)

	/* insert a value to a struct */
	#define MLX5_VSC_SET(typ, p, fld, v) do { \
	@@ -393,6 +395,8 @@
	MLX5_OPCODE_UMR = 0x25,
	MLX5_OPCODE_QOS_REMAP = 0x2a,

	+ MLX5_OPCODE_ACCESS_ASO = 0x2d,
	+
	MLX5_OPCODE_SIGNATURE_CANCELED = (1 << 15),
	};

	@@ -569,6 +573,11 @@
	__be32 rsvd1[6];
	};

	+struct mlx5_eqe_obj_change {
	+ u8 rsvd0[2];
	+ __be16 obj_type;
	+ __be32 obj_id;
	+};

	#define PORT_MODULE_EVENT_MODULE_STATUS_MASK 0xF
	#define PORT_MODULE_EVENT_ERROR_TYPE_MASK 0xF
	@@ -640,6 +649,7 @@
	struct mlx5_eqe_dct dct;
	struct mlx5_eqe_temp_warning temp_warning;
	struct mlx5_eqe_xrq_err xrq_err;
	+ struct mlx5_eqe_obj_change obj_change;
	} __packed;

	struct mlx5_eqe {
	@@ -705,7 +715,12 @@
	u8 l4_hdr_type_etc;
	__be16 vlan_info;
	__be32 srqn; /* [31:24]: lro_num_seg, [23:0]: srqn */
	- __be32 imm_inval_pkey;
	+ union {
	+ __be32 immediate;
	+ __be32 inval_rkey;
	+ __be32 pkey;
	+ __be32 ft_metadata;
	+ };
	u8 rsvd40[4];
	__be32 byte_cnt;
	__be64 timestamp;
	@@ -921,6 +936,7 @@
	MLX5_MATCH_OUTER_HEADERS = 1 << 0,
	MLX5_MATCH_MISC_PARAMETERS = 1 << 1,
	MLX5_MATCH_INNER_HEADERS = 1 << 2,
	+ MLX5_MATCH_MISC_PARAMETERS_2 = 1 << 3,

	};

	@@ -990,12 +1006,19 @@
	MLX5_CAP_VECTOR_CALC,
	MLX5_CAP_QOS,
	MLX5_CAP_DEBUG,
	- MLX5_CAP_NVME,
	- MLX5_CAP_DMC,
	- MLX5_CAP_DEC,
	+ MLX5_CAP_RESERVED_14,
	+ MLX5_CAP_DEV_MEM,
	+ MLX5_CAP_RESERVED_16,
	MLX5_CAP_TLS,
	+ MLX5_CAP_VDPA_EMULATION = 0x13,
	MLX5_CAP_DEV_EVENT = 0x14,
	+ MLX5_CAP_IPSEC,
	+ MLX5_CAP_CRYPTO = 0x1a,
	+ MLX5_CAP_DEV_SHAMPO = 0x1d,
	+ MLX5_CAP_MACSEC = 0x1f,
	MLX5_CAP_GENERAL_2 = 0x20,
	+ MLX5_CAP_PORT_SELECTION = 0x25,
	+ MLX5_CAP_ADV_VIRTUALIZATION = 0x26,
	/* NUM OF CAP Types */
	MLX5_CAP_NUM
	};
	@@ -1060,6 +1083,9 @@
	#define MLX5_CAP_FLOWTABLE(mdev, cap) \
	MLX5_GET(flow_table_nic_cap, mdev->hca_caps_cur[MLX5_CAP_FLOW_TABLE], cap)

	+#define MLX5_CAP64_FLOWTABLE(mdev, cap) \
	+ MLX5_GET64(flow_table_nic_cap, (mdev)->hca_caps_cur[MLX5_CAP_FLOW_TABLE], cap)
	+
	#define MLX5_CAP_FLOWTABLE_MAX(mdev, cap) \
	MLX5_GET(flow_table_nic_cap, mdev->hca_caps_max[MLX5_CAP_FLOW_TABLE], cap)

	@@ -1069,6 +1095,54 @@
	#define MLX5_CAP_FLOWTABLE_NIC_RX_MAX(mdev, cap) \
	MLX5_CAP_FLOWTABLE_MAX(mdev, flow_table_properties_nic_receive.cap)

	+#define MLX5_CAP_FLOWTABLE_NIC_TX(mdev, cap) \
	+ MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_transmit.cap)
	+
	+#define MLX5_CAP_FLOWTABLE_NIC_TX_MAX(mdev, cap) \
	+ MLX5_CAP_FLOWTABLE_MAX(mdev, flow_table_properties_nic_transmit.cap)
	+
	+#define MLX5_CAP_FLOWTABLE_SNIFFER_RX(mdev, cap) \
	+ MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_receive_sniffer.cap)
	+
	+#define MLX5_CAP_FLOWTABLE_SNIFFER_RX_MAX(mdev, cap) \
	+ MLX5_CAP_FLOWTABLE_MAX(mdev, flow_table_properties_nic_receive_sniffer.cap)
	+
	+#define MLX5_CAP_FLOWTABLE_SNIFFER_TX(mdev, cap) \
	+ MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_transmit_sniffer.cap)
	+
	+#define MLX5_CAP_FLOWTABLE_SNIFFER_TX_MAX(mdev, cap) \
	+ MLX5_CAP_FLOWTABLE_MAX(mdev, flow_table_properties_nic_transmit_sniffer.cap)
	+
	+#define MLX5_CAP_FLOWTABLE_RDMA_RX(mdev, cap) \
	+ MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_receive_rdma.cap)
	+
	+#define MLX5_CAP_FLOWTABLE_RDMA_RX_MAX(mdev, cap) \
	+ MLX5_CAP_FLOWTABLE_MAX(mdev, flow_table_properties_nic_receive_rdma.cap)
	+
	+#define MLX5_CAP_FLOWTABLE_RDMA_TX(mdev, cap) \
	+ MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_transmit_rdma.cap)
	+
	+#define MLX5_CAP_FLOWTABLE_RDMA_TX_MAX(mdev, cap) \
	+ MLX5_CAP_FLOWTABLE_MAX(mdev, flow_table_properties_nic_transmit_rdma.cap)
	+
	+#define MLX5_CAP_FLOWTABLE_NIC_TX(mdev, cap) \
	+ MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_transmit.cap)
	+
	+#define MLX5_CAP_FLOWTABLE_NIC_TX_MAX(mdev, cap) \
	+ MLX5_CAP_FLOWTABLE_MAX(mdev, flow_table_properties_nic_transmit.cap)
	+
	+#define MLX5_CAP_FLOWTABLE_RDMA_RX(mdev, cap) \
	+ MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_receive_rdma.cap)
	+
	+#define MLX5_CAP_FLOWTABLE_RDMA_RX_MAX(mdev, cap) \
	+ MLX5_CAP_FLOWTABLE_MAX(mdev, flow_table_properties_nic_receive_rdma.cap)
	+
	+#define MLX5_CAP_FLOWTABLE_RDMA_TX(mdev, cap) \
	+ MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_transmit_rdma.cap)
	+
	+#define MLX5_CAP_FLOWTABLE_RDMA_TX_MAX(mdev, cap) \
	+ MLX5_CAP_FLOWTABLE_MAX(mdev, flow_table_properties_nic_transmit_rdma.cap)
	+
	#define MLX5_CAP_ESW_FLOWTABLE(mdev, cap) \
	MLX5_GET(flow_table_eswitch_cap, \
	mdev->hca_caps_cur[MLX5_CAP_ESWITCH_FLOW_TABLE], cap)
	@@ -1095,14 +1169,46 @@
	#define MLX5_CAP_ESW_INGRESS_ACL_MAX(mdev, cap) \
	MLX5_CAP_ESW_FLOWTABLE_MAX(mdev, flow_table_properties_esw_acl_ingress.cap)

	+#define MLX5_CAP_ESW_FT_FIELD_SUPPORT_2(mdev, cap) \
	+ MLX5_CAP_ESW_FLOWTABLE(mdev, ft_field_support_2_esw_fdb.cap)
	+
	+#define MLX5_CAP_ESW_FT_FIELD_SUPPORT_2_MAX(mdev, cap) \
	+ MLX5_CAP_ESW_FLOWTABLE_MAX(mdev, ft_field_support_2_esw_fdb.cap)
	+
	#define MLX5_CAP_ESW(mdev, cap) \
	MLX5_GET(e_switch_cap, \
	mdev->hca_caps_cur[MLX5_CAP_ESWITCH], cap)

	+#define MLX5_CAP64_ESW_FLOWTABLE(mdev, cap) \
	+ MLX5_GET64(flow_table_eswitch_cap, \
	+ (mdev)->hca_caps_cur[MLX5_CAP_ESWITCH_FLOW_TABLE], cap)
	+
	#define MLX5_CAP_ESW_MAX(mdev, cap) \
	MLX5_GET(e_switch_cap, \
	mdev->hca_caps_max[MLX5_CAP_ESWITCH], cap)

	+#define MLX5_CAP_PORT_SELECTION(mdev, cap) \
	+ MLX5_GET(port_selection_cap, \
	+ mdev->hca_caps_cur[MLX5_CAP_PORT_SELECTION], cap)
	+
	+#define MLX5_CAP_PORT_SELECTION_MAX(mdev, cap) \
	+ MLX5_GET(port_selection_cap, \
	+ mdev->hca_caps_max[MLX5_CAP_PORT_SELECTION], cap)
	+
	+#define MLX5_CAP_ADV_VIRTUALIZATION(mdev, cap) \
	+ MLX5_GET(adv_virtualization_cap, \
	+ mdev->hca_caps_cur[MLX5_CAP_ADV_VIRTUALIZATION], cap)
	+
	+#define MLX5_CAP_ADV_VIRTUALIZATION_MAX(mdev, cap) \
	+ MLX5_GET(adv_virtualization_cap, \
	+ mdev->hca_caps_max[MLX5_CAP_ADV_VIRTUALIZATION], cap)
	+
	+#define MLX5_CAP_FLOWTABLE_PORT_SELECTION(mdev, cap) \
	+ MLX5_CAP_PORT_SELECTION(mdev, flow_table_properties_port_selection.cap)
	+
	+#define MLX5_CAP_FLOWTABLE_PORT_SELECTION_MAX(mdev, cap) \
	+ MLX5_CAP_PORT_SELECTION_MAX(mdev, flow_table_properties_port_selection.cap)
	+
	#define MLX5_CAP_ODP(mdev, cap)\
	MLX5_GET(odp_cap, mdev->hca_caps_cur[MLX5_CAP_ODP], cap)

	@@ -1171,6 +1277,9 @@
	#define MLX5_CAP_DEV_EVENT(mdev, cap)\
	MLX5_ADDR_OF(device_event_cap, (mdev)->hca_caps_cur[MLX5_CAP_DEV_EVENT], cap)

	+#define MLX5_CAP_IPSEC(mdev, cap) \
	+ MLX5_GET(ipsec_cap, (mdev)->hca_caps_cur[MLX5_CAP_IPSEC], cap)
	+
	enum {
	MLX5_CMD_STAT_OK = 0x0,
	MLX5_CMD_STAT_INT_ERR = 0x1,
	diff --git a/sys/dev/mlx5/doorbell.h b/sys/dev/mlx5/doorbell.h
	--- a/sys/dev/mlx5/doorbell.h
	+++ b/sys/dev/mlx5/doorbell.h
	@@ -63,10 +63,12 @@
	{
	unsigned long flags;

	- spin_lock_irqsave(doorbell_lock, flags);
	+ if (doorbell_lock)
	+ spin_lock_irqsave(doorbell_lock, flags);
	__raw_writel((__force u32) val[0], dest);
	__raw_writel((__force u32) val[1], dest + 4);
	- spin_unlock_irqrestore(doorbell_lock, flags);
	+ if (doorbell_lock)
	+ spin_unlock_irqrestore(doorbell_lock, flags);
	}

	#endif
	diff --git a/sys/dev/mlx5/driver.h b/sys/dev/mlx5/driver.h
	--- a/sys/dev/mlx5/driver.h
	+++ b/sys/dev/mlx5/driver.h
	@@ -635,11 +635,13 @@

	struct mlx5_pme_stats pme_stats;

	+ struct mlx5_flow_steering *steering;
	struct mlx5_eswitch *eswitch;

	struct mlx5_bfreg_data bfregs;
	struct mlx5_uars_page *uar;
	struct mlx5_fc_stats fc_stats;
	+ struct mlx5_ft_pool *ft_pool;
	};

	enum mlx5_device_state {
	@@ -728,6 +730,10 @@
	struct mlx5_flow_root_namespace *esw_ingress_root_ns;
	struct mlx5_flow_root_namespace *sniffer_rx_root_ns;
	struct mlx5_flow_root_namespace *sniffer_tx_root_ns;
	+ struct mlx5_flow_root_namespace *nic_tx_root_ns;
	+ struct mlx5_flow_root_namespace *rdma_tx_root_ns;
	+ struct mlx5_flow_root_namespace *rdma_rx_root_ns;
	+
	u32 num_q_counter_allocated[MLX5_INTERFACE_NUMBER];
	struct mlx5_crspace_regmap *dump_rege;
	uint32_t *dump_data;
	@@ -756,6 +762,7 @@
	#ifdef CONFIG_MLX5_FPGA
	struct mlx5_fpga_device *fpga;
	#endif
	+ struct xarray ipsec_sadb;
	};

	enum {
	diff --git a/sys/dev/mlx5/fs.h b/sys/dev/mlx5/fs.h
	--- a/sys/dev/mlx5/fs.h
	+++ b/sys/dev/mlx5/fs.h
	@@ -35,8 +35,33 @@
	#include <dev/mlx5/device.h>
	#include <dev/mlx5/driver.h>

	+enum mlx5_flow_destination_type {
	+ MLX5_FLOW_DESTINATION_TYPE_NONE,
	+ MLX5_FLOW_DESTINATION_TYPE_VPORT,
	+ MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE,
	+ MLX5_FLOW_DESTINATION_TYPE_TIR,
	+ MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER,
	+ MLX5_FLOW_DESTINATION_TYPE_UPLINK,
	+ MLX5_FLOW_DESTINATION_TYPE_PORT,
	+ MLX5_FLOW_DESTINATION_TYPE_COUNTER,
	+ MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM,
	+ MLX5_FLOW_DESTINATION_TYPE_RANGE,
	+ MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE,
	+};
	+
	+enum {
	+ MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO = 1 << 16,
	+ MLX5_FLOW_CONTEXT_ACTION_ENCRYPT = 1 << 17,
	+ MLX5_FLOW_CONTEXT_ACTION_DECRYPT = 1 << 18,
	+ MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS = 1 << 19,
	+};
	+
	enum {
	- MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO = 1 << 16,
	+ MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT = BIT(0),
	+ MLX5_FLOW_TABLE_TUNNEL_EN_DECAP = BIT(1),
	+ MLX5_FLOW_TABLE_TERMINATION = BIT(2),
	+ MLX5_FLOW_TABLE_UNMANAGED = BIT(3),
	+ MLX5_FLOW_TABLE_OTHER_VPORT = BIT(4),
	};

	/Flow tag/
	@@ -63,54 +88,149 @@

	enum mlx5_flow_namespace_type {
	MLX5_FLOW_NAMESPACE_BYPASS,
	+ MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC,
	+ MLX5_FLOW_NAMESPACE_LAG,
	MLX5_FLOW_NAMESPACE_OFFLOADS,
	+ MLX5_FLOW_NAMESPACE_ETHTOOL,
	MLX5_FLOW_NAMESPACE_KERNEL,
	MLX5_FLOW_NAMESPACE_LEFTOVERS,
	- MLX5_FLOW_NAMESPACE_SNIFFER_RX,
	- MLX5_FLOW_NAMESPACE_SNIFFER_TX,
	+ MLX5_FLOW_NAMESPACE_ANCHOR,
	+ MLX5_FLOW_NAMESPACE_FDB_BYPASS,
	MLX5_FLOW_NAMESPACE_FDB,
	MLX5_FLOW_NAMESPACE_ESW_EGRESS,
	MLX5_FLOW_NAMESPACE_ESW_INGRESS,
	+ MLX5_FLOW_NAMESPACE_SNIFFER_RX,
	+ MLX5_FLOW_NAMESPACE_SNIFFER_TX,
	+ MLX5_FLOW_NAMESPACE_EGRESS,
	+ MLX5_FLOW_NAMESPACE_EGRESS_IPSEC,
	+ MLX5_FLOW_NAMESPACE_EGRESS_MACSEC,
	+ MLX5_FLOW_NAMESPACE_RDMA_RX,
	+ MLX5_FLOW_NAMESPACE_RDMA_RX_KERNEL,
	+ MLX5_FLOW_NAMESPACE_RDMA_TX,
	+ MLX5_FLOW_NAMESPACE_PORT_SEL,
	+ MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS,
	+ MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS,
	+ MLX5_FLOW_NAMESPACE_RDMA_RX_IPSEC,
	+ MLX5_FLOW_NAMESPACE_RDMA_TX_IPSEC,
	+};
	+
	+enum {
	+ FDB_BYPASS_PATH,
	+ FDB_TC_OFFLOAD,
	+ FDB_FT_OFFLOAD,
	+ FDB_TC_MISS,
	+ FDB_BR_OFFLOAD,
	+ FDB_SLOW_PATH,
	+ FDB_PER_VPORT,
	};

	struct mlx5_flow_table;
	struct mlx5_flow_group;
	struct mlx5_flow_rule;
	struct mlx5_flow_namespace;
	+struct mlx5_flow_handle;
	+
	+enum {
	+ FLOW_CONTEXT_HAS_TAG = BIT(0),
	+};
	+
	+struct mlx5_flow_context {
	+ u32 flags;
	+ u32 flow_tag;
	+ u32 flow_source;
	+};

	struct mlx5_flow_spec {
	u8 match_criteria_enable;
	u32 match_criteria[MLX5_ST_SZ_DW(fte_match_param)];
	u32 match_value[MLX5_ST_SZ_DW(fte_match_param)];
	+ struct mlx5_flow_context flow_context;
	+};
	+
	+enum {
	+ MLX5_FLOW_DEST_VPORT_VHCA_ID = BIT(0),
	+ MLX5_FLOW_DEST_VPORT_REFORMAT_ID = BIT(1),
	+};
	+
	+enum mlx5_flow_dest_range_field {
	+ MLX5_FLOW_DEST_RANGE_FIELD_PKT_LEN = 0,
	};

	struct mlx5_flow_destination {
	- u32 type;
	+ enum mlx5_flow_destination_type type;
	union {
	- u32 tir_num;
	- struct mlx5_flow_table *ft;
	- u32 vport_num;
	+ u32 tir_num;
	+ u32 ft_num;
	+ struct mlx5_flow_table *ft;
	+ u32 counter_id;
	+ struct {
	+ u16 num;
	+ u16 vhca_id;
	+ struct mlx5_pkt_reformat *pkt_reformat;
	+ u8 flags;
	+ } vport;
	+ struct {
	+ struct mlx5_flow_table *hit_ft;
	+ struct mlx5_flow_table *miss_ft;
	+ enum mlx5_flow_dest_range_field field;
	+ u32 min;
	+ u32 max;
	+ } range;
	+ u32 sampler_id;
	};
	};

	-enum mlx5_flow_act_actions {
	- MLX5_FLOW_ACT_ACTIONS_FLOW_TAG = 1 << 0,
	- MLX5_FLOW_ACT_ACTIONS_MODIFY_HDR = 1 << 1,
	- MLX5_FLOW_ACT_ACTIONS_PACKET_REFORMAT = 1 << 2,
	- MLX5_FLOW_ACT_ACTIONS_COUNT = 1 << 3,
	+struct mlx5_exe_aso {
	+ u32 object_id;
	+ u8 type;
	+ u8 return_reg_id;
	+ union {
	+ u32 ctrl_data;
	+ struct {
	+ u8 meter_idx;
	+ u8 init_color;
	+ } flow_meter;
	+ };
	+};
	+
	+enum {
	+ FLOW_ACT_NO_APPEND = BIT(0),
	+ FLOW_ACT_IGNORE_FLOW_LEVEL = BIT(1),
	+};
	+
	+struct mlx5_fs_vlan {
	+ u16 ethtype;
	+ u16 vid;
	+ u8 prio;
	+};
	+
	+#define MLX5_FS_VLAN_DEPTH 2
	+
	+enum mlx5_flow_act_crypto_type {
	+ MLX5_FLOW_ACT_CRYPTO_TYPE_IPSEC,
	+};
	+
	+enum mlx5_flow_act_crypto_op {
	+ MLX5_FLOW_ACT_CRYPTO_OP_ENCRYPT,
	+ MLX5_FLOW_ACT_CRYPTO_OP_DECRYPT,
	};

	-enum MLX5_FLOW_ACT_FLAGS {
	- MLX5_FLOW_ACT_NO_APPEND = 1 << 0,
	+struct mlx5_flow_act_crypto_params {
	+ u32 obj_id;
	+ u8 type; /* see enum mlx5_flow_act_crypto_type */
	+ u8 op; /* see enum mlx5_flow_act_crypto_op */
	};

	struct mlx5_flow_act {
	- u32 actions; /* See enum mlx5_flow_act_actions */
	- u32 flags;
	- u32 flow_tag;
	- struct mlx5_modify_hdr *modify_hdr;
	+ u32 action;
	+ struct mlx5_modify_hdr *modify_hdr;
	struct mlx5_pkt_reformat *pkt_reformat;
	- struct mlx5_fc *counter;
	+ struct mlx5_flow_act_crypto_params crypto;
	+ u32 flags;
	+ struct mlx5_fs_vlan vlan[MLX5_FS_VLAN_DEPTH];
	+ struct ib_counters *counters;
	+ struct mlx5_flow_group *fg;
	+ struct mlx5_exe_aso exe_aso;
	};

	#define FT_NAME_STR_SZ 20
	@@ -137,6 +257,28 @@
	size - 1);
	}

	+struct mlx5_flow_namespace *
	+mlx5_get_flow_vport_acl_namespace(struct mlx5_core_dev *dev,
	+ enum mlx5_flow_namespace_type type,
	+ int vport);
	+
	+struct mlx5_flow_table_attr {
	+ int prio;
	+ int max_fte;
	+ u32 level;
	+ u32 flags;
	+ u16 uid;
	+ struct mlx5_flow_table *next_ft;
	+
	+ struct {
	+ int max_num_groups;
	+ int num_reserved_entries;
	+ } autogroup;
	+};
	+
	+struct mlx5_flow_namespace *
	+mlx5_get_fdb_sub_ns(struct mlx5_core_dev *dev, int n);
	+
	struct mlx5_flow_namespace *
	mlx5_get_flow_namespace(struct mlx5_core_dev *dev,
	enum mlx5_flow_namespace_type type);
	@@ -147,24 +289,19 @@
	*/
	struct mlx5_flow_table *
	mlx5_create_auto_grouped_flow_table(struct mlx5_flow_namespace *ns,
	- int prio,
	- const char *name,
	- int num_flow_table_entries,
	- int max_num_groups,
	- int num_reserved_entries);
	+ struct mlx5_flow_table_attr *ft_attr);

	struct mlx5_flow_table *
	mlx5_create_vport_flow_table(struct mlx5_flow_namespace *ns,
	- u16 vport,
	- int prio,
	- const char *name,
	- int num_flow_table_entries);
	+ struct mlx5_flow_table_attr *ft_attr, u16 vport);
	+
	+struct mlx5_flow_table *mlx5_create_lag_demux_flow_table(
	+ struct mlx5_flow_namespace *ns,
	+ int prio, u32 level);

	struct mlx5_flow_table *
	mlx5_create_flow_table(struct mlx5_flow_namespace *ns,
	- int prio,
	- const char *name,
	- int num_flow_table_entries);
	+ struct mlx5_flow_table_attr *ft_attr);
	int mlx5_destroy_flow_table(struct mlx5_flow_table *ft);

	/* inbox should be set with the following values:
	@@ -177,18 +314,17 @@
	mlx5_create_flow_group(struct mlx5_flow_table ft, u32 in);
	void mlx5_destroy_flow_group(struct mlx5_flow_group *fg);

	-/* Single destination per rule.
	- * Group ID is implied by the match criteria.
	- */
	-struct mlx5_flow_rule *
	-mlx5_add_flow_rule(struct mlx5_flow_table *ft,
	- u8 match_criteria_enable,
	- u32 *match_criteria,
	- u32 *match_value,
	- u32 sw_action,
	- struct mlx5_flow_act *flow_act,
	- struct mlx5_flow_destination *dest);
	-void mlx5_del_flow_rule(struct mlx5_flow_rule **);
	+struct mlx5_flow_handle *
	+mlx5_add_flow_rules(struct mlx5_flow_table *ft,
	+ const struct mlx5_flow_spec *spec,
	+ struct mlx5_flow_act *flow_act,
	+ struct mlx5_flow_destination *dest,
	+ int num_dest);
	+void mlx5_del_flow_rules(struct mlx5_flow_handle **pp);
	+
	+int mlx5_modify_rule_destination(struct mlx5_flow_handle *handler,
	+ struct mlx5_flow_destination *new_dest,
	+ struct mlx5_flow_destination *old_dest);

	/The following API is for sniffer/
	typedef int (rule_event_fn)(struct mlx5_flow_rule rule,
	@@ -294,4 +430,8 @@
	u64 packets, u64 bytes);
	u32 mlx5_fc_id(struct mlx5_fc *counter);
	/***** End of Flow counters API ****/
	+
	+u32 mlx5_flow_table_id(struct mlx5_flow_table *ft);
	+int mlx5_fs_add_rx_underlay_qpn(struct mlx5_core_dev *dev, u32 underlay_qpn);
	+int mlx5_fs_remove_rx_underlay_qpn(struct mlx5_core_dev *dev, u32 underlay_qpn);
	#endif
	diff --git a/sys/dev/mlx5/mlx5_accel/ipsec.h b/sys/dev/mlx5/mlx5_accel/ipsec.h
	--- a/sys/dev/mlx5/mlx5_accel/ipsec.h
	+++ b/sys/dev/mlx5/mlx5_accel/ipsec.h
	@@ -1,33 +1,26 @@
	/*-
	- * Copyright (c) 2017 Mellanox Technologies. All rights reserved.
	+ * Copyright (c) 2023 NVIDIA corporation & affiliates.
	*
	- * This software is available to you under a choice of one of two
	- * licenses. You may choose to be licensed under the terms of the GNU
	- * General Public License (GPL) Version 2, available from the file
	- * COPYING in the main directory of this source tree, or the
	- * OpenIB.org BSD license below:
	+ * Redistribution and use in source and binary forms, with or without
	+ * modification, are permitted provided that the following conditions
	+ * are met:
	+ * 1. Redistributions of source code must retain the above copyright
	+ * notice, this list of conditions and the following disclaimer.
	+ * 2. Redistributions in binary form must reproduce the above copyright
	+ * notice, this list of conditions and the following disclaimer in the
	+ * documentation and/or other materials provided with the distribution.
	*
	- * Redistribution and use in source and binary forms, with or
	- * without modification, are permitted provided that the following
	- * conditions are met:
	- *
	- * - Redistributions of source code must retain the above
	- * copyright notice, this list of conditions and the following
	- * disclaimer.
	- *
	- * - Redistributions in binary form must reproduce the above
	- * copyright notice, this list of conditions and the following
	- * disclaimer in the documentation and/or other materials
	- * provided with the distribution.
	- *
	- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	- * SOFTWARE.
	+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
	+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
	+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	+ * SUCH DAMAGE.
	*
	* $FreeBSD$
	*/
	@@ -35,105 +28,245 @@
	#ifndef __MLX5_ACCEL_IPSEC_H__
	#define __MLX5_ACCEL_IPSEC_H__

	-#ifdef CONFIG_MLX5_ACCEL
	-
	+#include <sys/mbuf.h>
	#include <dev/mlx5/driver.h>
	+#include <dev/mlx5/qp.h>
	+#include <dev/mlx5/mlx5_core/mlx5_core.h>
	+#include <dev/mlx5/mlx5_en/en.h>
	+#include <dev/mlx5/mlx5_lib/aso.h>
	+
	+#define MLX5E_IPSEC_SADB_RX_BITS 10
	+#define MLX5_IPSEC_METADATA_MARKER(ipsec_metadata) ((ipsec_metadata >> 31) & 0x1)
	+
	+struct mlx5e_priv;
	+struct mlx5e_tx_wqe;
	+struct mlx5e_ipsec_tx;
	+struct mlx5e_ipsec_rx;
	+
	+struct aes_gcm_keymat {
	+ u64 seq_iv;

	-enum {
	- MLX5_ACCEL_IPSEC_DEVICE = BIT(1),
	- MLX5_ACCEL_IPSEC_IPV6 = BIT(2),
	- MLX5_ACCEL_IPSEC_ESP = BIT(3),
	- MLX5_ACCEL_IPSEC_LSO = BIT(4),
	+ u32 salt;
	+ u32 icv_len;
	+
	+ u32 key_len;
	+ u32 aes_key[256 / 32];
	+};
	+
	+struct mlx5e_ipsec_work {
	+ struct work_struct work;
	+ struct mlx5e_ipsec_sa_entry *sa_entry;
	+ void *data;
	};

	-#define MLX5_IPSEC_SADB_IP_AH BIT(7)
	-#define MLX5_IPSEC_SADB_IP_ESP BIT(6)
	-#define MLX5_IPSEC_SADB_SA_VALID BIT(5)
	-#define MLX5_IPSEC_SADB_SPI_EN BIT(4)
	-#define MLX5_IPSEC_SADB_DIR_SX BIT(3)
	-#define MLX5_IPSEC_SADB_IPV6 BIT(2)
	+struct mlx5e_ipsec_dwork {
	+ struct delayed_work dwork;
	+ struct mlx5e_ipsec_sa_entry *sa_entry;
	+};

	-enum {
	- MLX5_IPSEC_CMD_ADD_SA = 0,
	- MLX5_IPSEC_CMD_DEL_SA = 1,
	+struct mlx5e_ipsec_aso {
	+ u8 __aligned(64) ctx[MLX5_ST_SZ_BYTES(ipsec_aso)];
	+ dma_addr_t dma_addr;
	+ struct mlx5_aso *aso;
	+ /* Protect ASO WQ access, as it is global to whole IPsec */
	+ spinlock_t lock;
	};

	-enum mlx5_accel_ipsec_enc_mode {
	- MLX5_IPSEC_SADB_MODE_NONE = 0,
	- MLX5_IPSEC_SADB_MODE_AES_GCM_128_AUTH_128 = 1,
	- MLX5_IPSEC_SADB_MODE_AES_GCM_256_AUTH_128 = 3,
	+struct mlx5_replay_esn {
	+ u32 replay_window;
	+ u32 esn;
	+ u32 esn_msb;
	+ u8 overlap : 1;
	+ u8 trigger : 1;
	};

	-#define MLX5_IPSEC_DEV(mdev) (mlx5_accel_ipsec_device_caps(mdev) & \
	- MLX5_ACCEL_IPSEC_DEVICE)
	+struct mlx5_accel_esp_xfrm_attrs {
	+ u32 spi;
	+ struct aes_gcm_keymat aes_gcm;

	-struct mlx5_accel_ipsec_sa {
	- __be32 cmd;
	- u8 key_enc[32];
	- u8 key_auth[32];
	- __be32 sip[4];
	- __be32 dip[4];
	union {
	- struct {
	- __be32 reserved;
	- u8 salt_iv[8];
	- __be32 salt;
	- } __packed gcm;
	- struct {
	- u8 salt[16];
	- } __packed cbc;
	- };
	- __be32 spi;
	- __be32 sw_sa_handle;
	- __be16 tfclen;
	- u8 enc_mode;
	- u8 sip_masklen;
	- u8 dip_masklen;
	- u8 flags;
	- u8 reserved[2];
	-} __packed;
	-
	-/**
	- * mlx5_accel_ipsec_sa_cmd_exec - Execute an IPSec SADB command
	- * @mdev: mlx5 device
	- * @cmd: command to execute
	- * May be called from atomic context. Returns context pointer, or error
	- * Caller must eventually call mlx5_accel_ipsec_sa_cmd_wait from non-atomic
	- * context, to cleanup the context pointer
	- */
	-void mlx5_accel_ipsec_sa_cmd_exec(struct mlx5_core_dev mdev,
	- struct mlx5_accel_ipsec_sa *cmd);
	-
	-/**
	- * mlx5_accel_ipsec_sa_cmd_wait - Wait for command execution completion
	- * @context: Context pointer returned from call to mlx5_accel_ipsec_sa_cmd_exec
	- * Sleeps (killable) until command execution is complete.
	- * Returns the command result, or -EINTR if killed
	- */
	-int mlx5_accel_ipsec_sa_cmd_wait(void *context);
	+ __be32 a4;
	+ __be32 a6[4];
	+ } saddr;

	-u32 mlx5_accel_ipsec_device_caps(struct mlx5_core_dev *mdev);
	+ union {
	+ __be32 a4;
	+ __be32 a6[4];
	+ } daddr;
	+
	+ u8 dir : 2;
	+ u8 encap : 1;
	+ u8 drop : 1;
	+ u8 family;
	+ struct mlx5_replay_esn replay_esn;
	+ u32 authsize;
	+ u32 reqid;
	+ u16 sport;
	+ u16 dport;
	+};
	+
	+enum mlx5_ipsec_cap {
	+ MLX5_IPSEC_CAP_CRYPTO = 1 << 0,
	+ MLX5_IPSEC_CAP_ESN = 1 << 1,
	+ MLX5_IPSEC_CAP_PACKET_OFFLOAD = 1 << 2,
	+ MLX5_IPSEC_CAP_ROCE = 1 << 3,
	+ MLX5_IPSEC_CAP_PRIO = 1 << 4,
	+ MLX5_IPSEC_CAP_TUNNEL = 1 << 5,
	+ MLX5_IPSEC_CAP_ESPINUDP = 1 << 6,
	+};
	+
	+struct mlx5e_ipsec {
	+ struct mlx5_core_dev *mdev;
	+ struct workqueue_struct *wq;
	+ struct mlx5e_ipsec_tx *tx;
	+ struct mlx5e_ipsec_rx *rx_ipv4;
	+ struct mlx5e_ipsec_rx *rx_ipv6;
	+ struct mlx5e_ipsec_aso *aso;
	+ u32 pdn;
	+ u32 mkey;
	+};
	+
	+struct mlx5e_ipsec_rule {
	+ struct mlx5_flow_handle *rule;
	+ struct mlx5_flow_handle *kspi_rule;
	+ struct mlx5_flow_handle *reqid_rule;
	+ struct mlx5_modify_hdr *modify_hdr;
	+ struct mlx5_pkt_reformat *pkt_reformat;
	+ struct mlx5_fc *fc;
	+};

	-unsigned int mlx5_accel_ipsec_counters_count(struct mlx5_core_dev *mdev);
	-int mlx5_accel_ipsec_counters_read(struct mlx5_core_dev mdev, u64 counters,
	- unsigned int count);
	+struct mlx5e_ipsec_esn_state {
	+ u32 esn;
	+ u32 esn_msb;
	+ u8 overlap: 1;
	+};

	-int mlx5_accel_ipsec_init(struct mlx5_core_dev *mdev);
	-void mlx5_accel_ipsec_cleanup(struct mlx5_core_dev *mdev);
	+struct mlx5e_ipsec_sa_entry {
	+ struct secasvar *savp;
	+ if_t ifp;
	+ struct mlx5e_ipsec *ipsec;
	+ struct mlx5_accel_esp_xfrm_attrs attrs;
	+ struct mlx5e_ipsec_rule ipsec_rule;
	+ struct mlx5e_ipsec_dwork *dwork;
	+ struct mlx5e_ipsec_work *work;
	+ u32 ipsec_obj_id;
	+ u32 enc_key_id;
	+ u16 kspi; /* Stack allocated unique SA identifier */
	+ struct mlx5e_ipsec_esn_state esn_state;
	+};
	+
	+struct upspec {
	+ u16 dport;
	+ u16 sport;
	+ u8 proto;
	+};

	-#else
	+struct mlx5_accel_pol_xfrm_attrs {
	+ union {
	+ __be32 a4;
	+ __be32 a6[4];
	+ } saddr;

	-#define MLX5_IPSEC_DEV(mdev) false
	+ union {
	+ __be32 a4;
	+ __be32 a6[4];
	+ } daddr;

	-static inline int mlx5_accel_ipsec_init(struct mlx5_core_dev *mdev)
	+ struct upspec upspec;
	+
	+ u8 family;
	+ u8 action;
	+ u8 dir : 2;
	+ u32 reqid;
	+ u32 prio;
	+};
	+
	+struct mlx5e_ipsec_pol_entry {
	+ struct secpolicy *sp;
	+ struct mlx5e_ipsec *ipsec;
	+ struct mlx5e_ipsec_rule ipsec_rule;
	+ struct mlx5_accel_pol_xfrm_attrs attrs;
	+};
	+
	+/* This function doesn't really belong here, but let's put it here for now */
	+void mlx5_object_change_event(struct mlx5_core_dev dev, struct mlx5_eqe eqe);
	+
	+int mlx5e_ipsec_init(struct mlx5e_priv *priv);
	+void mlx5e_ipsec_cleanup(struct mlx5e_priv *priv);
	+
	+int mlx5e_ipsec_aso_init(struct mlx5e_ipsec *ipsec);
	+void mlx5e_ipsec_aso_cleanup(struct mlx5e_ipsec *ipsec);
	+
	+int mlx5_ipsec_create_sa_ctx(struct mlx5e_ipsec_sa_entry *sa_entry);
	+void mlx5_ipsec_free_sa_ctx(struct mlx5e_ipsec_sa_entry *sa_entry);
	+
	+u32 mlx5_ipsec_device_caps(struct mlx5_core_dev *mdev);
	+
	+static inline struct mlx5_core_dev *
	+mlx5e_ipsec_sa2dev(struct mlx5e_ipsec_sa_entry *sa_entry)
	+{
	+ return sa_entry->ipsec->mdev;
	+}
	+
	+static inline struct mlx5_core_dev *
	+mlx5e_ipsec_pol2dev(struct mlx5e_ipsec_pol_entry *pol_entry)
	+{
	+ return pol_entry->ipsec->mdev;
	+}
	+
	+void mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry,
	+ struct mlx5_accel_esp_xfrm_attrs *attrs,
	+ u8 dir);
	+
	+int mlx5e_accel_ipsec_fs_init(struct mlx5e_ipsec *ipsec);
	+void mlx5e_accel_ipsec_fs_cleanup(struct mlx5e_ipsec *ipsec);
	+int mlx5e_accel_ipsec_fs_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry);
	+void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_ipsec_sa_entry *sa_entry);
	+void mlx5e_accel_ipsec_fs_modify(struct mlx5e_ipsec_sa_entry *sa_entry);
	+struct ipsec_accel_out_tag;
	+void mlx5e_accel_ipsec_handle_tx_wqe(struct mbuf mb, struct mlx5e_tx_wqe wqe,
	+ struct ipsec_accel_out_tag *tag);
	+int mlx5e_accel_ipsec_fs_add_pol(struct mlx5e_ipsec_pol_entry *pol_entry);
	+void mlx5e_accel_ipsec_fs_del_pol(struct mlx5e_ipsec_pol_entry *pol_entry);
	+static inline int mlx5e_accel_ipsec_get_metadata(unsigned int id)
	{
	- return 0;
	+ return MLX5_ETH_WQE_FT_META_IPSEC << 23 \| id;
	}
	+static inline void
	+mlx5e_accel_ipsec_handle_tx(struct mbuf mb, struct mlx5e_tx_wqe wqe)
	+{
	+ struct ipsec_accel_out_tag *tag;

	-static inline void mlx5_accel_ipsec_cleanup(struct mlx5_core_dev *mdev)
	+ tag = (struct ipsec_accel_out_tag *)m_tag_find(mb,
	+ PACKET_TAG_IPSEC_ACCEL_OUT, NULL);
	+ if (tag != NULL)
	+ mlx5e_accel_ipsec_handle_tx_wqe(mb, wqe, tag);
	+}
	+void mlx5e_accel_ipsec_fs_rx_tables_destroy(struct mlx5e_priv *priv);
	+int mlx5e_accel_ipsec_fs_rx_tables_create(struct mlx5e_priv *priv);
	+void mlx5e_accel_ipsec_fs_rx_catchall_rules_destroy(struct mlx5e_priv *priv);
	+int mlx5e_accel_ipsec_fs_rx_catchall_rules(struct mlx5e_priv *priv);
	+int mlx5_accel_ipsec_rx_tag_add(if_t ifp, struct mbuf *mb);
	+int mlx5e_accel_ipsec_handle_rx_cqe(struct mbuf mb, struct mlx5_cqe64 cqe);
	+static inline int mlx5e_accel_ipsec_flow(struct mlx5_cqe64 *cqe)
	{
	+ return MLX5_IPSEC_METADATA_MARKER(be32_to_cpu(cqe->ft_metadata));
	}

	-#endif
	+static inline void mlx5e_accel_ipsec_handle_rx(struct mbuf mb, struct mlx5_cqe64 cqe)
	+{
	+ u32 ipsec_meta_data = be32_to_cpu(cqe->ft_metadata);
	+
	+ if (!MLX5_IPSEC_METADATA_MARKER(ipsec_meta_data)) {
	+ struct m_tag *mtag;

	+ mtag = m_tag_find(mb, PACKET_TAG_IPSEC_ACCEL_IN, NULL);
	+ if (mtag != NULL)
	+ m_tag_delete(mb, mtag);
	+
	+ return;
	+ }
	+
	+ mlx5e_accel_ipsec_handle_rx_cqe(mb, cqe);
	+}
	#endif /* __MLX5_ACCEL_IPSEC_H__ */
	diff --git a/sys/dev/mlx5/mlx5_accel/mlx5_ipsec.c b/sys/dev/mlx5/mlx5_accel/mlx5_ipsec.c
	new file mode 100644
	--- /dev/null
	+++ b/sys/dev/mlx5/mlx5_accel/mlx5_ipsec.c
	@@ -0,0 +1,657 @@
	+/*-
	+ * Copyright (c) 2023 NVIDIA corporation & affiliates.
	+ *
	+ * Redistribution and use in source and binary forms, with or without
	+ * modification, are permitted provided that the following conditions
	+ * are met:
	+ * 1. Redistributions of source code must retain the above copyright
	+ * notice, this list of conditions and the following disclaimer.
	+ * 2. Redistributions in binary form must reproduce the above copyright
	+ * notice, this list of conditions and the following disclaimer in the
	+ * documentation and/or other materials provided with the distribution.
	+ *
	+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
	+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
	+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	+ * SUCH DAMAGE.
	+ *
	+ * $FreeBSD$
	+ */
	+
	+#include "opt_ipsec.h"
	+
	+#include <sys/types.h>
	+#include <netinet/in.h>
	+#include <sys/socket.h>
	+#include <sys/param.h>
	+#include <sys/systm.h>
	+#include <net/if.h>
	+#include <net/if_var.h>
	+#include <net/pfkeyv2.h>
	+#include <netipsec/key_var.h>
	+#include <netipsec/keydb.h>
	+#include <netipsec/ipsec.h>
	+#include <netipsec/xform.h>
	+#include <netipsec/ipsec_accel.h>
	+#include <dev/mlx5/fs.h>
	+#include <dev/mlx5/mlx5_en/en.h>
	+#include <dev/mlx5/mlx5_accel/ipsec.h>
	+
	+#define MLX5_IPSEC_RESCHED msecs_to_jiffies(1000)
	+
	+static struct mlx5e_ipsec_sa_entry to_ipsec_sa_entry(void x)
	+{
	+ return (struct mlx5e_ipsec_sa_entry *)x;
	+}
	+
	+static struct mlx5e_ipsec_pol_entry to_ipsec_pol_entry(void x)
	+{
	+ return (struct mlx5e_ipsec_pol_entry *)x;
	+}
	+
	+static void mlx5e_ipsec_handle_counters(struct work_struct *_work)
	+{
	+ struct mlx5e_ipsec_dwork *dwork =
	+ container_of(_work, struct mlx5e_ipsec_dwork, dwork.work);
	+ struct mlx5e_ipsec_sa_entry *sa_entry = dwork->sa_entry;
	+ struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule;
	+ struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
	+ u64 bytes, packets;
	+
	+ if (sa_entry->attrs.drop)
	+ return;
	+
	+ mlx5_fc_query(mdev, ipsec_rule->fc, &packets, &bytes);
	+#ifdef IPSEC_ACCEL
	+ ipsec_accel_drv_sa_lifetime_update(sa_entry->savp, sa_entry->ifp,
	+ sa_entry->kspi, bytes, packets);
	+#endif
	+
	+ queue_delayed_work(sa_entry->ipsec->wq, &dwork->dwork, MLX5_IPSEC_RESCHED);
	+}
	+
	+static int mlx5e_ipsec_create_dwork(struct mlx5e_ipsec_sa_entry *sa_entry)
	+{
	+ struct mlx5e_ipsec_dwork *dwork;
	+
	+ dwork = kzalloc(sizeof(*dwork), GFP_KERNEL);
	+ if (!dwork)
	+ return (ENOMEM);
	+
	+ dwork->sa_entry = sa_entry;
	+ INIT_DELAYED_WORK(&dwork->dwork, mlx5e_ipsec_handle_counters);
	+ sa_entry->dwork = dwork;
	+ return 0;
	+}
	+
	+static int mlx5_xform_ah_authsize(const struct auth_hash *esph)
	+{
	+ int alen;
	+
	+ if (esph == NULL)
	+ return 0;
	+
	+ switch (esph->type) {
	+ case CRYPTO_SHA2_256_HMAC:
	+ case CRYPTO_SHA2_384_HMAC:
	+ case CRYPTO_SHA2_512_HMAC:
	+ alen = esph->hashsize / 2; /* RFC4868 2.3 */
	+ break;
	+
	+ case CRYPTO_POLY1305:
	+ case CRYPTO_AES_NIST_GMAC:
	+ alen = esph->hashsize;
	+ break;
	+
	+ default:
	+ alen = AH_HMAC_HASHLEN;
	+ break;
	+ }
	+
	+ return alen;
	+}
	+
	+void mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry,
	+ struct mlx5_accel_esp_xfrm_attrs *attrs,
	+ u8 dir)
	+{
	+ struct secasvar *savp = sa_entry->savp;
	+ const struct auth_hash *esph = savp->tdb_authalgxform;
	+ struct aes_gcm_keymat *aes_gcm = &attrs->aes_gcm;
	+ struct secasindex *saidx = &savp->sah->saidx;
	+ struct seckey *key_encap = savp->key_enc;
	+ int key_len;
	+
	+ memset(attrs, 0, sizeof(*attrs));
	+
	+ /* subtract off the salt, RFC4106, 8.1 and RFC3686, 5.1 */
	+ key_len = _KEYLEN(key_encap) - SAV_ISCTRORGCM(savp) * 4 - SAV_ISCHACHA(savp) * 4;
	+
	+ memcpy(aes_gcm->aes_key, key_encap->key_data, key_len);
	+ aes_gcm->key_len = key_len;
	+
	+ /* salt and seq_iv */
	+ aes_gcm->seq_iv = 0;
	+ memcpy(&aes_gcm->salt, key_encap->key_data + key_len,
	+ sizeof(aes_gcm->salt));
	+
	+ switch (savp->alg_enc) {
	+ case SADB_X_EALG_AESGCM8:
	+ attrs->authsize = 8 / 4; /* in dwords */
	+ break;
	+ case SADB_X_EALG_AESGCM12:
	+ attrs->authsize = 12 / 4; /* in dwords */
	+ break;
	+ case SADB_X_EALG_AESGCM16:
	+ attrs->authsize = 16 / 4; /* in dwords */
	+ break;
	+ default: break;
	+ }
	+
	+ /* iv len */
	+ aes_gcm->icv_len = mlx5_xform_ah_authsize(esph); //TBD: check if value make sense
	+
	+ attrs->dir = dir;
	+ /* spi - host order */
	+ attrs->spi = ntohl(savp->spi);
	+ attrs->family = saidx->dst.sa.sa_family;
	+ attrs->reqid = saidx->reqid;
	+
	+ if (saidx->src.sa.sa_family == AF_INET) {
	+ attrs->saddr.a4 = saidx->src.sin.sin_addr.s_addr;
	+ attrs->daddr.a4 = saidx->dst.sin.sin_addr.s_addr;
	+ } else {
	+ memcpy(&attrs->saddr.a6, &saidx->src.sin6.sin6_addr, 16);
	+ memcpy(&attrs->daddr.a6, &saidx->dst.sin6.sin6_addr, 16);
	+ }
	+
	+ if (savp->natt) {
	+ attrs->encap = true;
	+ attrs->sport = savp->natt->sport;
	+ attrs->dport = savp->natt->dport;
	+ }
	+
	+ if (savp->flags & SADB_X_SAFLAGS_ESN) {
	+ /* We support replay window with ESN only */
	+ attrs->replay_esn.trigger = true;
	+ if (sa_entry->esn_state.esn_msb)
	+ attrs->replay_esn.esn = sa_entry->esn_state.esn;
	+ else
	+ /* According to RFC4303, section "3.3.3. Sequence Number Generation",
	+ * the first packet sent using a given SA will contain a sequence
	+ * number of 1.
	+ */
	+ attrs->replay_esn.esn = max_t(u32, sa_entry->esn_state.esn, 1);
	+ attrs->replay_esn.esn_msb = sa_entry->esn_state.esn_msb;
	+ attrs->replay_esn.overlap = sa_entry->esn_state.overlap;
	+
	+ if (savp->replay) {
	+ switch (savp->replay->wsize) {
	+ case 4:
	+ attrs->replay_esn.replay_window = MLX5_IPSEC_ASO_REPLAY_WIN_32BIT;
	+ break;
	+ case 8:
	+ attrs->replay_esn.replay_window = MLX5_IPSEC_ASO_REPLAY_WIN_64BIT;
	+ break;
	+ case 16:
	+ attrs->replay_esn.replay_window = MLX5_IPSEC_ASO_REPLAY_WIN_128BIT;
	+ break;
	+ case 32:
	+ attrs->replay_esn.replay_window = MLX5_IPSEC_ASO_REPLAY_WIN_256BIT;
	+ break;
	+ default:
	+ /* Do nothing */
	+ break;
	+ }
	+ }
	+ }
	+}
	+
	+static int mlx5e_xfrm_validate_state(struct mlx5_core_dev *mdev,
	+ struct secasvar *savp)
	+{
	+ struct secasindex *saidx = &savp->sah->saidx;
	+ struct seckey *key_encp = savp->key_enc;
	+ int keylen;
	+
	+ if (!(mlx5_ipsec_device_caps(mdev) &
	+ MLX5_IPSEC_CAP_PACKET_OFFLOAD)) {
	+ mlx5_core_err(mdev, "FULL offload is not supported\n");
	+ return (EINVAL);
	+ }
	+ if (savp->alg_enc == SADB_EALG_NONE) {
	+ mlx5_core_err(mdev, "Cannot offload authenticated xfrm states\n");
	+ return (EINVAL);
	+ }
	+ if (savp->alg_enc != SADB_X_EALG_AESGCM16) {
	+ mlx5_core_err(mdev, "Only IPSec aes-gcm-16 encryption protocol may be offloaded\n");
	+ return (EINVAL);
	+ }
	+ if (savp->tdb_compalgxform) {
	+ mlx5_core_err(mdev, "Cannot offload compressed xfrm states\n");
	+ return (EINVAL);
	+ }
	+ if (savp->alg_auth != SADB_X_AALG_AES128GMAC && savp->alg_auth != SADB_X_AALG_AES256GMAC) {
	+ mlx5_core_err(mdev, "Cannot offload xfrm states with AEAD key length other than 128/256 bits\n");
	+ return (EINVAL);
	+ }
	+ if ((saidx->dst.sa.sa_family != AF_INET && saidx->dst.sa.sa_family != AF_INET6) \|\|
	+ (saidx->src.sa.sa_family != AF_INET && saidx->src.sa.sa_family != AF_INET6)) {
	+ mlx5_core_err(mdev, "Only IPv4/6 xfrm states may be offloaded\n");
	+ return (EINVAL);
	+ }
	+ if (saidx->proto != IPPROTO_ESP) {
	+ mlx5_core_err(mdev, "Only ESP xfrm state may be offloaded\n");
	+ return (EINVAL);
	+ }
	+ /* subtract off the salt, RFC4106, 8.1 and RFC3686, 5.1 */
	+ keylen = _KEYLEN(key_encp) - SAV_ISCTRORGCM(savp) * 4 - SAV_ISCHACHA(savp) * 4;
	+ if (keylen != 128/8 && keylen != 256 / 8) {
	+ mlx5_core_err(mdev, "Cannot offload xfrm states with AEAD key length other than 128/256 bit\n");
	+ return (EINVAL);
	+ }
	+
	+ if (saidx->mode != IPSEC_MODE_TRANSPORT) {
	+ mlx5_core_err(mdev, "Only transport xfrm states may be offloaded in full offlaod mode\n");
	+ return (EINVAL);
	+ }
	+
	+ if (savp->natt) {
	+ if (!(mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_ESPINUDP)) {
	+ mlx5_core_err(mdev, "Encapsulation is not supported\n");
	+ return (EINVAL);
	+ }
	+ }
	+
	+ if (savp->replay && savp->replay->wsize != 0 && savp->replay->wsize != 4 &&
	+ savp->replay->wsize != 8 && savp->replay->wsize != 16 && savp->replay->wsize != 32) {
	+ mlx5_core_err(mdev, "Unsupported replay window size %d\n", savp->replay->wsize);
	+ return (EINVAL);
	+ }
	+
	+ if ((savp->flags & SADB_X_SAFLAGS_ESN) != 0) {
	+ if ((mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_ESN) == 0) {
	+ mlx5_core_err(mdev, "ESN is not supported\n");
	+ return (EINVAL);
	+ }
	+ } else {
	+ mlx5_core_warn(mdev, "non-ESN SA offload is not supported\n");
	+ return (EINVAL);
	+ }
	+ return 0;
	+}
	+
	+static int
	+mlx5e_if_sa_newkey(struct ifnet ifp, void sav, int dir, uint32_t drv_spi,
	+ void **privp)
	+{
	+ struct mlx5e_ipsec_sa_entry *sa_entry = NULL;
	+ struct mlx5e_priv *priv = if_getsoftc(ifp);
	+ struct mlx5_core_dev *mdev = priv->mdev;
	+ struct mlx5e_ipsec *ipsec = priv->ipsec;
	+ int err;
	+
	+ if (priv->gone != 0 \|\| ipsec == NULL)
	+ return (EOPNOTSUPP);
	+
	+ err = mlx5e_xfrm_validate_state(mdev, sav);
	+ if (err)
	+ return err;
	+
	+ sa_entry = kzalloc(sizeof(*sa_entry), GFP_KERNEL);
	+ if (sa_entry == NULL)
	+ return (ENOMEM);
	+
	+ sa_entry->kspi = drv_spi;
	+ sa_entry->savp = sav;
	+ sa_entry->ifp = ifp;
	+ sa_entry->ipsec = ipsec;
	+
	+ mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &sa_entry->attrs, dir);
	+
	+ err = mlx5e_ipsec_create_dwork(sa_entry);
	+ if (err)
	+ goto err_xfrm;
	+
	+ /* create hw context */
	+ err = mlx5_ipsec_create_sa_ctx(sa_entry);
	+ if (err)
	+ goto err_sa_ctx;
	+
	+ err = mlx5e_accel_ipsec_fs_add_rule(sa_entry);
	+ if (err)
	+ goto err_fs;
	+
	+ *privp = sa_entry;
	+ if (sa_entry->dwork)
	+ queue_delayed_work(ipsec->wq, &sa_entry->dwork->dwork, MLX5_IPSEC_RESCHED);
	+
	+ err = xa_insert(&mdev->ipsec_sadb, sa_entry->ipsec_obj_id, sa_entry, GFP_KERNEL);
	+ if (err)
	+ goto err_xa;
	+
	+ return 0;
	+
	+err_xa:
	+ if (sa_entry->dwork)
	+ cancel_delayed_work_sync(&sa_entry->dwork->dwork);
	+ mlx5e_accel_ipsec_fs_del_rule(sa_entry);
	+err_fs:
	+ mlx5_ipsec_free_sa_ctx(sa_entry);
	+err_sa_ctx:
	+ kfree(sa_entry->dwork);
	+err_xfrm:
	+ kfree(sa_entry);
	+ mlx5_en_err(ifp, "Device failed to offload this state");
	+ return err;
	+}
	+
	+static int mlx5e_if_sa_deinstall(struct ifnet ifp, uint32_t dev_spi, void priv)
	+{
	+ struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(priv);
	+ struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
	+ struct mlx5e_ipsec_sa_entry *old;
	+
	+ old = xa_erase(&mdev->ipsec_sadb, sa_entry->ipsec_obj_id);
	+ WARN_ON(old != sa_entry);
	+
	+ if (sa_entry->dwork)
	+ cancel_delayed_work_sync(&sa_entry->dwork->dwork);
	+ mlx5e_accel_ipsec_fs_del_rule(sa_entry);
	+ mlx5_ipsec_free_sa_ctx(sa_entry);
	+ kfree(sa_entry->dwork);
	+ kfree(sa_entry);
	+ return 0;
	+}
	+
	+static int mlx5e_if_sa_cnt(struct ifnet ifp, void sa, uint32_t drv_spi,
	+ void priv, struct seclifetime lt)
	+{
	+ struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(priv);
	+ struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule;
	+ struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
	+ u64 packets;
	+
	+ mlx5_fc_query(mdev, ipsec_rule->fc, &packets, &lt->bytes);
	+ /* TODO: remove this casting once Kostia changes allocation type to be u64 */
	+ lt->allocations = (uint32_t)packets;
	+ return 0;
	+}
	+
	+static int mlx5e_xfrm_validate_policy(struct mlx5_core_dev *mdev,
	+ struct secpolicy sp, struct inpcb inp)
	+{
	+ struct secpolicyindex *spidx = &sp->spidx;
	+
	+ if (!(mlx5_ipsec_device_caps(mdev) &
	+ MLX5_IPSEC_CAP_PACKET_OFFLOAD)) {
	+ mlx5_core_err(mdev, "FULL offload is not supported\n");
	+ return (EINVAL);
	+ }
	+
	+ if (sp->tcount > 1) {
	+ mlx5_core_err(mdev, "Can offload exactly one template, "
	+ "not %d\n", sp->tcount);
	+ return (EINVAL);
	+ }
	+
	+ if (sp->policy == IPSEC_POLICY_BYPASS &&
	+ !(mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PRIO)) {
	+ mlx5_core_err(mdev, "Device does not support policy priority\n");
	+ return (EINVAL);
	+ }
	+
	+ if (sp->tcount > 0 && inp != NULL) {
	+ mlx5_core_err(mdev, "Not valid input data\n");
	+ return (EINVAL);
	+ }
	+
	+ if (spidx->dir != IPSEC_DIR_INBOUND && spidx->dir != IPSEC_DIR_OUTBOUND) {
	+ mlx5_core_err(mdev, "Wrong policy direction\n");
	+ return (EINVAL);
	+ }
	+
	+ if (sp->tcount > 0 && sp->req[0]->saidx.mode != IPSEC_MODE_TRANSPORT) {
	+ mlx5_core_err(mdev, "Device supports transport mode only");
	+ return (EINVAL);
	+ }
	+
	+ if (sp->policy != IPSEC_POLICY_DISCARD &&
	+ sp->policy != IPSEC_POLICY_IPSEC && sp->policy != IPSEC_POLICY_BYPASS) {
	+ mlx5_core_err(mdev, "Offloaded policy must be specific on its action\n");
	+ return (EINVAL);
	+ }
	+
	+ if (sp->policy == IPSEC_POLICY_BYPASS && !inp) {
	+ mlx5_core_err(mdev, "Missing port information for IKE bypass\n");
	+ return (EINVAL);
	+ }
	+
	+ if (inp != NULL) {
	+ INP_RLOCK(inp);
	+ if (inp->inp_socket == NULL \|\| inp->inp_socket->so_proto->
	+ pr_protocol != IPPROTO_UDP) {
	+ mlx5_core_err(mdev, "Unsupported IKE bypass protocol %d\n",
	+ inp->inp_socket == NULL ? -1 :
	+ inp->inp_socket->so_proto->pr_protocol);
	+ INP_RUNLOCK(inp);
	+ return (EINVAL);
	+ }
	+ INP_RUNLOCK(inp);
	+ }
	+
	+ /* TODO fill relevant bits */
	+ return 0;
	+}
	+
	+static void mlx5e_ipsec_build_accel_pol_attrs(struct mlx5e_ipsec_pol_entry *pol_entry,
	+ struct mlx5_accel_pol_xfrm_attrs *attrs,
	+ struct inpcb *inp)
	+{
	+ struct secpolicy *sp = pol_entry->sp;
	+ struct secpolicyindex *spidx = &sp->spidx;
	+
	+ memset(attrs, 0, sizeof(*attrs));
	+
	+ if (!inp) {
	+ if (spidx->src.sa.sa_family == AF_INET) {
	+ attrs->saddr.a4 = spidx->src.sin.sin_addr.s_addr;
	+ attrs->daddr.a4 = spidx->dst.sin.sin_addr.s_addr;
	+ } else if (spidx->src.sa.sa_family == AF_INET6) {
	+ memcpy(&attrs->saddr.a6, &spidx->src.sin6.sin6_addr, 16);
	+ memcpy(&attrs->daddr.a6, &spidx->dst.sin6.sin6_addr, 16);
	+ } else {
	+ KASSERT(0, ("unsupported family %d", spidx->src.sa.sa_family));
	+ }
	+ attrs->family = spidx->src.sa.sa_family;
	+ attrs->prio = 0;
	+ attrs->action = sp->policy;
	+ attrs->reqid = sp->req[0]->saidx.reqid;
	+ } else {
	+ INP_RLOCK(inp);
	+ if ((inp->inp_vflag & INP_IPV4) != 0) {
	+ attrs->saddr.a4 = inp->inp_laddr.s_addr;
	+ attrs->daddr.a4 = inp->inp_faddr.s_addr;
	+ attrs->family = AF_INET;
	+ } else if ((inp->inp_vflag & INP_IPV6) != 0) {
	+ memcpy(&attrs->saddr.a6, &inp->in6p_laddr, 16);
	+ memcpy(&attrs->daddr.a6, &inp->in6p_laddr, 16);
	+ attrs->family = AF_INET6;
	+ } else {
	+ KASSERT(0, ("unsupported family %d", inp->inp_vflag));
	+ }
	+ attrs->upspec.dport = inp->inp_fport;
	+ attrs->upspec.sport = inp->inp_lport;
	+ attrs->upspec.proto = inp->inp_ip_p;
	+ INP_RUNLOCK(inp);
	+
	+ /* Give highest priority for PCB policies */
	+ attrs->prio = 1;
	+ attrs->action = IPSEC_POLICY_IPSEC;
	+ }
	+ attrs->dir = spidx->dir;
	+}
	+
	+static int mlx5e_if_spd_install(struct ifnet ifp, void sp, void *inp1,
	+ void **ifdatap)
	+{
	+ struct mlx5e_ipsec_pol_entry *pol_entry;
	+ struct mlx5e_priv *priv;
	+ int err;
	+
	+ priv = if_getsoftc(ifp);
	+ if (priv->gone \|\| !priv->ipsec)
	+ return (EOPNOTSUPP);
	+
	+ err = mlx5e_xfrm_validate_policy(priv->mdev, sp, inp1);
	+ if (err)
	+ return err;
	+
	+ pol_entry = kzalloc(sizeof(*pol_entry), GFP_KERNEL);
	+ if (!pol_entry)
	+ return (ENOMEM);
	+
	+ pol_entry->sp = sp;
	+ pol_entry->ipsec = priv->ipsec;
	+
	+ mlx5e_ipsec_build_accel_pol_attrs(pol_entry, &pol_entry->attrs, inp1);
	+ err = mlx5e_accel_ipsec_fs_add_pol(pol_entry);
	+ if (err)
	+ goto err_pol;
	+ *ifdatap = pol_entry;
	+
	+ return 0;
	+
	+err_pol:
	+ kfree(pol_entry);
	+ mlx5_en_err(ifp, "Device failed to offload this policy");
	+ return err;
	+}
	+
	+
	+static int mlx5e_if_spd_deinstall(struct ifnet ifp, void sp, void *ifdata)
	+{
	+ struct mlx5e_ipsec_pol_entry *pol_entry = to_ipsec_pol_entry(ifdata);
	+
	+ mlx5e_accel_ipsec_fs_del_pol(pol_entry);
	+ kfree(pol_entry);
	+ return 0;
	+}
	+
	+void mlx5e_ipsec_cleanup(struct mlx5e_priv *priv)
	+{
	+ struct mlx5e_ipsec *pipsec = priv->ipsec;
	+ if (!pipsec)
	+ return;
	+
	+ mlx5e_accel_ipsec_fs_cleanup(pipsec);
	+ destroy_workqueue(pipsec->wq);
	+ mlx5e_ipsec_aso_cleanup(pipsec);
	+ kfree(pipsec);
	+ priv->ipsec = NULL;
	+}
	+
	+static int
	+mlx5e_if_ipsec_hwassist(if_t ifnet, void *sav __unused,
	+ uint32_t drv_spi __unused, void *priv __unused)
	+{
	+ return (if_gethwassist(ifnet) & (CSUM_TSO \| CSUM_TCP \| CSUM_UDP \|
	+ CSUM_IP \| CSUM_IP6_TSO \| CSUM_IP6_TCP \| CSUM_IP6_UDP));
	+}
	+
	+static const struct if_ipsec_accel_methods mlx5e_ipsec_funcs = {
	+ .if_sa_newkey = mlx5e_if_sa_newkey,
	+ .if_sa_deinstall = mlx5e_if_sa_deinstall,
	+ .if_spdadd = mlx5e_if_spd_install,
	+ .if_spddel = mlx5e_if_spd_deinstall,
	+ .if_sa_cnt = mlx5e_if_sa_cnt,
	+ .if_hwassist = mlx5e_if_ipsec_hwassist,
	+};
	+
	+int mlx5e_ipsec_init(struct mlx5e_priv *priv)
	+{
	+ struct mlx5_core_dev *mdev = priv->mdev;
	+ struct mlx5e_ipsec *pipsec;
	+ if_t ifp = priv->ifp;
	+ int ret;
	+
	+ mlx5_core_info(mdev, "ipsec "
	+ "offload %d log_max_dek %d gen_obj_types %d "
	+ "ipsec_encrypt %d ipsec_decrypt %d "
	+ "esp_aes_gcm_128_encrypt %d esp_aes_gcm_128_decrypt %d "
	+ "ipsec_full_offload %d "
	+ "reformat_add_esp_trasport %d reformat_del_esp_trasport %d "
	+ "decap %d "
	+ "ignore_flow_level_tx %d ignore_flow_level_rx %d "
	+ "reformat_natt_tx %d reformat_natt_rx %d "
	+ "ipsec_esn %d\n",
	+ MLX5_CAP_GEN(mdev, ipsec_offload) != 0,
	+ MLX5_CAP_GEN(mdev, log_max_dek) != 0,
	+ (MLX5_CAP_GEN_64(mdev, general_obj_types) &
	+ MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_IPSEC) != 0,
	+ MLX5_CAP_FLOWTABLE_NIC_TX(mdev, ipsec_encrypt) != 0,
	+ MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ipsec_decrypt) != 0,
	+ MLX5_CAP_IPSEC(mdev, ipsec_crypto_esp_aes_gcm_128_encrypt) != 0,
	+ MLX5_CAP_IPSEC(mdev, ipsec_crypto_esp_aes_gcm_128_decrypt) != 0,
	+ MLX5_CAP_IPSEC(mdev, ipsec_full_offload) != 0,
	+ MLX5_CAP_FLOWTABLE_NIC_TX(mdev, reformat_add_esp_trasport) != 0,
	+ MLX5_CAP_FLOWTABLE_NIC_RX(mdev, reformat_del_esp_trasport) != 0,
	+ MLX5_CAP_FLOWTABLE_NIC_RX(mdev, decap) != 0,
	+ MLX5_CAP_FLOWTABLE_NIC_TX(mdev, ignore_flow_level) != 0,
	+ MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ignore_flow_level) != 0,
	+ MLX5_CAP_FLOWTABLE_NIC_TX(mdev,
	+ reformat_add_esp_transport_over_udp) != 0,
	+ MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
	+ reformat_del_esp_transport_over_udp) != 0,
	+ MLX5_CAP_IPSEC(mdev, ipsec_esn) != 0);
	+
	+ if (!(mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PACKET_OFFLOAD)) {
	+ mlx5_core_dbg(mdev, "Not an IPSec offload device\n");
	+ return 0;
	+ }
	+
	+ xa_init_flags(&mdev->ipsec_sadb, XA_FLAGS_ALLOC);
	+
	+ pipsec = kzalloc(sizeof(*pipsec), GFP_KERNEL);
	+ if (pipsec == NULL)
	+ return (ENOMEM);
	+
	+ pipsec->mdev = mdev;
	+ pipsec->pdn = priv->pdn;
	+ pipsec->mkey = priv->mr.key;
	+
	+ ret = mlx5e_ipsec_aso_init(pipsec);
	+ if (ret)
	+ goto err_ipsec_aso;
	+
	+ pipsec->wq = alloc_workqueue("mlx5e_ipsec", WQ_UNBOUND, 0);
	+ if (pipsec->wq == NULL) {
	+ ret = ENOMEM;
	+ goto err_ipsec_wq;
	+ }
	+
	+ ret = mlx5e_accel_ipsec_fs_init(pipsec);
	+ if (ret)
	+ goto err_ipsec_alloc;
	+
	+ if_setipsec_accel_methods(ifp, &mlx5e_ipsec_funcs);
	+ priv->ipsec = pipsec;
	+ mlx5_core_dbg(mdev, "IPSec attached to netdevice\n");
	+ return 0;
	+
	+err_ipsec_alloc:
	+ destroy_workqueue(pipsec->wq);
	+err_ipsec_wq:
	+ mlx5e_ipsec_aso_cleanup(pipsec);
	+err_ipsec_aso:
	+ kfree(pipsec);
	+ mlx5_core_err(priv->mdev, "IPSec initialization failed, %d\n", ret);
	+ return ret;
	+}
	diff --git a/sys/dev/mlx5/mlx5_accel/mlx5_ipsec_fs.c b/sys/dev/mlx5/mlx5_accel/mlx5_ipsec_fs.c
	new file mode 100644
	--- /dev/null
	+++ b/sys/dev/mlx5/mlx5_accel/mlx5_ipsec_fs.c
	@@ -0,0 +1,2048 @@
	+/*-
	+ * Copyright (c) 2023 NVIDIA corporation & affiliates.
	+ *
	+ * Redistribution and use in source and binary forms, with or without
	+ * modification, are permitted provided that the following conditions
	+ * are met:
	+ * 1. Redistributions of source code must retain the above copyright
	+ * notice, this list of conditions and the following disclaimer.
	+ * 2. Redistributions in binary form must reproduce the above copyright
	+ * notice, this list of conditions and the following disclaimer in the
	+ * documentation and/or other materials provided with the distribution.
	+ *
	+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
	+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
	+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	+ * SUCH DAMAGE.
	+ *
	+ * $FreeBSD$
	+ */
	+
	+#include "opt_ipsec.h"
	+
	+#include <sys/types.h>
	+#include <netinet/in.h>
	+#include <sys/socket.h>
	+#include <sys/param.h>
	+#include <sys/systm.h>
	+#include <net/pfkeyv2.h>
	+#include <netipsec/key_var.h>
	+#include <netipsec/keydb.h>
	+#include <netipsec/ipsec.h>
	+#include <netipsec/xform.h>
	+#include <netipsec/ipsec_accel.h>
	+#include <dev/mlx5/fs.h>
	+#include <dev/mlx5/mlx5_en/en.h>
	+#include <dev/mlx5/qp.h>
	+#include <dev/mlx5/mlx5_accel/ipsec.h>
	+#include <dev/mlx5/mlx5_core/fs_core.h>
	+#include <dev/mlx5/mlx5_core/fs_chains.h>
	+
	+/*
	+ * TX tables are organized differently for Ethernet and for RoCE:
	+ *
	+ * +=========+
	+ * Ethernet Tx \| SA KSPI \| match
	+ * --------------------->\|Flowtable\|----->+ +
	+ * \| \|\ \| / \
	+ * +=========+ \| \| / \ +=========+ +=========+
	+ * miss \| \| / \ \| Status \| \| \|
	+ * DROP<--------+ \|---->\|Encrypt\|------>\|Flowtable\|---->\| TX NS \|
	+ * \| \ / \| \| \| \|
	+ * \| \ / +=========+ +=========+
	+ * +=========+ +=========+ \| \ / \|
	+ * RoCE \| Policy \| match\|SA ReqId \|match\| + \|
	+ * Tx \|Flowtable\|----->\|Flowtable\|---->+ \|
	+ * ---->\|IP header\| \|ReqId+IP \| \|
	+ * \| \| \| header \|--------------------------------+
	+ * +=========+ +=========+ miss \|
	+ * \| \|
	+ * \| miss \|
	+ * +-------------------------------------------------------
	+ *
	+ * +=========+
	+ * \| RDMA \|
	+ * \|Flowtable\|
	+ * \| \|
	+ * Rx Tables and rules: +=========+
	+ * + /
	+ * +=========+ +=========+ / \ +=========+ +=========+ /match
	+ * \| Policy \| \| SA \| / \ \| Status \| \| RoCE \|/
	+ * ---->\|Flowtable\| match\|Flowtable\| match / \ \|Flowtable\|----->\|Flowtable\|
	+ * \|IP header\|----->\|IP header\|----->\|Decrypt\|----->\| \| \| Roce V2 \|
	+ * \| \| \|+ESP+SPI \| \ / \| \| \| UDP port\|\
	+ * +=========+ +=========+ \ / +=========+ +=========+ \miss
	+ * \| \| \ / \
	+ * \| \| + +=========+
	+ * \| miss \| miss \| Ethernet\|
	+ * +--------------->---------------------------------------------------->\| RX NS \|
	+ * \| \|
	+ * +=========+
	+ *
	+ */
	+
	+#define NUM_IPSEC_FTE BIT(15)
	+#define IPSEC_TUNNEL_DEFAULT_TTL 0x40
	+
	+struct mlx5e_ipsec_fc {
	+ struct mlx5_fc *cnt;
	+ struct mlx5_fc *drop;
	+};
	+
	+struct mlx5e_ipsec_ft {
	+ struct mutex mutex; /* Protect changes to this struct */
	+ struct mlx5_flow_table *pol;
	+ struct mlx5_flow_table *sa_kspi;
	+ struct mlx5_flow_table *sa;
	+ struct mlx5_flow_table *status;
	+ u32 refcnt;
	+};
	+
	+struct mlx5e_ipsec_tx_roce {
	+ struct mlx5_flow_group *g;
	+ struct mlx5_flow_table *ft;
	+ struct mlx5_flow_handle *rule;
	+ struct mlx5_flow_namespace *ns;
	+};
	+
	+struct mlx5e_ipsec_miss {
	+ struct mlx5_flow_group *group;
	+ struct mlx5_flow_handle *rule;
	+};
	+
	+struct mlx5e_ipsec_tx {
	+ struct mlx5e_ipsec_ft ft;
	+ struct mlx5e_ipsec_miss pol;
	+ struct mlx5e_ipsec_miss kspi_miss;
	+ struct mlx5e_ipsec_rule status;
	+ struct mlx5e_ipsec_rule kspi_bypass_rule; /rule for IPSEC bypass/
	+ struct mlx5_flow_namespace *ns;
	+ struct mlx5e_ipsec_fc *fc;
	+ struct mlx5_fs_chains *chains;
	+ struct mlx5e_ipsec_tx_roce roce;
	+};
	+
	+struct mlx5e_ipsec_rx_roce {
	+ struct mlx5_flow_group *g;
	+ struct mlx5_flow_table *ft;
	+ struct mlx5_flow_handle *rule;
	+ struct mlx5e_ipsec_miss roce_miss;
	+
	+ struct mlx5_flow_table *ft_rdma;
	+ struct mlx5_flow_namespace *ns_rdma;
	+};
	+
	+struct mlx5e_ipsec_rx {
	+ struct mlx5e_ipsec_ft ft;
	+ struct mlx5e_ipsec_miss pol;
	+ struct mlx5e_ipsec_miss sa;
	+ struct mlx5e_ipsec_rule status;
	+ struct mlx5_flow_namespace *ns;
	+ struct mlx5e_ipsec_fc *fc;
	+ struct mlx5_fs_chains *chains;
	+ struct mlx5e_ipsec_rx_roce roce;
	+};
	+
	+static void setup_fte_reg_a_with_tag(struct mlx5_flow_spec *spec,
	+ u16 kspi);
	+static void setup_fte_reg_a_no_tag(struct mlx5_flow_spec *spec);
	+
	+static void setup_fte_no_frags(struct mlx5_flow_spec *spec)
	+{
	+ /* Non fragmented */
	+ spec->match_criteria_enable \|= MLX5_MATCH_OUTER_HEADERS;
	+
	+ MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.frag);
	+ MLX5_SET(fte_match_param, spec->match_value, outer_headers.frag, 0);
	+}
	+
	+static void setup_fte_esp(struct mlx5_flow_spec *spec)
	+{
	+ /* ESP header */
	+ spec->match_criteria_enable \|= MLX5_MATCH_OUTER_HEADERS;
	+
	+ MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol);
	+ MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_ESP);
	+}
	+
	+static void setup_fte_spi(struct mlx5_flow_spec *spec, u32 spi, bool encap)
	+{
	+ /* SPI number */
	+ spec->match_criteria_enable \|= MLX5_MATCH_MISC_PARAMETERS;
	+
	+ if (encap) {
	+ MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters.inner_esp_spi);
	+ MLX5_SET(fte_match_param, spec->match_value, misc_parameters.inner_esp_spi, spi);
	+ } else {
	+ MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters.outer_esp_spi);
	+ MLX5_SET(fte_match_param, spec->match_value, misc_parameters.outer_esp_spi, spi);
	+ }
	+}
	+
	+static struct mlx5_fs_chains *
	+ipsec_chains_create(struct mlx5_core_dev mdev, struct mlx5_flow_table miss_ft,
	+ enum mlx5_flow_namespace_type ns, int base_prio,
	+ int base_level, struct mlx5_flow_table **root_ft)
	+{
	+ struct mlx5_chains_attr attr = {};
	+ struct mlx5_fs_chains *chains;
	+ struct mlx5_flow_table *ft;
	+ int err;
	+
	+ attr.flags = MLX5_CHAINS_AND_PRIOS_SUPPORTED \|
	+ MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED;
	+ attr.max_grp_num = 2;
	+ attr.default_ft = miss_ft;
	+ attr.ns = ns;
	+ attr.fs_base_prio = base_prio;
	+ attr.fs_base_level = base_level;
	+ chains = mlx5_chains_create(mdev, &attr);
	+ if (IS_ERR(chains))
	+ return chains;
	+
	+ /* Create chain 0, prio 1, level 0 to connect chains to prev in fs_core */
	+ ft = mlx5_chains_get_table(chains, 0, 1, 0);
	+ if (IS_ERR(ft)) {
	+ err = PTR_ERR(ft);
	+ goto err_chains_get;
	+ }
	+
	+ *root_ft = ft;
	+ return chains;
	+
	+err_chains_get:
	+ mlx5_chains_destroy(chains);
	+ return ERR_PTR(err);
	+}
	+
	+static void ipsec_chains_destroy(struct mlx5_fs_chains *chains)
	+{
	+ mlx5_chains_put_table(chains, 0, 1, 0);
	+ mlx5_chains_destroy(chains);
	+}
	+
	+static struct mlx5_flow_table *
	+ipsec_chains_get_table(struct mlx5_fs_chains *chains, u32 prio)
	+{
	+ return mlx5_chains_get_table(chains, 0, prio + 1, 0);
	+}
	+
	+static void ipsec_chains_put_table(struct mlx5_fs_chains *chains, u32 prio)
	+{
	+ mlx5_chains_put_table(chains, 0, prio + 1, 0);
	+}
	+
	+static struct mlx5_flow_table ipsec_rx_ft_create(struct mlx5_flow_namespace ns,
	+ int level, int prio,
	+ int max_num_groups)
	+{
	+ struct mlx5_flow_table_attr ft_attr = {};
	+
	+ ft_attr.max_fte = NUM_IPSEC_FTE;
	+ ft_attr.level = level;
	+ ft_attr.prio = prio;
	+ ft_attr.autogroup.max_num_groups = max_num_groups;
	+ ft_attr.autogroup.num_reserved_entries = 1;
	+
	+ return mlx5_create_auto_grouped_flow_table(ns, &ft_attr);
	+}
	+
	+static int ipsec_miss_create(struct mlx5_core_dev *mdev,
	+ struct mlx5_flow_table *ft,
	+ struct mlx5e_ipsec_miss *miss,
	+ struct mlx5_flow_destination *dest)
	+{
	+ int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
	+ struct mlx5_flow_act flow_act = {};
	+ struct mlx5_flow_spec *spec;
	+ u32 *flow_group_in;
	+ int err = 0;
	+
	+ flow_group_in = kvzalloc(inlen, GFP_KERNEL);
	+ spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
	+ if (!flow_group_in \|\| !spec) {
	+ err = -ENOMEM;
	+ goto out;
	+ }
	+
	+ /* Create miss_group */
	+ MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ft->max_fte - 1);
	+ MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, ft->max_fte - 1);
	+ miss->group = mlx5_create_flow_group(ft, flow_group_in);
	+ if (IS_ERR(miss->group)) {
	+ err = PTR_ERR(miss->group);
	+ mlx5_core_err(mdev, "fail to create IPsec miss_group err=%d\n",
	+ err);
	+ goto out;
	+ }
	+
	+ if (dest)
	+ flow_act.action = MLX5_FLOW_RULE_FWD_ACTION_DEST;
	+ else
	+ flow_act.action = MLX5_FLOW_RULE_FWD_ACTION_DROP;
	+ /* Create miss rule */
	+ miss->rule = mlx5_add_flow_rules(ft, NULL, &flow_act, dest, 1);
	+ if (IS_ERR(miss->rule)) {
	+ mlx5_destroy_flow_group(miss->group);
	+ err = PTR_ERR(miss->rule);
	+ mlx5_core_err(mdev, "fail to create IPsec miss_rule err=%d\n",
	+ err);
	+ goto out;
	+ }
	+out:
	+ kvfree(flow_group_in);
	+ kvfree(spec);
	+ return err;
	+}
	+
	+static int setup_modify_header(struct mlx5_core_dev *mdev, u32 val, u8 dir,
	+ struct mlx5_flow_act *flow_act)
	+{
	+ u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {};
	+ enum mlx5_flow_namespace_type ns_type;
	+ struct mlx5_modify_hdr *modify_hdr;
	+
	+ MLX5_SET(set_action_in, action, action_type, MLX5_ACTION_TYPE_SET);
	+ switch (dir) {
	+ case IPSEC_DIR_INBOUND:
	+ MLX5_SET(set_action_in, action, field,
	+ MLX5_ACTION_IN_FIELD_METADATA_REG_B);
	+ ns_type = MLX5_FLOW_NAMESPACE_KERNEL;
	+ break;
	+ case IPSEC_DIR_OUTBOUND:
	+ MLX5_SET(set_action_in, action, field,
	+ MLX5_ACTION_IN_FIELD_METADATA_REG_C_0);
	+ ns_type = MLX5_FLOW_NAMESPACE_EGRESS;
	+ break;
	+ default:
	+ return -EINVAL;
	+ }
	+
	+ MLX5_SET(set_action_in, action, data, val);
	+ MLX5_SET(set_action_in, action, offset, 0);
	+ MLX5_SET(set_action_in, action, length, 32);
	+
	+ modify_hdr = mlx5_modify_header_alloc(mdev, ns_type, 1, action);
	+ if (IS_ERR(modify_hdr)) {
	+ mlx5_core_err(mdev, "Failed to allocate modify_header %ld\n",
	+ PTR_ERR(modify_hdr));
	+ return PTR_ERR(modify_hdr);
	+ }
	+
	+ flow_act->modify_hdr = modify_hdr;
	+ flow_act->action \|= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
	+ return 0;
	+}
	+
	+static int
	+setup_pkt_transport_reformat(struct mlx5_accel_esp_xfrm_attrs *attrs,
	+ struct mlx5_pkt_reformat_params *reformat_params)
	+{
	+ struct udphdr *udphdr;
	+ size_t bfflen = 16;
	+ char *reformatbf;
	+ __be32 spi;
	+ void *hdr;
	+
	+ if (attrs->family == AF_INET)
	+ if (attrs->encap)
	+ reformat_params->type = MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_UDPV4;
	+ else
	+ reformat_params->type = MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV4;
	+ else
	+ reformat_params->type =
	+ MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV6;
	+
	+ if (attrs->encap)
	+ bfflen += sizeof(*udphdr);
	+ reformatbf = kzalloc(bfflen, GFP_KERNEL);
	+ if (!reformatbf)
	+ return -ENOMEM;
	+
	+ hdr = reformatbf;
	+ if (attrs->encap) {
	+ udphdr = (struct udphdr *)reformatbf;
	+ udphdr->uh_sport = attrs->sport;
	+ udphdr->uh_dport = attrs->dport;
	+ hdr += sizeof(*udphdr);
	+ }
	+
	+ /* convert to network format */
	+ spi = htonl(attrs->spi);
	+ memcpy(hdr, &spi, 4);
	+
	+ reformat_params->param_0 = attrs->authsize;
	+ reformat_params->size = bfflen;
	+ reformat_params->data = reformatbf;
	+
	+ return 0;
	+}
	+
	+static int setup_pkt_reformat(struct mlx5_core_dev *mdev,
	+ struct mlx5_accel_esp_xfrm_attrs *attrs,
	+ struct mlx5_flow_act *flow_act)
	+{
	+ enum mlx5_flow_namespace_type ns_type = MLX5_FLOW_NAMESPACE_EGRESS;
	+ struct mlx5_pkt_reformat_params reformat_params = {};
	+ struct mlx5_pkt_reformat *pkt_reformat;
	+ int ret;
	+
	+ if (attrs->dir == IPSEC_DIR_INBOUND) {
	+ if (attrs->encap)
	+ reformat_params.type = MLX5_REFORMAT_TYPE_DEL_ESP_TRANSPORT_OVER_UDP;
	+ else
	+ reformat_params.type = MLX5_REFORMAT_TYPE_DEL_ESP_TRANSPORT;
	+ ns_type = MLX5_FLOW_NAMESPACE_KERNEL;
	+ goto cmd;
	+ }
	+
	+ ret = setup_pkt_transport_reformat(attrs, &reformat_params);
	+ if (ret)
	+ return ret;
	+cmd:
	+ pkt_reformat =
	+ mlx5_packet_reformat_alloc(mdev, &reformat_params, ns_type);
	+ if (reformat_params.data)
	+ kfree(reformat_params.data);
	+ if (IS_ERR(pkt_reformat))
	+ return PTR_ERR(pkt_reformat);
	+
	+ flow_act->pkt_reformat = pkt_reformat;
	+ flow_act->action \|= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
	+ return 0;
	+}
	+
	+static void setup_fte_addr4(struct mlx5_flow_spec spec, __be32 saddr,
	+ __be32 *daddr)
	+{
	+ spec->match_criteria_enable \|= MLX5_MATCH_OUTER_HEADERS;
	+
	+ MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version);
	+ MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 4);
	+
	+ memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
	+ outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4), saddr, 4);
	+ memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
	+ outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4), daddr, 4);
	+ MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
	+ outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4);
	+ MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
	+ outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
	+}
	+
	+static void setup_fte_addr6(struct mlx5_flow_spec spec, __be32 saddr,
	+ __be32 *daddr)
	+{
	+ spec->match_criteria_enable \|= MLX5_MATCH_OUTER_HEADERS;
	+
	+ MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version);
	+ MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 6);
	+
	+ memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
	+ outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), saddr, 16);
	+ memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
	+ outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), daddr, 16);
	+ memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
	+ outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), 0xff, 16);
	+ memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
	+ outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 0xff, 16);
	+}
	+
	+static int rx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry)
	+{
	+ struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule;
	+ struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs;
	+ struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
	+ struct mlx5e_ipsec *ipsec = sa_entry->ipsec;
	+ struct mlx5_flow_destination dest[2] = {};
	+ struct mlx5_flow_act flow_act = {};
	+ struct mlx5_flow_handle *rule;
	+ struct mlx5_flow_spec *spec;
	+ struct mlx5e_ipsec_rx *rx;
	+ struct mlx5_fc *counter;
	+ int err;
	+
	+ rx = (attrs->family == AF_INET) ? ipsec->rx_ipv4 : ipsec->rx_ipv6;
	+
	+ spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
	+ if (!spec)
	+ return -ENOMEM;
	+
	+ if (attrs->family == AF_INET)
	+ setup_fte_addr4(spec, &attrs->saddr.a4, &attrs->daddr.a4);
	+ else
	+ setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6);
	+
	+ if (!attrs->encap)
	+ setup_fte_esp(spec);
	+
	+ setup_fte_spi(spec, attrs->spi, attrs->encap);
	+ setup_fte_no_frags(spec);
	+
	+ if (!attrs->drop) {
	+ err = setup_modify_header(mdev, sa_entry->kspi \| BIT(31), IPSEC_DIR_INBOUND,
	+ &flow_act);
	+ if (err)
	+ goto err_mod_header;
	+ }
	+
	+ err = setup_pkt_reformat(mdev, attrs, &flow_act);
	+ if (err)
	+ goto err_pkt_reformat;
	+
	+ counter = mlx5_fc_create(mdev, false);
	+ if (IS_ERR(counter)) {
	+ err = PTR_ERR(counter);
	+ goto err_add_cnt;
	+ }
	+
	+ flow_act.crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_IPSEC;
	+ flow_act.crypto.op = MLX5_FLOW_ACT_CRYPTO_OP_DECRYPT;
	+ flow_act.crypto.obj_id = sa_entry->ipsec_obj_id;
	+ flow_act.flags \|= FLOW_ACT_NO_APPEND;
	+
	+ flow_act.action \|= MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT \|
	+ MLX5_FLOW_CONTEXT_ACTION_COUNT;
	+
	+ if (attrs->drop)
	+ flow_act.action \|= MLX5_FLOW_CONTEXT_ACTION_DROP;
	+ else
	+ flow_act.action \|= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
	+
	+ dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
	+ dest[0].ft = rx->ft.status;
	+ dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
	+ dest[1].counter_id = mlx5_fc_id(counter);
	+
	+ rule = mlx5_add_flow_rules(rx->ft.sa, spec, &flow_act, dest, 2);
	+ if (IS_ERR(rule)) {
	+ err = PTR_ERR(rule);
	+ mlx5_core_err(mdev, "fail to add RX ipsec rule err=%d\n", err);
	+ goto err_add_flow;
	+ }
	+
	+ kvfree(spec);
	+ ipsec_rule->rule = rule;
	+ ipsec_rule->fc = counter;
	+ ipsec_rule->modify_hdr = flow_act.modify_hdr;
	+ ipsec_rule->pkt_reformat = flow_act.pkt_reformat;
	+ return 0;
	+
	+err_add_flow:
	+ mlx5_fc_destroy(mdev, counter);
	+err_add_cnt:
	+ mlx5_packet_reformat_dealloc(mdev, flow_act.pkt_reformat);
	+err_pkt_reformat:
	+ if (flow_act.modify_hdr)
	+ mlx5_modify_header_dealloc(mdev, flow_act.modify_hdr);
	+err_mod_header:
	+ kvfree(spec);
	+
	+ return err;
	+}
	+
	+static struct mlx5_flow_table ipsec_tx_ft_create(struct mlx5_flow_namespace ns,
	+ int level, int prio,
	+ int max_num_groups)
	+{
	+ struct mlx5_flow_table_attr ft_attr = {};
	+
	+ ft_attr.autogroup.num_reserved_entries = 1;
	+ ft_attr.autogroup.max_num_groups = max_num_groups;
	+ ft_attr.max_fte = NUM_IPSEC_FTE;
	+ ft_attr.level = level;
	+ ft_attr.prio = prio;
	+
	+ return mlx5_create_auto_grouped_flow_table(ns, &ft_attr);
	+}
	+
	+static int ipsec_counter_rule_tx(struct mlx5_core_dev mdev, struct mlx5e_ipsec_tx tx)
	+{
	+ struct mlx5_flow_destination dest = {};
	+ struct mlx5_flow_act flow_act = {};
	+ struct mlx5_flow_handle *fte;
	+ int err;
	+
	+ /* create fte */
	+ flow_act.action = MLX5_FLOW_CONTEXT_ACTION_COUNT \|
	+ MLX5_FLOW_CONTEXT_ACTION_ALLOW;
	+
	+ dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
	+ dest.counter_id = mlx5_fc_id(tx->fc->cnt);
	+ fte = mlx5_add_flow_rules(tx->ft.status, NULL, &flow_act, &dest, 1);
	+ if (IS_ERR_OR_NULL(fte)) {
	+ err = PTR_ERR(fte);
	+ mlx5_core_err(mdev, "Fail to add ipsec tx counter rule err=%d\n", err);
	+ goto err_rule;
	+ }
	+
	+ tx->status.rule = fte;
	+ return 0;
	+
	+err_rule:
	+ return err;
	+}
	+
	+static void tx_destroy_roce(struct mlx5e_ipsec_tx *tx) {
	+ if (!tx->roce.ft)
	+ return;
	+
	+ mlx5_del_flow_rules(&tx->roce.rule);
	+ mlx5_destroy_flow_group(tx->roce.g);
	+ mlx5_destroy_flow_table(tx->roce.ft);
	+ tx->roce.ft = NULL;
	+}
	+
	+/* IPsec TX flow steering */
	+static void tx_destroy(struct mlx5e_ipsec_tx *tx)
	+{
	+ tx_destroy_roce(tx);
	+ if (tx->chains) {
	+ ipsec_chains_destroy(tx->chains);
	+ } else {
	+ mlx5_del_flow_rules(&tx->pol.rule);
	+ mlx5_destroy_flow_group(tx->pol.group);
	+ mlx5_destroy_flow_table(tx->ft.pol);
	+ }
	+ mlx5_destroy_flow_table(tx->ft.sa);
	+ mlx5_del_flow_rules(&tx->kspi_miss.rule);
	+ mlx5_destroy_flow_group(tx->kspi_miss.group);
	+ mlx5_del_flow_rules(&tx->kspi_bypass_rule.rule);
	+ mlx5_del_flow_rules(&tx->kspi_bypass_rule.kspi_rule);
	+ mlx5_destroy_flow_table(tx->ft.sa_kspi);
	+ mlx5_del_flow_rules(&tx->status.rule);
	+ mlx5_destroy_flow_table(tx->ft.status);
	+}
	+
	+static int ipsec_tx_roce_rule_setup(struct mlx5_core_dev *mdev,
	+ struct mlx5e_ipsec_tx *tx)
	+{
	+ struct mlx5_flow_destination dst = {};
	+ struct mlx5_flow_act flow_act = {};
	+ struct mlx5_flow_handle *rule;
	+ int err = 0;
	+
	+ flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
	+ dst.type = MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE;
	+ dst.ft = tx->ft.pol;
	+ rule = mlx5_add_flow_rules(tx->roce.ft, NULL, &flow_act, &dst, 1);
	+ if (IS_ERR(rule)) {
	+ err = PTR_ERR(rule);
	+ mlx5_core_err(mdev, "Fail to add TX roce ipsec rule err=%d\n",
	+ err);
	+ goto out;
	+ }
	+ tx->roce.rule = rule;
	+
	+out:
	+ return err;
	+}
	+
	+static int ipsec_tx_create_roce(struct mlx5_core_dev mdev, struct mlx5e_ipsec_tx tx)
	+{
	+ struct mlx5_flow_table_attr ft_attr = {};
	+ struct mlx5_flow_table *ft;
	+ struct mlx5_flow_group *g;
	+ int ix = 0;
	+ int err;
	+ u32 *in;
	+
	+ if (!tx->roce.ns)
	+ return -EOPNOTSUPP;
	+
	+ in = kvzalloc(MLX5_ST_SZ_BYTES(create_flow_group_in), GFP_KERNEL);
	+ if (!in)
	+ return -ENOMEM;
	+
	+ ft_attr.max_fte = 1;
	+ ft = mlx5_create_flow_table(tx->roce.ns, &ft_attr);
	+ if (IS_ERR(ft)) {
	+ err = PTR_ERR(ft);
	+ mlx5_core_err(mdev, "Fail to create ipsec tx roce ft err=%d\n",
	+ err);
	+ goto fail_table;
	+ }
	+ tx->roce.ft = ft;
	+
	+ MLX5_SET_CFG(in, start_flow_index, ix);
	+ ix += 1;
	+ MLX5_SET_CFG(in, end_flow_index, ix - 1);
	+ g = mlx5_create_flow_group(ft, in);
	+ if (IS_ERR(g)) {
	+ err = PTR_ERR(g);
	+ mlx5_core_err(mdev, "Fail to create ipsec tx roce group err=%d\n",
	+ err);
	+ goto fail_group;
	+ }
	+ tx->roce.g = g;
	+
	+ err = ipsec_tx_roce_rule_setup(mdev, tx);
	+ if (err) {
	+ mlx5_core_err(mdev, "Fail to create RoCE IPsec tx rules err=%d\n", err);
	+ goto fail_rule;
	+ }
	+
	+ kvfree(in);
	+ return 0;
	+
	+fail_rule:
	+ mlx5_destroy_flow_group(tx->roce.g);
	+fail_group:
	+ mlx5_destroy_flow_table(tx->roce.ft);
	+ tx->roce.ft = NULL;
	+fail_table:
	+ kvfree(in);
	+ return err;
	+}
	+
	+/*
	+ * Setting a rule in KSPI table for values that should bypass IPSEC.
	+ *
	+ * mdev - mlx5 core device
	+ * tx - IPSEC TX
	+ * return - 0 for success errno for failure
	+ */
	+static int tx_create_kspi_bypass_rules(struct mlx5_core_dev *mdev,
	+ struct mlx5e_ipsec_tx *tx)
	+{
	+ struct mlx5_flow_destination dest = {};
	+ struct mlx5_flow_act flow_act = {};
	+ struct mlx5_flow_act flow_act_kspi = {};
	+ struct mlx5_flow_handle *rule;
	+ struct mlx5_flow_spec *spec;
	+ int err;
	+
	+ spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
	+ if (!spec)
	+ return -ENOMEM;
	+
	+ dest.ft = tx->ft.status;
	+ dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
	+ flow_act_kspi.action \|= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
	+
	+ setup_fte_reg_a_with_tag(spec, IPSEC_ACCEL_DRV_SPI_BYPASS);
	+ rule = mlx5_add_flow_rules(tx->ft.sa_kspi, spec, &flow_act_kspi,
	+ &dest, 1);
	+ if (IS_ERR(rule)) {
	+ err = PTR_ERR(rule);
	+ mlx5_core_err(mdev, "Fail to add ipsec kspi bypass rule err=%d\n",
	+ err);
	+ goto err_add_kspi_rule;
	+ }
	+ tx->kspi_bypass_rule.kspi_rule = rule;
	+
	+ /* set the rule for packets withoiut ipsec tag. */
	+ flow_act.action \|= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
	+ memset(spec, 0, sizeof(*spec));
	+ setup_fte_reg_a_no_tag(spec);
	+ rule = mlx5_add_flow_rules(tx->ft.sa_kspi, spec, &flow_act, &dest, 1);
	+ if (IS_ERR(rule)) {
	+ err = PTR_ERR(rule);
	+ mlx5_core_err(mdev, "Fail to add ipsec kspi bypass rule err=%d\n", err);
	+ goto err_add_rule;
	+ }
	+ tx->kspi_bypass_rule.rule = rule;
	+
	+ kvfree(spec);
	+ return 0;
	+err_add_rule:
	+ mlx5_del_flow_rules(&tx->kspi_bypass_rule.kspi_rule);
	+err_add_kspi_rule:
	+ kvfree(spec);
	+ return err;
	+}
	+
	+
	+static int tx_create(struct mlx5_core_dev mdev, struct mlx5e_ipsec_tx tx)
	+{
	+ struct mlx5_flow_destination dest = {};
	+ struct mlx5_flow_table *ft;
	+ int err;
	+
	+ /*
	+ * Tx flow is different for ethernet traffic then for RoCE packets
	+ * For Ethernet packets we start in SA KSPI table that matches KSPI of SA rule
	+ * to the KSPI in the packet metadata
	+ * For RoCE traffic we start in Policy table, then move to SA table
	+ * which matches either reqid of the SA rule to reqid reported by policy table
	+ * or ip header fields of SA to the packet IP header fields.
	+ * Tables are ordered by their level so we set kspi
	+ * with level 0 to have it first one for ethernet traffic.
	+ * For RoCE the RoCE TX table direct the packets to policy table explicitly
	+ */
	+ ft = ipsec_tx_ft_create(tx->ns, 0, 0, 4);
	+ if (IS_ERR(ft))
	+ return PTR_ERR(ft);
	+ tx->ft.sa_kspi = ft;
	+
	+ ft = ipsec_tx_ft_create(tx->ns, 2, 0, 4);
	+ if (IS_ERR(ft)) {
	+ err = PTR_ERR(ft);
	+ goto err_reqid_ft;
	+ }
	+ tx->ft.sa = ft;
	+
	+ if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PRIO) {
	+ tx->chains = ipsec_chains_create(
	+ mdev, tx->ft.sa, MLX5_FLOW_NAMESPACE_EGRESS_IPSEC, 0, 1,
	+ &tx->ft.pol);
	+ if (IS_ERR(tx->chains)) {
	+ err = PTR_ERR(tx->chains);
	+ goto err_pol_ft;
	+ }
	+ } else {
	+ ft = ipsec_tx_ft_create(tx->ns, 1, 0, 2);
	+ if (IS_ERR(ft)) {
	+ err = PTR_ERR(ft);
	+ goto err_pol_ft;
	+ }
	+ tx->ft.pol = ft;
	+ dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
	+ dest.ft = tx->ft.sa;
	+ err = ipsec_miss_create(mdev, tx->ft.pol, &tx->pol, &dest);
	+ if (err)
	+ goto err_pol_miss;
	+ }
	+
	+ ft = ipsec_tx_ft_create(tx->ns, 2, 0, 1);
	+ if (IS_ERR(ft)) {
	+ err = PTR_ERR(ft);
	+ goto err_status_ft;
	+ }
	+ tx->ft.status = ft;
	+
	+ /* set miss rule for kspi table with drop action*/
	+ err = ipsec_miss_create(mdev, tx->ft.sa_kspi, &tx->kspi_miss, NULL);
	+ if (err)
	+ goto err_kspi_miss;
	+
	+ err = tx_create_kspi_bypass_rules(mdev, tx);
	+ if (err)
	+ goto err_kspi_rule;
	+
	+ err = ipsec_counter_rule_tx(mdev, tx);
	+ if (err)
	+ goto err_status_rule;
	+
	+ err = ipsec_tx_create_roce(mdev, tx);
	+ if (err)
	+ goto err_counter_rule;
	+
	+ return 0;
	+
	+err_counter_rule:
	+ mlx5_del_flow_rules(&tx->status.rule);
	+err_status_rule:
	+ mlx5_del_flow_rules(&tx->kspi_bypass_rule.rule);
	+ mlx5_del_flow_rules(&tx->kspi_bypass_rule.kspi_rule);
	+err_kspi_rule:
	+ mlx5_destroy_flow_table(tx->ft.status);
	+err_status_ft:
	+ if (tx->chains) {
	+ ipsec_chains_destroy(tx->chains);
	+ } else {
	+ mlx5_del_flow_rules(&tx->pol.rule);
	+ mlx5_destroy_flow_group(tx->pol.group);
	+ }
	+err_pol_miss:
	+ if (!tx->chains)
	+ mlx5_destroy_flow_table(tx->ft.pol);
	+err_pol_ft:
	+ mlx5_del_flow_rules(&tx->kspi_miss.rule);
	+ mlx5_destroy_flow_group(tx->kspi_miss.group);
	+err_kspi_miss:
	+ mlx5_destroy_flow_table(tx->ft.sa);
	+err_reqid_ft:
	+ mlx5_destroy_flow_table(tx->ft.sa_kspi);
	+ return err;
	+}
	+
	+static int tx_get(struct mlx5_core_dev mdev, struct mlx5e_ipsec ipsec,
	+ struct mlx5e_ipsec_tx *tx)
	+{
	+ int err;
	+
	+ if (tx->ft.refcnt)
	+ goto skip;
	+
	+ err = tx_create(mdev, tx);
	+ if (err)
	+ return err;
	+
	+skip:
	+ tx->ft.refcnt++;
	+ return 0;
	+}
	+
	+static void tx_put(struct mlx5e_ipsec ipsec, struct mlx5e_ipsec_tx tx)
	+{
	+ if (--tx->ft.refcnt)
	+ return;
	+
	+ tx_destroy(tx);
	+}
	+
	+static struct mlx5e_ipsec_tx tx_ft_get(struct mlx5_core_dev mdev,
	+ struct mlx5e_ipsec *ipsec)
	+{
	+ struct mlx5e_ipsec_tx *tx = ipsec->tx;
	+ int err;
	+
	+ mutex_lock(&tx->ft.mutex);
	+ err = tx_get(mdev, ipsec, tx);
	+ mutex_unlock(&tx->ft.mutex);
	+ if (err)
	+ return ERR_PTR(err);
	+
	+ return tx;
	+}
	+
	+static struct mlx5_flow_table tx_ft_get_policy(struct mlx5_core_dev mdev,
	+ struct mlx5e_ipsec *ipsec,
	+ u32 prio)
	+{
	+ struct mlx5e_ipsec_tx *tx = ipsec->tx;
	+ struct mlx5_flow_table *ft;
	+ int err;
	+
	+ mutex_lock(&tx->ft.mutex);
	+ err = tx_get(mdev, ipsec, tx);
	+ if (err)
	+ goto err_get;
	+
	+ ft = tx->chains ? ipsec_chains_get_table(tx->chains, prio) : tx->ft.pol;
	+ if (IS_ERR(ft)) {
	+ err = PTR_ERR(ft);
	+ goto err_get_ft;
	+ }
	+
	+ mutex_unlock(&tx->ft.mutex);
	+ return ft;
	+
	+err_get_ft:
	+ tx_put(ipsec, tx);
	+err_get:
	+ mutex_unlock(&tx->ft.mutex);
	+ return ERR_PTR(err);
	+}
	+
	+static void tx_ft_put_policy(struct mlx5e_ipsec *ipsec, u32 prio)
	+{
	+ struct mlx5e_ipsec_tx *tx = ipsec->tx;
	+
	+ mutex_lock(&tx->ft.mutex);
	+ if (tx->chains)
	+ ipsec_chains_put_table(tx->chains, prio);
	+
	+ tx_put(ipsec, tx);
	+ mutex_unlock(&tx->ft.mutex);
	+}
	+
	+static void tx_ft_put(struct mlx5e_ipsec *ipsec)
	+{
	+ struct mlx5e_ipsec_tx *tx = ipsec->tx;
	+
	+ mutex_lock(&tx->ft.mutex);
	+ tx_put(ipsec, tx);
	+ mutex_unlock(&tx->ft.mutex);
	+}
	+
	+static void setup_fte_reg_a_with_tag(struct mlx5_flow_spec *spec,
	+ u16 kspi)
	+{
	+ /* Add IPsec indicator in metadata_reg_a. */
	+ spec->match_criteria_enable \|= MLX5_MATCH_MISC_PARAMETERS_2;
	+
	+ MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
	+ misc_parameters_2.metadata_reg_a);
	+ MLX5_SET(fte_match_param, spec->match_value,
	+ misc_parameters_2.metadata_reg_a,
	+ MLX5_ETH_WQE_FT_META_IPSEC << 23 \| kspi);
	+}
	+
	+static void setup_fte_reg_a_no_tag(struct mlx5_flow_spec *spec)
	+{
	+ /* Add IPsec indicator in metadata_reg_a. */
	+ spec->match_criteria_enable \|= MLX5_MATCH_MISC_PARAMETERS_2;
	+
	+ MLX5_SET(fte_match_param, spec->match_criteria,
	+ misc_parameters_2.metadata_reg_a,
	+ MLX5_ETH_WQE_FT_META_IPSEC << 23);
	+ MLX5_SET(fte_match_param, spec->match_value,
	+ misc_parameters_2.metadata_reg_a,
	+ 0);
	+}
	+
	+static void setup_fte_reg_c0(struct mlx5_flow_spec *spec, u32 reqid)
	+{
	+ /* Pass policy check before choosing this SA */
	+ spec->match_criteria_enable \|= MLX5_MATCH_MISC_PARAMETERS_2;
	+
	+ MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
	+ misc_parameters_2.metadata_reg_c_0);
	+ MLX5_SET(fte_match_param, spec->match_value,
	+ misc_parameters_2.metadata_reg_c_0, reqid);
	+}
	+
	+static void setup_fte_upper_proto_match(struct mlx5_flow_spec spec, struct upspec upspec)
	+{
	+ switch (upspec->proto) {
	+ case IPPROTO_UDP:
	+ if (upspec->dport) {
	+ MLX5_SET_TO_ONES(fte_match_set_lyr_2_4,
	+ spec->match_criteria, udp_dport);
	+ MLX5_SET(fte_match_set_lyr_2_4, spec->match_value,
	+ udp_dport, upspec->dport);
	+ }
	+
	+ if (upspec->sport) {
	+ MLX5_SET_TO_ONES(fte_match_set_lyr_2_4,
	+ spec->match_criteria, udp_sport);
	+ MLX5_SET(fte_match_set_lyr_2_4, spec->match_value,
	+ udp_dport, upspec->sport);
	+ }
	+ break;
	+ case IPPROTO_TCP:
	+ if (upspec->dport) {
	+ MLX5_SET_TO_ONES(fte_match_set_lyr_2_4,
	+ spec->match_criteria, tcp_dport);
	+ MLX5_SET(fte_match_set_lyr_2_4, spec->match_value,
	+ tcp_dport, upspec->dport);
	+ }
	+
	+ if (upspec->sport) {
	+ MLX5_SET_TO_ONES(fte_match_set_lyr_2_4,
	+ spec->match_criteria, tcp_sport);
	+ MLX5_SET(fte_match_set_lyr_2_4, spec->match_value,
	+ tcp_dport, upspec->sport);
	+ }
	+ break;
	+ default:
	+ return;
	+ }
	+
	+ spec->match_criteria_enable \|= MLX5_MATCH_OUTER_HEADERS;
	+ MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, spec->match_criteria, ip_protocol);
	+ MLX5_SET(fte_match_set_lyr_2_4, spec->match_value, ip_protocol, upspec->proto);
	+}
	+
	+static int tx_add_kspi_rule(struct mlx5e_ipsec_sa_entry *sa_entry,
	+ struct mlx5e_ipsec_tx *tx,
	+ struct mlx5_flow_act *flow_act,
	+ struct mlx5_flow_destination *dest,
	+ int num_dest)
	+{
	+ struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule;
	+ struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
	+ struct mlx5_flow_handle *rule;
	+ struct mlx5_flow_spec *spec;
	+ int err;
	+
	+ spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
	+ if (!spec)
	+ return -ENOMEM;
	+
	+ setup_fte_no_frags(spec);
	+ setup_fte_reg_a_with_tag(spec, sa_entry->kspi);
	+
	+ rule = mlx5_add_flow_rules(tx->ft.sa_kspi, spec, flow_act, dest, num_dest);
	+ if (IS_ERR(rule)) {
	+ err = PTR_ERR(rule);
	+ mlx5_core_err(mdev, "fail to add TX ipsec kspi rule err=%d\n", err);
	+ goto err_add_kspi_flow;
	+ }
	+ ipsec_rule->kspi_rule = rule;
	+ kvfree(spec);
	+ return 0;
	+
	+err_add_kspi_flow:
	+ kvfree(spec);
	+ return err;
	+}
	+
	+static int tx_add_reqid_ip_rules(struct mlx5e_ipsec_sa_entry *sa_entry,
	+ struct mlx5e_ipsec_tx *tx,
	+ struct mlx5_flow_act *flow_act,
	+ struct mlx5_flow_destination *dest,
	+ int num_dest)
	+{
	+ struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule;
	+ struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs;
	+ struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
	+ struct mlx5_flow_handle *rule;
	+ struct mlx5_flow_spec *spec;
	+ int err;
	+
	+ spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
	+ if (!spec)
	+ return -ENOMEM;
	+
	+ flow_act->flags \|= FLOW_ACT_IGNORE_FLOW_LEVEL;
	+
	+ if(attrs->reqid) {
	+ setup_fte_no_frags(spec);
	+ setup_fte_reg_c0(spec, attrs->reqid);
	+ rule = mlx5_add_flow_rules(tx->ft.sa, spec, flow_act, dest, num_dest);
	+ if (IS_ERR(rule)) {
	+ err = PTR_ERR(rule);
	+ mlx5_core_err(mdev, "fail to add TX ipsec reqid rule err=%d\n", err);
	+ goto err_add_reqid_rule;
	+ }
	+ ipsec_rule->reqid_rule = rule;
	+ memset(spec, 0, sizeof(*spec));
	+ }
	+
	+ if (attrs->family == AF_INET)
	+ setup_fte_addr4(spec, &attrs->saddr.a4, &attrs->daddr.a4);
	+ else
	+ setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6);
	+ setup_fte_no_frags(spec);
	+
	+ rule = mlx5_add_flow_rules(tx->ft.sa, spec, flow_act, dest, num_dest);
	+ if (IS_ERR(rule)) {
	+ err = PTR_ERR(rule);
	+ mlx5_core_err(mdev, "fail to add TX ipsec ip rule err=%d\n", err);
	+ goto err_add_ip_rule;
	+ }
	+ ipsec_rule->rule = rule;
	+ kvfree(spec);
	+ return 0;
	+
	+err_add_ip_rule:
	+ mlx5_del_flow_rules(&ipsec_rule->reqid_rule);
	+err_add_reqid_rule:
	+ kvfree(spec);
	+ return err;
	+}
	+
	+static int tx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry)
	+{
	+ struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule;
	+ struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs;
	+ struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
	+ struct mlx5e_ipsec *ipsec = sa_entry->ipsec;
	+ struct mlx5_flow_destination dest[2] = {};
	+ struct mlx5_flow_act flow_act = {};
	+ struct mlx5e_ipsec_tx *tx;
	+ struct mlx5_fc *counter;
	+ int err;
	+
	+ tx = tx_ft_get(mdev, ipsec);
	+ if (IS_ERR(tx))
	+ return PTR_ERR(tx);
	+
	+ err = setup_pkt_reformat(mdev, attrs, &flow_act);
	+ if (err)
	+ goto err_pkt_reformat;
	+
	+ counter = mlx5_fc_create(mdev, false);
	+ if (IS_ERR(counter)) {
	+ err = PTR_ERR(counter);
	+ goto err_add_cnt;
	+ }
	+
	+ flow_act.crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_IPSEC;
	+ flow_act.crypto.obj_id = sa_entry->ipsec_obj_id;
	+ flow_act.flags \|= FLOW_ACT_NO_APPEND;
	+ flow_act.action \|= MLX5_FLOW_CONTEXT_ACTION_CRYPTO_ENCRYPT \|
	+ MLX5_FLOW_CONTEXT_ACTION_COUNT;
	+
	+ if (attrs->drop)
	+ flow_act.action \|= MLX5_FLOW_CONTEXT_ACTION_DROP;
	+ else
	+ flow_act.action \|= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
	+
	+ dest[0].ft = tx->ft.status;
	+ dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
	+ dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
	+ dest[1].counter_id = mlx5_fc_id(counter);
	+
	+ err = tx_add_kspi_rule(sa_entry, tx, &flow_act, dest, 2);
	+ if (err) {
	+ goto err_add_kspi_rule;
	+ }
	+
	+ err = tx_add_reqid_ip_rules(sa_entry, tx, &flow_act, dest, 2);
	+ if (err) {
	+ goto err_add_reqid_ip_rule;
	+ }
	+
	+ ipsec_rule->fc = counter;
	+ ipsec_rule->pkt_reformat = flow_act.pkt_reformat;
	+ return 0;
	+
	+err_add_reqid_ip_rule:
	+ mlx5_del_flow_rules(&ipsec_rule->kspi_rule);
	+err_add_kspi_rule:
	+ mlx5_fc_destroy(mdev, counter);
	+err_add_cnt:
	+ if (flow_act.pkt_reformat)
	+ mlx5_packet_reformat_dealloc(mdev, flow_act.pkt_reformat);
	+err_pkt_reformat:
	+ tx_ft_put(ipsec);
	+ return err;
	+}
	+
	+static int tx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry)
	+{
	+ struct mlx5_accel_pol_xfrm_attrs *attrs = &pol_entry->attrs;
	+ struct mlx5_core_dev *mdev = mlx5e_ipsec_pol2dev(pol_entry);
	+ struct mlx5e_ipsec_tx *tx = pol_entry->ipsec->tx;
	+ struct mlx5_flow_destination dest[2] = {};
	+ struct mlx5_flow_act flow_act = {};
	+ struct mlx5_flow_handle *rule;
	+ struct mlx5_flow_spec *spec;
	+ struct mlx5_flow_table *ft;
	+ int err, dstn = 0;
	+
	+ ft = tx_ft_get_policy(mdev, pol_entry->ipsec, attrs->prio);
	+ if (IS_ERR(ft))
	+ return PTR_ERR(ft);
	+
	+ spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
	+ if (!spec) {
	+ err = -ENOMEM;
	+ goto err_alloc;
	+ }
	+
	+ if (attrs->family == AF_INET)
	+ setup_fte_addr4(spec, &attrs->saddr.a4, &attrs->daddr.a4);
	+ else
	+ setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6);
	+
	+ setup_fte_no_frags(spec);
	+ setup_fte_upper_proto_match(spec, &attrs->upspec);
	+
	+ switch (attrs->action) {
	+ case IPSEC_POLICY_IPSEC:
	+ flow_act.action \|= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
	+ /*if (!attrs->reqid)
	+ break;*/
	+ err = setup_modify_header(mdev, attrs->reqid,
	+ IPSEC_DIR_OUTBOUND, &flow_act);
	+ if (err)
	+ goto err_mod_header;
	+ break;
	+ case IPSEC_POLICY_DISCARD:
	+ flow_act.action \|= MLX5_FLOW_CONTEXT_ACTION_DROP \|
	+ MLX5_FLOW_CONTEXT_ACTION_COUNT;
	+ dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
	+ dest[dstn].counter_id = mlx5_fc_id(tx->fc->drop);
	+ dstn++;
	+ break;
	+ default:
	+ err = -EINVAL;
	+ goto err_mod_header;
	+ }
	+
	+ flow_act.flags \|= FLOW_ACT_NO_APPEND;
	+ dest[dstn].ft = tx->ft.sa;
	+ dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
	+ dstn++;
	+ rule = mlx5_add_flow_rules(ft, spec, &flow_act, dest, dstn);
	+ if (IS_ERR(rule)) {
	+ err = PTR_ERR(rule);
	+ mlx5_core_err(mdev, "fail to add TX ipsec rule err=%d\n", err);
	+ goto err_action;
	+ }
	+
	+ kvfree(spec);
	+ pol_entry->ipsec_rule.rule = rule;
	+ pol_entry->ipsec_rule.modify_hdr = flow_act.modify_hdr;
	+ return 0;
	+
	+err_action:
	+ if (flow_act.modify_hdr)
	+ mlx5_modify_header_dealloc(mdev, flow_act.modify_hdr);
	+err_mod_header:
	+ kvfree(spec);
	+err_alloc:
	+ tx_ft_put_policy(pol_entry->ipsec, attrs->prio);
	+ return err;
	+}
	+
	+static int rx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry)
	+{
	+ struct mlx5_accel_pol_xfrm_attrs *attrs = &pol_entry->attrs;
	+ struct mlx5_core_dev *mdev = mlx5e_ipsec_pol2dev(pol_entry);
	+ struct mlx5e_ipsec *ipsec = pol_entry->ipsec;
	+ struct mlx5_flow_destination dest[2];
	+ struct mlx5_flow_act flow_act = {};
	+ struct mlx5_flow_handle *rule;
	+ struct mlx5_flow_spec *spec;
	+ struct mlx5_flow_table *ft;
	+ struct mlx5e_ipsec_rx *rx;
	+ int err, dstn = 0;
	+
	+ rx = (attrs->family == AF_INET) ? ipsec->rx_ipv4 : ipsec->rx_ipv6;
	+ ft = rx->chains ? ipsec_chains_get_table(rx->chains, attrs->prio) : rx->ft.pol;
	+ if (IS_ERR(ft))
	+ return PTR_ERR(ft);
	+
	+ spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
	+ if (!spec) {
	+ err = -ENOMEM;
	+ goto err_alloc;
	+ }
	+
	+ if (attrs->family == AF_INET)
	+ setup_fte_addr4(spec, &attrs->saddr.a4, &attrs->daddr.a4);
	+ else
	+ setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6);
	+
	+ setup_fte_no_frags(spec);
	+ setup_fte_upper_proto_match(spec, &attrs->upspec);
	+
	+ switch (attrs->action) {
	+ case IPSEC_POLICY_IPSEC:
	+ flow_act.action \|= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
	+ break;
	+ case IPSEC_POLICY_DISCARD:
	+ flow_act.action \|= MLX5_FLOW_CONTEXT_ACTION_DROP \| MLX5_FLOW_CONTEXT_ACTION_COUNT;
	+ dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
	+ dest[dstn].counter_id = mlx5_fc_id(rx->fc->drop);
	+ dstn++;
	+ break;
	+ default:
	+ err = -EINVAL;
	+ goto err_action;
	+ }
	+
	+ flow_act.flags \|= FLOW_ACT_NO_APPEND;
	+ dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
	+ dest[dstn].ft = rx->ft.sa;
	+ dstn++;
	+ rule = mlx5_add_flow_rules(ft, spec, &flow_act, dest, dstn);
	+ if (IS_ERR(rule)) {
	+ err = PTR_ERR(rule);
	+ mlx5_core_err(mdev, "Fail to add RX IPsec policy rule err=%d\n", err);
	+ goto err_action;
	+ }
	+
	+ kvfree(spec);
	+ pol_entry->ipsec_rule.rule = rule;
	+ return 0;
	+
	+err_action:
	+ kvfree(spec);
	+err_alloc:
	+ if (rx->chains)
	+ ipsec_chains_put_table(rx->chains, attrs->prio);
	+ return err;
	+}
	+
	+static void ipsec_fs_destroy_counters(struct mlx5e_ipsec *ipsec)
	+{
	+ struct mlx5e_ipsec_rx *rx_ipv4 = ipsec->rx_ipv4;
	+ struct mlx5_core_dev *mdev = ipsec->mdev;
	+ struct mlx5e_ipsec_tx *tx = ipsec->tx;
	+
	+ mlx5_fc_destroy(mdev, rx_ipv4->fc->drop);
	+ mlx5_fc_destroy(mdev, rx_ipv4->fc->cnt);
	+ kfree(rx_ipv4->fc);
	+ mlx5_fc_destroy(mdev, tx->fc->drop);
	+ mlx5_fc_destroy(mdev, tx->fc->cnt);
	+ kfree(tx->fc);
	+}
	+
	+static int ipsec_fs_init_counters(struct mlx5e_ipsec *ipsec)
	+{
	+ struct mlx5e_ipsec_rx *rx_ipv4 = ipsec->rx_ipv4;
	+ struct mlx5e_ipsec_rx *rx_ipv6 = ipsec->rx_ipv6;
	+ struct mlx5_core_dev *mdev = ipsec->mdev;
	+ struct mlx5e_ipsec_tx *tx = ipsec->tx;
	+ struct mlx5e_ipsec_fc *fc;
	+ struct mlx5_fc *counter;
	+ int err;
	+
	+ fc = kzalloc(sizeof(*tx->fc), GFP_KERNEL);
	+ if (!fc)
	+ return -ENOMEM;
	+
	+ tx->fc = fc;
	+ counter = mlx5_fc_create(mdev, false);
	+ if (IS_ERR(counter)) {
	+ err = PTR_ERR(counter);
	+ goto err_tx_fc_alloc;
	+ }
	+
	+ fc->cnt = counter;
	+ counter = mlx5_fc_create(mdev, false);
	+ if (IS_ERR(counter)) {
	+ err = PTR_ERR(counter);
	+ goto err_tx_fc_cnt;
	+ }
	+
	+ fc->drop = counter;
	+
	+ fc = kzalloc(sizeof(*tx->fc), GFP_KERNEL);
	+ if (!fc) {
	+ err = -ENOMEM;
	+ goto err_tx_fc_drop;
	+ }
	+
	+ /* Both IPv4 and IPv6 point to same flow counters struct. */
	+ rx_ipv4->fc = fc;
	+ rx_ipv6->fc = fc;
	+ counter = mlx5_fc_create(mdev, false);
	+ if (IS_ERR(counter)) {
	+ err = PTR_ERR(counter);
	+ goto err_rx_fc_alloc;
	+ }
	+
	+ fc->cnt = counter;
	+ counter = mlx5_fc_create(mdev, false);
	+ if (IS_ERR(counter)) {
	+ err = PTR_ERR(counter);
	+ goto err_rx_fc_cnt;
	+ }
	+
	+ fc->drop = counter;
	+ return 0;
	+
	+err_rx_fc_cnt:
	+ mlx5_fc_destroy(mdev, rx_ipv4->fc->cnt);
	+err_rx_fc_alloc:
	+ kfree(rx_ipv4->fc);
	+err_tx_fc_drop:
	+ mlx5_fc_destroy(mdev, tx->fc->drop);
	+err_tx_fc_cnt:
	+ mlx5_fc_destroy(mdev, tx->fc->cnt);
	+err_tx_fc_alloc:
	+ kfree(tx->fc);
	+ return err;
	+}
	+
	+static int ipsec_status_rule(struct mlx5_core_dev *mdev,
	+ struct mlx5e_ipsec_rx *rx,
	+ struct mlx5_flow_destination *dest)
	+{
	+ u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {};
	+ struct mlx5_flow_act flow_act = {};
	+ struct mlx5_modify_hdr *modify_hdr;
	+ struct mlx5_flow_handle *rule;
	+ struct mlx5_flow_spec *spec;
	+ int err;
	+
	+ spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
	+ if (!spec)
	+ return -ENOMEM;
	+
	+ /* Action to copy 7 bit ipsec_syndrome to regB[24:30] */
	+ MLX5_SET(copy_action_in, action, action_type, MLX5_ACTION_TYPE_COPY);
	+ MLX5_SET(copy_action_in, action, src_field, MLX5_ACTION_IN_FIELD_IPSEC_SYNDROME);
	+ MLX5_SET(copy_action_in, action, src_offset, 0);
	+ MLX5_SET(copy_action_in, action, length, 7);
	+ MLX5_SET(copy_action_in, action, dst_field, MLX5_ACTION_IN_FIELD_METADATA_REG_B);
	+ MLX5_SET(copy_action_in, action, dst_offset, 24);
	+
	+ modify_hdr = mlx5_modify_header_alloc(mdev, MLX5_FLOW_NAMESPACE_KERNEL,
	+ 1, action);
	+
	+ if (IS_ERR(modify_hdr)) {
	+ err = PTR_ERR(modify_hdr);
	+ mlx5_core_err(mdev,
	+ "fail to alloc ipsec copy modify_header_id err=%d\n", err);
	+ goto out_spec;
	+ }
	+
	+ /* create fte */
	+ flow_act.action = MLX5_FLOW_CONTEXT_ACTION_MOD_HDR \|
	+ MLX5_FLOW_CONTEXT_ACTION_FWD_DEST \|
	+ MLX5_FLOW_CONTEXT_ACTION_COUNT;
	+ flow_act.modify_hdr = modify_hdr;
	+
	+ rule = mlx5_add_flow_rules(rx->ft.status, spec, &flow_act, dest, 2);
	+ if (IS_ERR(rule)) {
	+ err = PTR_ERR(rule);
	+ mlx5_core_err(mdev, "fail to add ipsec rx err copy rule err=%d\n", err);
	+ goto out;
	+ }
	+
	+ kvfree(spec);
	+ rx->status.rule = rule;
	+ rx->status.modify_hdr = modify_hdr;
	+ return 0;
	+
	+out:
	+ mlx5_modify_header_dealloc(mdev, modify_hdr);
	+out_spec:
	+ kvfree(spec);
	+ return err;
	+}
	+
	+static void ipsec_fs_rx_roce_rules_destroy(struct mlx5e_ipsec_rx_roce *rx_roce)
	+{
	+ if (!rx_roce->ns_rdma)
	+ return;
	+
	+ mlx5_del_flow_rules(&rx_roce->roce_miss.rule);
	+ mlx5_del_flow_rules(&rx_roce->rule);
	+ mlx5_destroy_flow_group(rx_roce->roce_miss.group);
	+ mlx5_destroy_flow_group(rx_roce->g);
	+}
	+
	+static void ipsec_fs_rx_catchall_rules_destroy(struct mlx5_core_dev mdev, struct mlx5e_ipsec_rx rx)
	+{
	+ mutex_lock(&rx->ft.mutex);
	+ mlx5_del_flow_rules(&rx->sa.rule);
	+ mlx5_destroy_flow_group(rx->sa.group);
	+ if (rx->chains == NULL) {
	+ mlx5_del_flow_rules(&rx->pol.rule);
	+ mlx5_destroy_flow_group(rx->pol.group);
	+ }
	+ mlx5_del_flow_rules(&rx->status.rule);
	+ mlx5_modify_header_dealloc(mdev, rx->status.modify_hdr);
	+ ipsec_fs_rx_roce_rules_destroy(&rx->roce);
	+ mutex_unlock(&rx->ft.mutex);
	+}
	+
	+static void ipsec_fs_rx_roce_table_destroy(struct mlx5e_ipsec_rx_roce *rx_roce)
	+{
	+ if (!rx_roce->ns_rdma)
	+ return;
	+
	+ mlx5_destroy_flow_table(rx_roce->ft_rdma);
	+ mlx5_destroy_flow_table(rx_roce->ft);
	+}
	+
	+static void ipsec_fs_rx_table_destroy(struct mlx5_core_dev mdev, struct mlx5e_ipsec_rx rx)
	+{
	+ mutex_lock(&rx->ft.mutex);
	+ if (rx->chains) {
	+ ipsec_chains_destroy(rx->chains);
	+ } else {
	+ mlx5_del_flow_rules(&rx->pol.rule);
	+ mlx5_destroy_flow_table(rx->ft.pol);
	+ }
	+ mlx5_destroy_flow_table(rx->ft.sa);
	+ mlx5_destroy_flow_table(rx->ft.status);
	+ ipsec_fs_rx_roce_table_destroy(&rx->roce);
	+ mutex_unlock(&rx->ft.mutex);
	+}
	+
	+static void ipsec_roce_setup_udp_dport(struct mlx5_flow_spec *spec, u16 dport)
	+{
	+ spec->match_criteria_enable \|= MLX5_MATCH_OUTER_HEADERS;
	+ MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol);
	+ MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_UDP);
	+ MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.udp_dport);
	+ MLX5_SET(fte_match_param, spec->match_value, outer_headers.udp_dport, dport);
	+}
	+
	+static int ipsec_roce_rx_rule_setup(struct mlx5_flow_destination *default_dst,
	+ struct mlx5e_ipsec_rx_roce roce, struct mlx5_core_dev mdev)
	+{
	+ struct mlx5_flow_destination dst = {};
	+ struct mlx5_flow_act flow_act = {};
	+ struct mlx5_flow_handle *rule;
	+ struct mlx5_flow_spec *spec;
	+ int err = 0;
	+
	+ spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
	+ if (!spec)
	+ return -ENOMEM;
	+
	+ ipsec_roce_setup_udp_dport(spec, ROCE_V2_UDP_DPORT);
	+
	+ //flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;//not needed it is added in command
	+ dst.type = MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE;
	+ dst.ft = roce->ft_rdma;
	+
	+ flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
	+ rule = mlx5_add_flow_rules(roce->ft, spec, &flow_act, &dst, 1);
	+ if (IS_ERR(rule)) {
	+ err = PTR_ERR(rule);
	+ mlx5_core_err(mdev, "Fail to add RX roce ipsec rule err=%d\n",
	+ err);
	+ goto fail_add_rule;
	+ }
	+
	+ roce->rule = rule;
	+
	+ rule = mlx5_add_flow_rules(roce->ft, NULL, &flow_act, default_dst, 1);
	+ if (IS_ERR(rule)) {
	+ err = PTR_ERR(rule);
	+ mlx5_core_err(mdev, "Fail to add RX roce ipsec miss rule err=%d\n",
	+ err);
	+ goto fail_add_default_rule;
	+ }
	+
	+ roce->roce_miss.rule = rule;
	+
	+ kvfree(spec);
	+ return 0;
	+
	+fail_add_default_rule:
	+ mlx5_del_flow_rules(&roce->rule);
	+fail_add_rule:
	+ kvfree(spec);
	+ return err;
	+}
	+
	+static int ipsec_roce_rx_rules(struct mlx5e_ipsec_rx rx, struct mlx5_flow_destination defdst,
	+ struct mlx5_core_dev *mdev)
	+{
	+ int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
	+ struct mlx5_flow_group *g;
	+ void *outer_headers_c;
	+ u32 *in;
	+ int err = 0;
	+ int ix = 0;
	+ u8 *mc;
	+
	+ if (!rx->roce.ns_rdma)
	+ return 0;
	+
	+ in = kvzalloc(inlen, GFP_KERNEL);
	+ if (!in)
	+ return -ENOMEM;
	+
	+ mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
	+ outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, outer_headers);
	+ MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_protocol);
	+ MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, udp_dport);
	+
	+ MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
	+ MLX5_SET_CFG(in, start_flow_index, ix);
	+ ix += 1;
	+ MLX5_SET_CFG(in, end_flow_index, ix - 1);
	+ g = mlx5_create_flow_group(rx->roce.ft, in);
	+ if (IS_ERR(g)) {
	+ err = PTR_ERR(g);
	+ mlx5_core_err(mdev, "Fail to create ipsec rx roce group at nic err=%d\n", err);
	+ goto fail_group;
	+ }
	+ rx->roce.g = g;
	+
	+ memset(in, 0, MLX5_ST_SZ_BYTES(create_flow_group_in));
	+ MLX5_SET_CFG(in, start_flow_index, ix);
	+ ix += 1;
	+ MLX5_SET_CFG(in, end_flow_index, ix - 1);
	+ g = mlx5_create_flow_group(rx->roce.ft, in);
	+ if (IS_ERR(g)) {
	+ err = PTR_ERR(g);
	+ mlx5_core_err(mdev, "Fail to create ipsec rx roce miss group at nic err=%d\n",
	+ err);
	+ goto fail_mgroup;
	+ }
	+ rx->roce.roce_miss.group = g;
	+
	+ err = ipsec_roce_rx_rule_setup(defdst, &rx->roce, mdev);
	+ if (err)
	+ goto fail_setup_rule;
	+
	+ kvfree(in);
	+ return 0;
	+
	+fail_setup_rule:
	+ mlx5_destroy_flow_group(rx->roce.roce_miss.group);
	+fail_mgroup:
	+ mlx5_destroy_flow_group(rx->roce.g);
	+fail_group:
	+ kvfree(in);
	+ return err;
	+}
	+
	+static int ipsec_fs_rx_catchall_rules(struct mlx5e_priv *priv,
	+ struct mlx5e_ipsec_rx *rx,
	+ struct mlx5_flow_destination *defdst)
	+{
	+ struct mlx5_core_dev *mdev = priv->mdev;
	+ struct mlx5_flow_destination dest[2] = {};
	+ int err = 0;
	+
	+ mutex_lock(&rx->ft.mutex);
	+ /* IPsec RoCE RX rules */
	+ err = ipsec_roce_rx_rules(rx, defdst, mdev);
	+ if (err)
	+ goto out;
	+
	+ /* IPsec Rx IP Status table rule */
	+ dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
	+ if (rx->roce.ft)
	+ dest[0].ft = rx->roce.ft;
	+ else
	+ dest[0].ft = priv->fts.vlan.t;
	+
	+ dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
	+ dest[1].counter_id = mlx5_fc_id(rx->fc->cnt);
	+ err = ipsec_status_rule(mdev, rx, dest);
	+ if (err)
	+ goto err_roce_rules_destroy;
	+
	+ if (!rx->chains) {
	+ /* IPsec Rx IP policy default miss rule */
	+ err = ipsec_miss_create(mdev, rx->ft.pol, &rx->pol, defdst);
	+ if (err)
	+ goto err_status_rule_destroy;
	+ }
	+
	+ /* FIXME: This is workaround to current design
	+ * which installs SA on firt packet. So we need to forward this
	+ * packet to the stack. It doesn't work with RoCE and eswitch traffic,
	+ */
	+ err = ipsec_miss_create(mdev, rx->ft.sa, &rx->sa, defdst);
	+ if (err)
	+ goto err_status_sa_rule_destroy;
	+
	+ mutex_unlock(&rx->ft.mutex);
	+ return 0;
	+
	+err_status_sa_rule_destroy:
	+ if (!rx->chains) {
	+ mlx5_del_flow_rules(&rx->pol.rule);
	+ mlx5_destroy_flow_group(rx->pol.group);
	+ }
	+err_status_rule_destroy:
	+ mlx5_del_flow_rules(&rx->status.rule);
	+ mlx5_modify_header_dealloc(mdev, rx->status.modify_hdr);
	+err_roce_rules_destroy:
	+ ipsec_fs_rx_roce_rules_destroy(&rx->roce);
	+out:
	+ mutex_unlock(&rx->ft.mutex);
	+ return err;
	+}
	+
	+static int ipsec_fs_rx_roce_tables_create(struct mlx5e_ipsec_rx *rx,
	+ int rx_init_level, int rdma_init_level)
	+{
	+ struct mlx5_flow_table_attr ft_attr = {};
	+ struct mlx5_flow_table *ft;
	+ int err = 0;
	+
	+ if (!rx->roce.ns_rdma)
	+ return 0;
	+
	+ ft_attr.max_fte = 2;
	+ ft_attr.level = rx_init_level;
	+ ft = mlx5_create_flow_table(rx->ns, &ft_attr);
	+ if (IS_ERR(ft)) {
	+ err = PTR_ERR(ft);
	+ return err;
	+ }
	+ rx->roce.ft = ft;
	+
	+ ft_attr.max_fte = 0;
	+ ft_attr.level = rdma_init_level;
	+ ft = mlx5_create_flow_table(rx->roce.ns_rdma, &ft_attr);
	+ if (IS_ERR(ft)) {
	+ err = PTR_ERR(ft);
	+ goto out;
	+ }
	+ rx->roce.ft_rdma = ft;
	+
	+ return 0;
	+out:
	+ mlx5_destroy_flow_table(rx->roce.ft);
	+ rx->roce.ft = NULL;
	+ return err;
	+}
	+
	+static int ipsec_fs_rx_table_create(struct mlx5_core_dev mdev, struct mlx5e_ipsec_rx rx,
	+ int rx_init_level, int rdma_init_level)
	+{
	+ struct mlx5_flow_namespace *ns = rx->ns;
	+ struct mlx5_flow_table *ft;
	+ int err = 0;
	+
	+ mutex_lock(&rx->ft.mutex);
	+
	+ /* IPsec Rx IP SA table create */
	+ ft = ipsec_rx_ft_create(ns, rx_init_level + 1, 0, 1);
	+ if (IS_ERR(ft)) {
	+ err = PTR_ERR(ft);
	+ goto out;
	+ }
	+ rx->ft.sa = ft;
	+
	+ /* IPsec Rx IP Status table create */
	+ ft = ipsec_rx_ft_create(ns, rx_init_level + 2, 0, 1);
	+ if (IS_ERR(ft)) {
	+ err = PTR_ERR(ft);
	+ goto err_sa_table_destroy;
	+ }
	+ rx->ft.status = ft;
	+
	+ if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PRIO) {
	+ rx->chains = ipsec_chains_create(mdev, rx->ft.sa,
	+ MLX5_FLOW_NAMESPACE_KERNEL, 0,
	+ rx_init_level, &rx->ft.pol);
	+ if (IS_ERR(rx->chains)) {
	+ err = PTR_ERR(rx->chains);
	+ goto err_status_table_destroy;
	+ }
	+ } else {
	+ ft = ipsec_rx_ft_create(ns, rx_init_level, 0, 1);
	+ if (IS_ERR(ft)) {
	+ err = PTR_ERR(ft);
	+ goto err_status_table_destroy;
	+ }
	+ rx->ft.pol = ft;
	+ }
	+
	+ /* IPsec RoCE RX tables create*/
	+ err = ipsec_fs_rx_roce_tables_create(rx, rx_init_level + 3,
	+ rdma_init_level);
	+ if (err)
	+ goto err_pol_table_destroy;
	+
	+ goto out;
	+
	+err_pol_table_destroy:
	+ mlx5_destroy_flow_table(rx->ft.pol);
	+err_status_table_destroy:
	+ mlx5_destroy_flow_table(rx->ft.status);
	+err_sa_table_destroy:
	+ mlx5_destroy_flow_table(rx->ft.sa);
	+out:
	+ mutex_unlock(&rx->ft.mutex);
	+ return err;
	+}
	+
	+#define NIC_RDMA_BOTH_DIRS_CAPS (MLX5_FT_NIC_RX_2_NIC_RX_RDMA \| MLX5_FT_NIC_TX_RDMA_2_NIC_TX)
	+
	+static void mlx5e_accel_ipsec_fs_init_roce(struct mlx5e_ipsec *ipsec)
	+{
	+ struct mlx5_core_dev *mdev = ipsec->mdev;
	+ struct mlx5_flow_namespace *ns;
	+
	+ if ((MLX5_CAP_GEN_2(ipsec->mdev, flow_table_type_2_type) &
	+ NIC_RDMA_BOTH_DIRS_CAPS) != NIC_RDMA_BOTH_DIRS_CAPS) {
	+ mlx5_core_dbg(mdev, "Failed to init roce ns, capabilities not supported\n");
	+ return;
	+ }
	+
	+ ns = mlx5_get_flow_namespace(ipsec->mdev, MLX5_FLOW_NAMESPACE_RDMA_RX_IPSEC);
	+ if (!ns) {
	+ mlx5_core_err(mdev, "Failed to init roce rx ns\n");
	+ return;
	+ }
	+
	+ ipsec->rx_ipv4->roce.ns_rdma = ns;
	+ ipsec->rx_ipv6->roce.ns_rdma = ns;
	+
	+ ns = mlx5_get_flow_namespace(ipsec->mdev, MLX5_FLOW_NAMESPACE_RDMA_TX_IPSEC);
	+ if (!ns) {
	+ ipsec->rx_ipv4->roce.ns_rdma = NULL;
	+ ipsec->rx_ipv6->roce.ns_rdma = NULL;
	+ mlx5_core_err(mdev, "Failed to init roce tx ns\n");
	+ return;
	+ }
	+
	+ ipsec->tx->roce.ns = ns;
	+}
	+
	+int mlx5e_accel_ipsec_fs_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry)
	+{
	+ if (sa_entry->attrs.dir == IPSEC_DIR_OUTBOUND)
	+ return tx_add_rule(sa_entry);
	+
	+ return rx_add_rule(sa_entry);
	+}
	+
	+void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_ipsec_sa_entry *sa_entry)
	+{
	+ struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule;
	+ struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
	+
	+ mlx5_del_flow_rules(&ipsec_rule->rule);
	+ mlx5_del_flow_rules(&ipsec_rule->kspi_rule);
	+ if (ipsec_rule->reqid_rule)
	+ mlx5_del_flow_rules(&ipsec_rule->reqid_rule);
	+ mlx5_fc_destroy(mdev, ipsec_rule->fc);
	+ mlx5_packet_reformat_dealloc(mdev, ipsec_rule->pkt_reformat);
	+ if (sa_entry->attrs.dir == IPSEC_DIR_OUTBOUND) {
	+ tx_ft_put(sa_entry->ipsec);
	+ return;
	+ }
	+
	+ if (ipsec_rule->modify_hdr)
	+ mlx5_modify_header_dealloc(mdev, ipsec_rule->modify_hdr);
	+}
	+
	+int mlx5e_accel_ipsec_fs_add_pol(struct mlx5e_ipsec_pol_entry *pol_entry)
	+{
	+ if (pol_entry->attrs.dir == IPSEC_DIR_OUTBOUND)
	+ return tx_add_policy(pol_entry);
	+
	+ return rx_add_policy(pol_entry);
	+}
	+
	+void mlx5e_accel_ipsec_fs_del_pol(struct mlx5e_ipsec_pol_entry *pol_entry)
	+{
	+ struct mlx5e_ipsec_rule *ipsec_rule = &pol_entry->ipsec_rule;
	+ struct mlx5_core_dev *mdev = mlx5e_ipsec_pol2dev(pol_entry);
	+
	+ mlx5_del_flow_rules(&ipsec_rule->rule);
	+
	+ if (pol_entry->attrs.dir == IPSEC_DIR_INBOUND) {
	+ struct mlx5e_ipsec_rx *rx;
	+
	+ rx = (pol_entry->attrs.family == AF_INET)
	+ ? pol_entry->ipsec->rx_ipv4
	+ : pol_entry->ipsec->rx_ipv6;
	+ if (rx->chains)
	+ ipsec_chains_put_table(rx->chains,
	+ pol_entry->attrs.prio);
	+ return;
	+ }
	+
	+ if (ipsec_rule->modify_hdr)
	+ mlx5_modify_header_dealloc(mdev, ipsec_rule->modify_hdr);
	+
	+ tx_ft_put_policy(pol_entry->ipsec, pol_entry->attrs.prio);
	+}
	+
	+void mlx5e_accel_ipsec_fs_rx_catchall_rules_destroy(struct mlx5e_priv *priv)
	+{
	+ /* Check if IPsec supported */
	+ if (!priv->ipsec)
	+ return;
	+
	+ ipsec_fs_rx_catchall_rules_destroy(priv->mdev, priv->ipsec->rx_ipv4);
	+ ipsec_fs_rx_catchall_rules_destroy(priv->mdev, priv->ipsec->rx_ipv6);
	+}
	+
	+int mlx5e_accel_ipsec_fs_rx_catchall_rules(struct mlx5e_priv *priv)
	+{
	+ struct mlx5e_ipsec *ipsec = priv->ipsec;
	+ struct mlx5_flow_destination dest = {};
	+ int err = 0;
	+
	+ /* Check if IPsec supported */
	+ if (!ipsec)
	+ return 0;
	+
	+ dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
	+ dest.ft = priv->fts.vlan.t;
	+ err = ipsec_fs_rx_catchall_rules(priv, ipsec->rx_ipv6, &dest);
	+ if (err)
	+ goto out;
	+
	+ err = ipsec_fs_rx_catchall_rules(priv, ipsec->rx_ipv4, &dest);
	+ if (err)
	+ ipsec_fs_rx_catchall_rules_destroy(priv->mdev, priv->ipsec->rx_ipv6);
	+out:
	+ return err;
	+}
	+
	+void mlx5e_accel_ipsec_fs_rx_tables_destroy(struct mlx5e_priv *priv)
	+{
	+ struct mlx5_core_dev *mdev = priv->mdev;
	+ struct mlx5e_ipsec *ipsec = priv->ipsec;
	+
	+ /* Check if IPsec supported */
	+ if (!ipsec)
	+ return;
	+
	+ ipsec_fs_rx_table_destroy(mdev, ipsec->rx_ipv6);
	+ ipsec_fs_rx_table_destroy(mdev, ipsec->rx_ipv4);
	+}
	+
	+int mlx5e_accel_ipsec_fs_rx_tables_create(struct mlx5e_priv *priv)
	+{
	+ struct mlx5e_ipsec *ipsec = priv->ipsec;
	+ int err = 0;
	+
	+ /* Check if IPsec supported */
	+ if (!ipsec)
	+ return 0;
	+
	+ err = ipsec_fs_rx_table_create(ipsec->mdev, ipsec->rx_ipv4, 0, 0);
	+ if (err)
	+ goto out;
	+
	+ err = ipsec_fs_rx_table_create(ipsec->mdev, ipsec->rx_ipv6, 4, 1);
	+ if (err) {
	+ ipsec_fs_rx_table_destroy(priv->mdev, ipsec->rx_ipv4);
	+ goto out;
	+ }
	+
	+ priv->fts.ipsec_ft = priv->ipsec->rx_ipv4->ft.pol;
	+out:
	+ return err;
	+}
	+
	+void mlx5e_accel_ipsec_fs_cleanup(struct mlx5e_ipsec *ipsec)
	+{
	+ WARN_ON(ipsec->tx->ft.refcnt);
	+ mutex_destroy(&ipsec->rx_ipv6->ft.mutex);
	+ mutex_destroy(&ipsec->rx_ipv4->ft.mutex);
	+ mutex_destroy(&ipsec->tx->ft.mutex);
	+ ipsec_fs_destroy_counters(ipsec);
	+ kfree(ipsec->rx_ipv6);
	+ kfree(ipsec->rx_ipv4);
	+ kfree(ipsec->tx);
	+}
	+
	+int mlx5e_accel_ipsec_fs_init(struct mlx5e_ipsec *ipsec)
	+{
	+ struct mlx5_flow_namespace tns, rns;
	+ int err = -ENOMEM;
	+
	+ tns = mlx5_get_flow_namespace(ipsec->mdev, MLX5_FLOW_NAMESPACE_EGRESS_IPSEC);
	+ if (!tns)
	+ return -EOPNOTSUPP;
	+
	+ rns = mlx5_get_flow_namespace(ipsec->mdev, MLX5_FLOW_NAMESPACE_KERNEL);
	+ if (!rns)
	+ return -EOPNOTSUPP;
	+
	+ ipsec->tx = kzalloc(sizeof(*ipsec->tx), GFP_KERNEL);
	+ if (!ipsec->tx)
	+ return -ENOMEM;
	+
	+ ipsec->rx_ipv4 = kzalloc(sizeof(*ipsec->rx_ipv4), GFP_KERNEL);
	+ if (!ipsec->rx_ipv4)
	+ goto err_tx;
	+
	+ ipsec->rx_ipv6 = kzalloc(sizeof(*ipsec->rx_ipv6), GFP_KERNEL);
	+ if (!ipsec->rx_ipv6)
	+ goto err_rx_ipv4;
	+
	+ err = ipsec_fs_init_counters(ipsec);
	+ if (err)
	+ goto err_rx_ipv6;
	+
	+ ipsec->tx->ns = tns;
	+ mutex_init(&ipsec->tx->ft.mutex);
	+ ipsec->rx_ipv4->ns = rns;
	+ ipsec->rx_ipv6->ns = rns;
	+ mutex_init(&ipsec->rx_ipv4->ft.mutex);
	+ mutex_init(&ipsec->rx_ipv6->ft.mutex);
	+
	+ mlx5e_accel_ipsec_fs_init_roce(ipsec);
	+
	+ return 0;
	+
	+err_rx_ipv6:
	+ kfree(ipsec->rx_ipv6);
	+err_rx_ipv4:
	+ kfree(ipsec->rx_ipv4);
	+err_tx:
	+ kfree(ipsec->tx);
	+ return err;
	+}
	+
	+void mlx5e_accel_ipsec_fs_modify(struct mlx5e_ipsec_sa_entry *sa_entry)
	+{
	+ struct mlx5e_ipsec_sa_entry sa_entry_shadow = {};
	+ int err;
	+
	+ memcpy(&sa_entry_shadow, sa_entry, sizeof(*sa_entry));
	+ memset(&sa_entry_shadow.ipsec_rule, 0x00, sizeof(sa_entry->ipsec_rule));
	+
	+ err = mlx5e_accel_ipsec_fs_add_rule(&sa_entry_shadow);
	+ if (err)
	+ return;
	+ mlx5e_accel_ipsec_fs_del_rule(sa_entry);
	+ memcpy(sa_entry, &sa_entry_shadow, sizeof(*sa_entry));
	+}
	diff --git a/sys/dev/mlx5/mlx5_accel/mlx5_ipsec_offload.c b/sys/dev/mlx5/mlx5_accel/mlx5_ipsec_offload.c
	new file mode 100644
	--- /dev/null
	+++ b/sys/dev/mlx5/mlx5_accel/mlx5_ipsec_offload.c
	@@ -0,0 +1,480 @@
	+/*-
	+ * Copyright (c) 2023 NVIDIA corporation & affiliates.
	+ *
	+ * Redistribution and use in source and binary forms, with or without
	+ * modification, are permitted provided that the following conditions
	+ * are met:
	+ * 1. Redistributions of source code must retain the above copyright
	+ * notice, this list of conditions and the following disclaimer.
	+ * 2. Redistributions in binary form must reproduce the above copyright
	+ * notice, this list of conditions and the following disclaimer in the
	+ * documentation and/or other materials provided with the distribution.
	+ *
	+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
	+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
	+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	+ * SUCH DAMAGE.
	+ *
	+ * $FreeBSD$
	+ */
	+
	+#include <sys/types.h>
	+#include <netinet/in.h>
	+#include <sys/socket.h>
	+#include <sys/param.h>
	+#include <sys/systm.h>
	+#include <net/pfkeyv2.h>
	+#include <netipsec/ipsec.h>
	+#include <dev/mlx5/mlx5_en/en.h>
	+#include <dev/mlx5/crypto.h>
	+#include <dev/mlx5/mlx5_accel/ipsec.h>
	+
	+u32 mlx5_ipsec_device_caps(struct mlx5_core_dev *mdev)
	+{
	+ u32 caps = 0;
	+
	+ if (!MLX5_CAP_GEN(mdev, ipsec_offload))
	+ return 0;
	+
	+ if (!MLX5_CAP_GEN(mdev, log_max_dek))
	+ return 0;
	+
	+ if (!(MLX5_CAP_GEN_64(mdev, general_obj_types) &
	+ MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_IPSEC))
	+ return 0;
	+
	+ if (!MLX5_CAP_FLOWTABLE_NIC_TX(mdev, ipsec_encrypt) \|\|
	+ !MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ipsec_decrypt))
	+ return 0;
	+
	+ if (!MLX5_CAP_IPSEC(mdev, ipsec_crypto_esp_aes_gcm_128_encrypt) \|\|
	+ !MLX5_CAP_IPSEC(mdev, ipsec_crypto_esp_aes_gcm_128_decrypt))
	+ return 0;
	+
	+ if (MLX5_CAP_IPSEC(mdev, ipsec_full_offload)) {
	+ if (MLX5_CAP_FLOWTABLE_NIC_TX(mdev,
	+ reformat_add_esp_trasport) &&
	+ MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
	+ reformat_del_esp_trasport) &&
	+ MLX5_CAP_FLOWTABLE_NIC_RX(mdev, decap))
	+ caps \|= MLX5_IPSEC_CAP_PACKET_OFFLOAD;
	+
	+ if (MLX5_CAP_FLOWTABLE_NIC_TX(mdev, ignore_flow_level) &&
	+ MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ignore_flow_level))
	+ caps \|= MLX5_IPSEC_CAP_PRIO;
	+
	+ if (MLX5_CAP_FLOWTABLE_NIC_TX(mdev, reformat_add_esp_transport_over_udp) &&
	+ MLX5_CAP_FLOWTABLE_NIC_RX(mdev, reformat_del_esp_transport_over_udp))
	+ caps \|= MLX5_IPSEC_CAP_ESPINUDP;
	+ }
	+
	+ if (!caps)
	+ return 0;
	+
	+ if (MLX5_CAP_IPSEC(mdev, ipsec_esn))
	+ caps \|= MLX5_IPSEC_CAP_ESN;
	+
	+ return caps;
	+}
	+EXPORT_SYMBOL_GPL(mlx5_ipsec_device_caps);
	+
	+static void mlx5e_ipsec_packet_setup(void *obj, u32 pdn,
	+ struct mlx5_accel_esp_xfrm_attrs *attrs)
	+{
	+ void *aso_ctx;
	+
	+ aso_ctx = MLX5_ADDR_OF(ipsec_obj, obj, ipsec_aso);
	+ /* ASO context */
	+ MLX5_SET(ipsec_obj, obj, ipsec_aso_access_pd, pdn);
	+ MLX5_SET(ipsec_obj, obj, full_offload, 1);
	+ MLX5_SET(ipsec_aso, aso_ctx, valid, 1);
	+ /* MLX5_IPSEC_ASO_REG_C_4_5 is type C register that is used
	+ * in flow steering to perform matching against. Please be
	+ * aware that this register was chosen arbitrary and can't
	+ * be used in other places as long as IPsec packet offload
	+ * active.
	+ */
	+ MLX5_SET(ipsec_obj, obj, aso_return_reg, MLX5_IPSEC_ASO_REG_C_4_5);
	+ if (attrs->replay_esn.trigger) {
	+ MLX5_SET(ipsec_aso, aso_ctx, esn_event_arm, 1);
	+
	+ if (attrs->dir == IPSEC_DIR_INBOUND) {
	+ MLX5_SET(ipsec_aso, aso_ctx, window_sz,
	+ attrs->replay_esn.replay_window);
	+ MLX5_SET(ipsec_aso, aso_ctx, mode,
	+ MLX5_IPSEC_ASO_REPLAY_PROTECTION);
	+ }
	+ MLX5_SET(ipsec_aso, aso_ctx, mode_parameter,
	+ attrs->replay_esn.esn);
	+ }
	+
	+ switch (attrs->dir) {
	+ case IPSEC_DIR_OUTBOUND:
	+ MLX5_SET(ipsec_aso, aso_ctx, mode, MLX5_IPSEC_ASO_INC_SN);
	+ break;
	+ default:
	+ break;
	+ }
	+}
	+
	+static int mlx5_create_ipsec_obj(struct mlx5e_ipsec_sa_entry *sa_entry)
	+{
	+ struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs;
	+ struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
	+ struct aes_gcm_keymat *aes_gcm = &attrs->aes_gcm;
	+ u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
	+ u32 in[MLX5_ST_SZ_DW(create_ipsec_obj_in)] = {};
	+ void obj, salt_p, *salt_iv_p;
	+ int err;
	+
	+ obj = MLX5_ADDR_OF(create_ipsec_obj_in, in, ipsec_object);
	+
	+ /* salt and seq_iv */
	+ salt_p = MLX5_ADDR_OF(ipsec_obj, obj, salt);
	+ memcpy(salt_p, &aes_gcm->salt, sizeof(aes_gcm->salt));
	+
	+ MLX5_SET(ipsec_obj, obj, icv_length, MLX5_IPSEC_OBJECT_ICV_LEN_16B);
	+ salt_iv_p = MLX5_ADDR_OF(ipsec_obj, obj, implicit_iv);
	+ memcpy(salt_iv_p, &aes_gcm->seq_iv, sizeof(aes_gcm->seq_iv));
	+
	+ /* esn */
	+ if (attrs->replay_esn.trigger) {
	+ MLX5_SET(ipsec_obj, obj, esn_en, 1);
	+ MLX5_SET(ipsec_obj, obj, esn_msb, attrs->replay_esn.esn_msb);
	+ MLX5_SET(ipsec_obj, obj, esn_overlap, attrs->replay_esn.overlap);
	+ }
	+
	+ /* enc./dec. key */
	+ MLX5_SET(ipsec_obj, obj, dekn, sa_entry->enc_key_id);
	+
	+ /* general object fields set */
	+ MLX5_SET(general_obj_in_cmd_hdr, in, opcode,
	+ MLX5_CMD_OP_CREATE_GENERAL_OBJ);
	+ MLX5_SET(general_obj_in_cmd_hdr, in, obj_type,
	+ MLX5_GENERAL_OBJECT_TYPES_IPSEC);
	+
	+ mlx5e_ipsec_packet_setup(obj, sa_entry->ipsec->pdn, attrs);
	+
	+ err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
	+ if (!err)
	+ sa_entry->ipsec_obj_id =
	+ MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
	+
	+ return err;
	+}
	+
	+static void mlx5_destroy_ipsec_obj(struct mlx5e_ipsec_sa_entry *sa_entry)
	+{
	+ struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
	+ u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {};
	+ u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
	+
	+ MLX5_SET(general_obj_in_cmd_hdr, in, opcode,
	+ MLX5_CMD_OP_DESTROY_GENERAL_OBJ);
	+ MLX5_SET(general_obj_in_cmd_hdr, in, obj_type,
	+ MLX5_GENERAL_OBJECT_TYPES_IPSEC);
	+ MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, sa_entry->ipsec_obj_id);
	+
	+ mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
	+}
	+
	+int mlx5_ipsec_create_sa_ctx(struct mlx5e_ipsec_sa_entry *sa_entry)
	+{
	+ struct aes_gcm_keymat *aes_gcm = &sa_entry->attrs.aes_gcm;
	+ struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
	+ int err;
	+
	+ /* key */
	+ err = mlx5_encryption_key_create(mdev, sa_entry->ipsec->pdn,
	+ MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_IPSEC,
	+ aes_gcm->aes_key,
	+ aes_gcm->key_len,
	+ &sa_entry->enc_key_id);
	+ if (err) {
	+ mlx5_core_dbg(mdev, "Failed to create encryption key (err = %d)\n", err);
	+ return err;
	+ }
	+
	+ err = mlx5_create_ipsec_obj(sa_entry);
	+ if (err) {
	+ mlx5_core_dbg(mdev, "Failed to create IPsec object (err = %d)\n", err);
	+ goto err_enc_key;
	+ }
	+
	+ return 0;
	+
	+err_enc_key:
	+ mlx5_encryption_key_destroy(mdev, sa_entry->enc_key_id);
	+ return err;
	+}
	+
	+void mlx5_ipsec_free_sa_ctx(struct mlx5e_ipsec_sa_entry *sa_entry)
	+{
	+ struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
	+
	+ mlx5_destroy_ipsec_obj(sa_entry);
	+ mlx5_encryption_key_destroy(mdev, sa_entry->enc_key_id);
	+}
	+
	+static void mlx5e_ipsec_aso_copy(struct mlx5_wqe_aso_ctrl_seg *ctrl,
	+ struct mlx5_wqe_aso_ctrl_seg *data)
	+{
	+ if (!data)
	+ return;
	+
	+ ctrl->data_mask_mode = data->data_mask_mode;
	+ ctrl->condition_1_0_operand = data->condition_1_0_operand;
	+ ctrl->condition_1_0_offset = data->condition_1_0_offset;
	+ ctrl->data_offset_condition_operand = data->data_offset_condition_operand;
	+ ctrl->condition_0_data = data->condition_0_data;
	+ ctrl->condition_0_mask = data->condition_0_mask;
	+ ctrl->condition_1_data = data->condition_1_data;
	+ ctrl->condition_1_mask = data->condition_1_mask;
	+ ctrl->bitwise_data = data->bitwise_data;
	+ ctrl->data_mask = data->data_mask;
	+}
	+
	+static int mlx5e_ipsec_aso_query(struct mlx5e_ipsec_sa_entry *sa_entry,
	+ struct mlx5_wqe_aso_ctrl_seg *data)
	+{
	+ struct mlx5e_ipsec *ipsec = sa_entry->ipsec;
	+ struct mlx5e_ipsec_aso *aso = ipsec->aso;
	+ struct mlx5_wqe_aso_ctrl_seg *ctrl;
	+ struct mlx5_aso_wqe *wqe;
	+ unsigned long expires;
	+ u8 ds_cnt;
	+ int ret;
	+
	+ spin_lock_bh(&aso->lock);
	+ memset(aso->ctx, 0, sizeof(aso->ctx));
	+ wqe = mlx5_aso_get_wqe(aso->aso);
	+ ds_cnt = DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_DS);
	+ mlx5_aso_build_wqe(aso->aso, ds_cnt, wqe, sa_entry->ipsec_obj_id,
	+ MLX5_ACCESS_ASO_OPC_MOD_IPSEC);
	+
	+ ctrl = &wqe->aso_ctrl;
	+ ctrl->va_l = cpu_to_be32(lower_32_bits(aso->dma_addr) \| ASO_CTRL_READ_EN);
	+ ctrl->va_h = cpu_to_be32(upper_32_bits(aso->dma_addr));
	+ ctrl->l_key = cpu_to_be32(ipsec->mkey);
	+ mlx5e_ipsec_aso_copy(ctrl, data);
	+
	+ mlx5_aso_post_wqe(aso->aso, false, &wqe->ctrl);
	+ expires = jiffies + msecs_to_jiffies(10);
	+ do {
	+ ret = mlx5_aso_poll_cq(aso->aso, false);
	+ if (ret)
	+ /* We are in atomic context */
	+ udelay(10);
	+ } while (ret && time_is_after_jiffies(expires));
	+ spin_unlock_bh(&aso->lock);
	+
	+ return ret;
	+}
	+
	+#define MLX5E_IPSEC_ESN_SCOPE_MID 0x80000000L
	+
	+static int mlx5_modify_ipsec_obj(struct mlx5e_ipsec_sa_entry *sa_entry,
	+ const struct mlx5_accel_esp_xfrm_attrs *attrs)
	+{
	+ struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
	+ u32 in[MLX5_ST_SZ_DW(modify_ipsec_obj_in)] = {};
	+ u32 out[MLX5_ST_SZ_DW(query_ipsec_obj_out)];
	+ u64 modify_field_select = 0;
	+ u64 general_obj_types;
	+ void *obj;
	+ int err;
	+
	+ general_obj_types = MLX5_CAP_GEN_64(mdev, general_obj_types);
	+ if (!(general_obj_types & MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_IPSEC))
	+ return -EINVAL;
	+
	+ /* general object fields set */
	+ MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_QUERY_GENERAL_OBJ);
	+ MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_IPSEC);
	+ MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, sa_entry->ipsec_obj_id);
	+ err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
	+ if (err) {
	+ mlx5_core_err(mdev, "Query IPsec object failed (Object id %d), err = %d\n",
	+ sa_entry->ipsec_obj_id, err);
	+ return err;
	+ }
	+
	+ obj = MLX5_ADDR_OF(query_ipsec_obj_out, out, ipsec_object);
	+ modify_field_select = MLX5_GET64(ipsec_obj, obj, modify_field_select);
	+
	+ /* esn */
	+ if (!(modify_field_select & MLX5_MODIFY_IPSEC_BITMASK_ESN_OVERLAP) \|\|
	+ !(modify_field_select & MLX5_MODIFY_IPSEC_BITMASK_ESN_MSB))
	+ return -EOPNOTSUPP;
	+
	+ obj = MLX5_ADDR_OF(modify_ipsec_obj_in, in, ipsec_object);
	+ MLX5_SET64(ipsec_obj, obj, modify_field_select,
	+ MLX5_MODIFY_IPSEC_BITMASK_ESN_OVERLAP \|
	+ MLX5_MODIFY_IPSEC_BITMASK_ESN_MSB);
	+ MLX5_SET(ipsec_obj, obj, esn_msb, attrs->replay_esn.esn_msb);
	+ MLX5_SET(ipsec_obj, obj, esn_overlap, attrs->replay_esn.overlap);
	+
	+ /* general object fields set */
	+ MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_MODIFY_GENERAL_OBJ);
	+
	+ return mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
	+}
	+
	+static void mlx5_accel_esp_modify_xfrm(struct mlx5e_ipsec_sa_entry *sa_entry,
	+ const struct mlx5_accel_esp_xfrm_attrs *attrs)
	+{
	+ int err;
	+
	+ err = mlx5_modify_ipsec_obj(sa_entry, attrs);
	+ if (err)
	+ return;
	+
	+ memcpy(&sa_entry->attrs, attrs, sizeof(sa_entry->attrs));
	+}
	+
	+static void mlx5e_ipsec_aso_update(struct mlx5e_ipsec_sa_entry *sa_entry,
	+ struct mlx5_wqe_aso_ctrl_seg *data)
	+{
	+ data->data_mask_mode = MLX5_ASO_DATA_MASK_MODE_BITWISE_64BIT << 6;
	+ data->condition_1_0_operand = MLX5_ASO_ALWAYS_TRUE \| MLX5_ASO_ALWAYS_TRUE << 4;
	+
	+ mlx5e_ipsec_aso_query(sa_entry, data);
	+}
	+
	+#define MLX5_IPSEC_ASO_REMOVE_FLOW_PKT_CNT_OFFSET 0
	+
	+static void mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry,
	+ u32 mode_param)
	+{
	+ struct mlx5_accel_esp_xfrm_attrs attrs = {};
	+ struct mlx5_wqe_aso_ctrl_seg data = {};
	+
	+ if (mode_param < MLX5E_IPSEC_ESN_SCOPE_MID) {
	+ sa_entry->esn_state.esn_msb++;
	+ sa_entry->esn_state.overlap = 0;
	+ } else {
	+ sa_entry->esn_state.overlap = 1;
	+ }
	+
	+ mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &attrs, sa_entry->attrs.dir);
	+
	+ mlx5_accel_esp_modify_xfrm(sa_entry, &attrs);
	+
	+ data.data_offset_condition_operand = MLX5_IPSEC_ASO_REMOVE_FLOW_PKT_CNT_OFFSET;
	+ data.bitwise_data = cpu_to_be64(BIT_ULL(54));
	+ data.data_mask = data.bitwise_data;
	+
	+ mlx5e_ipsec_aso_update(sa_entry, &data);
	+}
	+
	+static void mlx5e_ipsec_handle_event(struct work_struct *_work)
	+{
	+ struct mlx5e_ipsec_work *work =
	+ container_of(_work, struct mlx5e_ipsec_work, work);
	+ struct mlx5e_ipsec_sa_entry *sa_entry = work->data;
	+ struct mlx5_accel_esp_xfrm_attrs *attrs;
	+ struct mlx5e_ipsec_aso *aso;
	+ int ret;
	+
	+ aso = sa_entry->ipsec->aso;
	+ attrs = &sa_entry->attrs;
	+
	+ /* TODO: Kostia, this event should be locked/protected
	+ * from concurent SA delete.
	+ */
	+ ret = mlx5e_ipsec_aso_query(sa_entry, NULL);
	+ if (ret)
	+ goto unlock;
	+
	+ if (attrs->replay_esn.trigger &&
	+ !MLX5_GET(ipsec_aso, aso->ctx, esn_event_arm)) {
	+ u32 mode_param = MLX5_GET(ipsec_aso, aso->ctx, mode_parameter);
	+
	+ mlx5e_ipsec_update_esn_state(sa_entry, mode_param);
	+ }
	+
	+unlock:
	+ kfree(work);
	+}
	+
	+void mlx5_object_change_event(struct mlx5_core_dev dev, struct mlx5_eqe eqe)
	+{
	+ struct mlx5e_ipsec_sa_entry *sa_entry;
	+ struct mlx5_eqe_obj_change *object;
	+ struct mlx5e_ipsec_work *work;
	+ u16 type;
	+
	+ object = &eqe->data.obj_change;
	+ type = be16_to_cpu(object->obj_type);
	+
	+ if (type != MLX5_GENERAL_OBJECT_TYPES_IPSEC)
	+ return;
	+
	+ sa_entry = xa_load(&dev->ipsec_sadb, be32_to_cpu(object->obj_id));
	+ if (!sa_entry)
	+ return;
	+
	+ work = kmalloc(sizeof(*work), GFP_ATOMIC);
	+ if (!work)
	+ return;
	+
	+ INIT_WORK(&work->work, mlx5e_ipsec_handle_event);
	+ work->data = sa_entry;
	+
	+ queue_work(sa_entry->ipsec->wq, &work->work);
	+}
	+
	+int mlx5e_ipsec_aso_init(struct mlx5e_ipsec *ipsec)
	+{
	+ struct mlx5_core_dev *mdev = ipsec->mdev;
	+ struct mlx5e_ipsec_aso *aso;
	+ struct device *pdev;
	+ int err;
	+
	+ aso = kzalloc(sizeof(*ipsec->aso), GFP_KERNEL);
	+ if (!aso)
	+ return -ENOMEM;
	+
	+ pdev = &mdev->pdev->dev;
	+ aso->dma_addr = dma_map_single(pdev, aso->ctx, sizeof(aso->ctx), DMA_BIDIRECTIONAL);
	+ err = dma_mapping_error(pdev, aso->dma_addr);
	+ if (err)
	+ goto err_dma;
	+
	+ aso->aso = mlx5_aso_create(mdev, ipsec->pdn);
	+ if (IS_ERR(aso->aso)) {
	+ err = PTR_ERR(aso->aso);
	+ goto err_aso_create;
	+ }
	+
	+ spin_lock_init(&aso->lock);
	+ ipsec->aso = aso;
	+ return 0;
	+
	+err_aso_create:
	+ dma_unmap_single(pdev, aso->dma_addr, sizeof(aso->ctx), DMA_BIDIRECTIONAL);
	+err_dma:
	+ kfree(aso);
	+ return err;
	+}
	+
	+void mlx5e_ipsec_aso_cleanup(struct mlx5e_ipsec *ipsec)
	+{
	+ struct mlx5_core_dev *mdev = ipsec->mdev;
	+ struct mlx5e_ipsec_aso *aso;
	+ struct device *pdev;
	+
	+ aso = ipsec->aso;
	+ pdev = &mdev->pdev->dev;
	+
	+ mlx5_aso_destroy(aso->aso);
	+ dma_unmap_single(pdev, aso->dma_addr, sizeof(aso->ctx), DMA_BIDIRECTIONAL);
	+ kfree(aso);
	+}
	diff --git a/sys/dev/mlx5/mlx5_accel/mlx5_ipsec_rxtx.c b/sys/dev/mlx5/mlx5_accel/mlx5_ipsec_rxtx.c
	new file mode 100644
	--- /dev/null
	+++ b/sys/dev/mlx5/mlx5_accel/mlx5_ipsec_rxtx.c
	@@ -0,0 +1,77 @@
	+/*-
	+ * Copyright (c) 2023 NVIDIA corporation & affiliates.
	+ *
	+ * Redistribution and use in source and binary forms, with or without
	+ * modification, are permitted provided that the following conditions
	+ * are met:
	+ * 1. Redistributions of source code must retain the above copyright
	+ * notice, this list of conditions and the following disclaimer.
	+ * 2. Redistributions in binary form must reproduce the above copyright
	+ * notice, this list of conditions and the following disclaimer in the
	+ * documentation and/or other materials provided with the distribution.
	+ *
	+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
	+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
	+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	+ * SUCH DAMAGE.
	+ *
	+ * $FreeBSD$
	+ */
	+
	+#include <sys/mbuf.h>
	+#include <sys/socket.h>
	+#include <netinet/in.h>
	+#include <netipsec/keydb.h>
	+#include <netipsec/ipsec_accel.h>
	+#include <dev/mlx5/qp.h>
	+#include <dev/mlx5/mlx5_en/en.h>
	+#include <dev/mlx5/mlx5_accel/ipsec.h>
	+
	+#define MLX5_IPSEC_METADATA_HANDLE(ipsec_metadata) (ipsec_metadata & 0xFFFFFF)
	+
	+int mlx5_accel_ipsec_rx_tag_add(if_t ifp, struct mbuf *mb)
	+{
	+ struct mlx5e_priv *priv;
	+ struct ipsec_accel_in_tag *tag;
	+ struct m_tag *mtag;
	+
	+ priv = if_getsoftc(ifp);
	+ if (priv->ipsec == NULL)
	+ return (0);
	+
	+ mtag = m_tag_get(PACKET_TAG_IPSEC_ACCEL_IN, sizeof(*tag), M_NOWAIT);
	+ if (mtag == NULL)
	+ return -ENOMEM;
	+
	+ m_tag_prepend(mb, mtag);
	+ return 0;
	+}
	+
	+int mlx5e_accel_ipsec_handle_rx_cqe(struct mbuf mb, struct mlx5_cqe64 cqe)
	+{
	+ struct ipsec_accel_in_tag *tag;
	+ u32 drv_spi;
	+
	+ drv_spi = MLX5_IPSEC_METADATA_HANDLE(be32_to_cpu(cqe->ft_metadata));
	+ tag = (struct ipsec_accel_in_tag *) m_tag_find(mb, PACKET_TAG_IPSEC_ACCEL_IN, NULL);
	+ WARN_ON(tag == NULL);
	+ if (tag)
	+ tag->drv_spi = drv_spi;
	+
	+ return 0;
	+}
	+
	+void
	+mlx5e_accel_ipsec_handle_tx_wqe(struct mbuf mb, struct mlx5e_tx_wqe wqe,
	+ struct ipsec_accel_out_tag *tag)
	+{
	+ wqe->eth.flow_table_metadata = cpu_to_be32(
	+ mlx5e_accel_ipsec_get_metadata(tag->drv_spi));
	+}
	diff --git a/sys/dev/mlx5/mlx5_attic/mlx5_flow_table.c b/sys/dev/mlx5/mlx5_attic/mlx5_flow_table.c
	--- a/sys/dev/mlx5/mlx5_attic/mlx5_flow_table.c
	+++ b/sys/dev/mlx5/mlx5_attic/mlx5_flow_table.c
	@@ -235,9 +235,12 @@
	misc_parameters);
	void *mc_inner = MLX5_ADDR_OF(fte_match_param, match_criteria,
	inner_headers);
	+ void *mc_misc2 = MLX5_ADDR_OF(fte_match_param, match_criteria,
	+ misc_parameters_2);
	int mc_outer_sz = MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4);
	int mc_misc_sz = MLX5_ST_SZ_BYTES(fte_match_set_misc);
	int mc_inner_sz = MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4);
	+ int mc_misc2_sz = MLX5_ST_SZ_BYTES(fte_match_set_misc2);
	int i;

	for (i = 0; i < ft->num_groups; i++) {
	@@ -251,6 +254,9 @@
	void *gmc_inner = MLX5_ADDR_OF(fte_match_param,
	g->match_criteria,
	inner_headers);
	+ void *gmc_misc2 = MLX5_ADDR_OF(fte_match_param,
	+ g->match_criteria,
	+ misc_parameters_2);

	if (g->match_criteria_enable != match_criteria_enable)
	continue;
	@@ -267,6 +273,9 @@
	if (memcmp(mc_inner, gmc_inner, mc_inner_sz))
	continue;

	+ if (match_criteria_enable & MLX5_MATCH_MISC_PARAMETERS_2)
	+ if (memcmp(mc_misc2, gmc_misc2, mc_misc2_sz))
	+ continue;
	*group_ix = i;
	return 0;
	}
	diff --git a/sys/dev/mlx5/mlx5_attic/mlx5_fs_debugfs.c b/sys/dev/mlx5/mlx5_attic/mlx5_fs_debugfs.c
	--- a/sys/dev/mlx5/mlx5_attic/mlx5_fs_debugfs.c
	+++ b/sys/dev/mlx5/mlx5_attic/mlx5_fs_debugfs.c
	@@ -858,6 +858,10 @@
	case MLX5_FLOW_DESTINATION_TYPE_VPORT:
	return simple_read_from_buffer(buf, count, pos, "vport\n",
	sizeof("vport\n") - 1);
	+ case MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE:
	+ return simple_read_from_buffer(buf, count, pos,
	+ "table type\n",
	+ sizeof("table type\n") - 1);
	}
	return 0;
	}
	@@ -1281,8 +1285,8 @@
	if (!dst->debugfs.type)
	return -ENOMEM;

	- if (dst->dest_attr.type ==
	- MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE) {
	+ if (dst->dest_attr.type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE \|\|
	+ dst->dest_attr.type == MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE) {
	char *path;

	if (IS_ERR_OR_NULL(dst->dest_attr.ft->base.debugfs.dir))
	@@ -1299,9 +1303,14 @@
	*/
	snprintf(path, 1024, "../../../../../../../%s",
	dst->dest_attr.ft->base.debugfs.dir->d_name.name);
	- dst->debugfs.ft = debugfs_create_symlink("flow_table",
	- base->debugfs.dir,
	- path);
	+ if (dst->dest_attr.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE)
	+ dst->debugfs.ft = debugfs_create_symlink("flow_table",
	+ base->debugfs.dir,
	+ path);
	+ else
	+ dst->debugfs.ft = debugfs_create_symlink("table_type",
	+ base->debugfs.dir,
	+ path);
	if (!dst->debugfs.ft) {
	kfree(path);
	return -ENOMEM;
	diff --git a/sys/dev/mlx5/mlx5_core/eswitch.h b/sys/dev/mlx5/mlx5_core/eswitch.h
	--- a/sys/dev/mlx5/mlx5_core/eswitch.h
	+++ b/sys/dev/mlx5/mlx5_core/eswitch.h
	@@ -31,6 +31,8 @@
	#include <linux/if_ether.h>
	#include <dev/mlx5/device.h>

	+#define MLX5_ESWITCH_MANAGER(mdev) MLX5_CAP_GEN(mdev, eswitch_flow_table)
	+
	#define MLX5_MAX_UC_PER_VPORT(dev) \
	(1 << MLX5_CAP_GEN(dev, log_max_current_uc_list))

	@@ -85,15 +87,15 @@
	struct vport_ingress {
	struct mlx5_flow_table *acl;
	struct mlx5_flow_group *drop_grp;
	- struct mlx5_flow_rule *drop_rule;
	+ struct mlx5_flow_handle *drop_rule;
	};

	struct vport_egress {
	struct mlx5_flow_table *acl;
	struct mlx5_flow_group *allowed_vlans_grp;
	struct mlx5_flow_group *drop_grp;
	- struct mlx5_flow_rule *allowed_vlan;
	- struct mlx5_flow_rule *drop_rule;
	+ struct mlx5_flow_handle *allowed_vlan;
	+ struct mlx5_flow_handle *drop_rule;
	};

	struct mlx5_vport {
	diff --git a/sys/dev/mlx5/mlx5_core/fs_chains.h b/sys/dev/mlx5/mlx5_core/fs_chains.h
	new file mode 100644
	--- /dev/null
	+++ b/sys/dev/mlx5/mlx5_core/fs_chains.h
	@@ -0,0 +1,71 @@
	+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
	+/* Copyright (c) 2020 Mellanox Technologies. */
	+
	+#ifndef __ML5_ESW_CHAINS_H__
	+#define __ML5_ESW_CHAINS_H__
	+
	+#include <dev/mlx5/fs.h>
	+
	+struct mlx5_fs_chains;
	+
	+enum mlx5_chains_flags {
	+ MLX5_CHAINS_AND_PRIOS_SUPPORTED = BIT(0),
	+ MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED = BIT(1),
	+ MLX5_CHAINS_FT_TUNNEL_SUPPORTED = BIT(2),
	+};
	+
	+struct mlx5_chains_attr {
	+ enum mlx5_flow_namespace_type ns;
	+ int fs_base_prio;
	+ int fs_base_level;
	+ u32 flags;
	+ u32 max_grp_num;
	+ struct mlx5_flow_table *default_ft;
	+};
	+
	+bool
	+mlx5_chains_prios_supported(struct mlx5_fs_chains *chains);
	+bool mlx5_chains_ignore_flow_level_supported(struct mlx5_fs_chains *chains);
	+bool
	+mlx5_chains_backwards_supported(struct mlx5_fs_chains *chains);
	+u32
	+mlx5_chains_get_prio_range(struct mlx5_fs_chains *chains);
	+u32
	+mlx5_chains_get_chain_range(struct mlx5_fs_chains *chains);
	+u32
	+mlx5_chains_get_nf_ft_chain(struct mlx5_fs_chains *chains);
	+
	+struct mlx5_flow_table *
	+mlx5_chains_get_table(struct mlx5_fs_chains *chains, u32 chain, u32 prio,
	+ u32 level);
	+void
	+mlx5_chains_put_table(struct mlx5_fs_chains *chains, u32 chain, u32 prio,
	+ u32 level);
	+
	+struct mlx5_flow_table *
	+mlx5_chains_get_tc_end_ft(struct mlx5_fs_chains *chains);
	+
	+struct mlx5_flow_table *
	+mlx5_chains_create_global_table(struct mlx5_fs_chains *chains);
	+void
	+mlx5_chains_destroy_global_table(struct mlx5_fs_chains *chains,
	+ struct mlx5_flow_table *ft);
	+
	+int
	+mlx5_chains_get_chain_mapping(struct mlx5_fs_chains *chains, u32 chain,
	+ u32 *chain_mapping);
	+int
	+mlx5_chains_put_chain_mapping(struct mlx5_fs_chains *chains,
	+ u32 chain_mapping);
	+
	+struct mlx5_fs_chains *
	+mlx5_chains_create(struct mlx5_core_dev dev, struct mlx5_chains_attr attr);
	+void mlx5_chains_destroy(struct mlx5_fs_chains *chains);
	+
	+void
	+mlx5_chains_set_end_ft(struct mlx5_fs_chains *chains,
	+ struct mlx5_flow_table *ft);
	+void
	+mlx5_chains_print_info(struct mlx5_fs_chains *chains);
	+
	+#endif /* __ML5_ESW_CHAINS_H__ */
	diff --git a/sys/dev/mlx5/mlx5_core/fs_cmd.h b/sys/dev/mlx5/mlx5_core/fs_cmd.h
	new file mode 100644
	--- /dev/null
	+++ b/sys/dev/mlx5/mlx5_core/fs_cmd.h
	@@ -0,0 +1,120 @@
	+/*
	+ * Copyright (c) 2015, Mellanox Technologies. All rights reserved.
	+ *
	+ * This software is available to you under a choice of one of two
	+ * licenses. You may choose to be licensed under the terms of the GNU
	+ * General Public License (GPL) Version 2, available from the file
	+ * COPYING in the main directory of this source tree, or the
	+ * OpenIB.org BSD license below:
	+ *
	+ * Redistribution and use in source and binary forms, with or
	+ * without modification, are permitted provided that the following
	+ * conditions are met:
	+ *
	+ * - Redistributions of source code must retain the above
	+ * copyright notice, this list of conditions and the following
	+ * disclaimer.
	+ *
	+ * - Redistributions in binary form must reproduce the above
	+ * copyright notice, this list of conditions and the following
	+ * disclaimer in the documentation and/or other materials
	+ * provided with the distribution.
	+ *
	+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	+ * SOFTWARE.
	+ */
	+
	+#ifndef _MLX5_FS_CMD_
	+#define _MLX5_FS_CMD_
	+
	+#include "fs_core.h"
	+
	+struct mlx5_flow_cmds {
	+ int (create_flow_table)(struct mlx5_flow_root_namespace ns,
	+ struct mlx5_flow_table *ft,
	+ struct mlx5_flow_table_attr *ft_attr,
	+ struct mlx5_flow_table *next_ft);
	+ int (destroy_flow_table)(struct mlx5_flow_root_namespace ns,
	+ struct mlx5_flow_table *ft);
	+
	+ int (modify_flow_table)(struct mlx5_flow_root_namespace ns,
	+ struct mlx5_flow_table *ft,
	+ struct mlx5_flow_table *next_ft);
	+
	+ int (create_flow_group)(struct mlx5_flow_root_namespace ns,
	+ struct mlx5_flow_table *ft,
	+ u32 *in,
	+ struct mlx5_flow_group *fg);
	+
	+ int (destroy_flow_group)(struct mlx5_flow_root_namespace ns,
	+ struct mlx5_flow_table *ft,
	+ struct mlx5_flow_group *fg);
	+
	+ int (create_fte)(struct mlx5_flow_root_namespace ns,
	+ struct mlx5_flow_table *ft,
	+ struct mlx5_flow_group *fg,
	+ struct fs_fte *fte);
	+
	+ int (update_fte)(struct mlx5_flow_root_namespace ns,
	+ struct mlx5_flow_table *ft,
	+ struct mlx5_flow_group *fg,
	+ int modify_mask,
	+ struct fs_fte *fte);
	+
	+ int (delete_fte)(struct mlx5_flow_root_namespace ns,
	+ struct mlx5_flow_table *ft,
	+ struct fs_fte *fte);
	+
	+ int (update_root_ft)(struct mlx5_flow_root_namespace ns,
	+ struct mlx5_flow_table *ft,
	+ u32 underlay_qpn,
	+ bool disconnect);
	+
	+ int (packet_reformat_alloc)(struct mlx5_flow_root_namespace ns,
	+ struct mlx5_pkt_reformat_params *params,
	+ enum mlx5_flow_namespace_type namespace,
	+ struct mlx5_pkt_reformat *pkt_reformat);
	+
	+ void (packet_reformat_dealloc)(struct mlx5_flow_root_namespace ns,
	+ struct mlx5_pkt_reformat *pkt_reformat);
	+
	+ int (modify_header_alloc)(struct mlx5_flow_root_namespace ns,
	+ u8 namespace, u8 num_actions,
	+ void *modify_actions,
	+ struct mlx5_modify_hdr *modify_hdr);
	+
	+ void (modify_header_dealloc)(struct mlx5_flow_root_namespace ns,
	+ struct mlx5_modify_hdr *modify_hdr);
	+
	+ int (set_peer)(struct mlx5_flow_root_namespace ns,
	+ struct mlx5_flow_root_namespace *peer_ns);
	+
	+ int (create_ns)(struct mlx5_flow_root_namespace ns);
	+ int (destroy_ns)(struct mlx5_flow_root_namespace ns);
	+
	+ u32 (get_capabilities)(struct mlx5_flow_root_namespace ns,
	+ enum fs_flow_table_type ft_type);
	+};
	+
	+int mlx5_cmd_fc_alloc(struct mlx5_core_dev dev, u32 id);
	+int mlx5_cmd_fc_bulk_alloc(struct mlx5_core_dev *dev,
	+ enum mlx5_fc_bulk_alloc_bitmask alloc_bitmask,
	+ u32 *id);
	+int mlx5_cmd_fc_free(struct mlx5_core_dev *dev, u32 id);
	+int mlx5_cmd_fc_query(struct mlx5_core_dev *dev, u32 id,
	+ u64 packets, u64 bytes);
	+
	+int mlx5_cmd_fc_get_bulk_query_out_len(int bulk_len);
	+int mlx5_cmd_fc_bulk_query(struct mlx5_core_dev *dev, u32 base_id, int bulk_len,
	+ u32 *out);
	+
	+const struct mlx5_flow_cmds *mlx5_fs_cmd_get_default(enum fs_flow_table_type type);
	+const struct mlx5_flow_cmds *mlx5_fs_cmd_get_fw_cmds(void);
	+
	+#endif
	diff --git a/sys/dev/mlx5/mlx5_core/fs_core.h b/sys/dev/mlx5/mlx5_core/fs_core.h
	--- a/sys/dev/mlx5/mlx5_core/fs_core.h
	+++ b/sys/dev/mlx5/mlx5_core/fs_core.h
	@@ -1,333 +1,359 @@
	-/*-
	- * Copyright (c) 2013-2017, Mellanox Technologies, Ltd. All rights reserved.
	+/*
	+ * Copyright (c) 2015, Mellanox Technologies. All rights reserved.
	*
	- * Redistribution and use in source and binary forms, with or without
	- * modification, are permitted provided that the following conditions
	- * are met:
	- * 1. Redistributions of source code must retain the above copyright
	- * notice, this list of conditions and the following disclaimer.
	- * 2. Redistributions in binary form must reproduce the above copyright
	- * notice, this list of conditions and the following disclaimer in the
	- * documentation and/or other materials provided with the distribution.
	+ * This software is available to you under a choice of one of two
	+ * licenses. You may choose to be licensed under the terms of the GNU
	+ * General Public License (GPL) Version 2, available from the file
	+ * COPYING in the main directory of this source tree, or the
	+ * OpenIB.org BSD license below:
	*
	- * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
	- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	- * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
	- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	- * SUCH DAMAGE.
	+ * Redistribution and use in source and binary forms, with or
	+ * without modification, are permitted provided that the following
	+ * conditions are met:
	*
	- * $FreeBSD$
	+ * - Redistributions of source code must retain the above
	+ * copyright notice, this list of conditions and the following
	+ * disclaimer.
	+ *
	+ * - Redistributions in binary form must reproduce the above
	+ * copyright notice, this list of conditions and the following
	+ * disclaimer in the documentation and/or other materials
	+ * provided with the distribution.
	+ *
	+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	+ * SOFTWARE.
	*/

	#ifndef _MLX5_FS_CORE_
	#define _MLX5_FS_CORE_

	-#include <asm/atomic.h>
	-#include <linux/completion.h>
	-#include <linux/mutex.h>
	#include <dev/mlx5/fs.h>

	-enum fs_type {
	+#define FDB_TC_MAX_CHAIN 3
	+#define FDB_FT_CHAIN (FDB_TC_MAX_CHAIN + 1)
	+#define FDB_TC_SLOW_PATH_CHAIN (FDB_FT_CHAIN + 1)
	+
	+/* The index of the last real chain (FT) + 1 as chain zero is valid as well */
	+#define FDB_NUM_CHAINS (FDB_FT_CHAIN + 1)
	+
	+#define FDB_TC_MAX_PRIO 16
	+#define FDB_TC_LEVELS_PER_PRIO 2
	+
	+struct mlx5_flow_definer {
	+ enum mlx5_flow_namespace_type ns_type;
	+ u32 id;
	+};
	+
	+struct mlx5_modify_hdr {
	+ enum mlx5_flow_namespace_type ns_type;
	+ union {
	+ u32 id;
	+ };
	+};
	+
	+struct mlx5_pkt_reformat {
	+ enum mlx5_flow_namespace_type ns_type;
	+ int reformat_type; /* from mlx5_ifc */
	+ union {
	+ u32 id;
	+ };
	+};
	+
	+/* FS_TYPE_PRIO_CHAINS is a PRIO that will have namespaces only,
	+ * and those are in parallel to one another when going over them to connect
	+ * a new flow table. Meaning the last flow table in a TYPE_PRIO prio in one
	+ * parallel namespace will not automatically connect to the first flow table
	+ * found in any prio in any next namespace, but skip the entire containing
	+ * TYPE_PRIO_CHAINS prio.
	+ *
	+ * This is used to implement tc chains, each chain of prios is a different
	+ * namespace inside a containing TYPE_PRIO_CHAINS prio.
	+ */
	+
	+enum fs_node_type {
	FS_TYPE_NAMESPACE,
	FS_TYPE_PRIO,
	+ FS_TYPE_PRIO_CHAINS,
	FS_TYPE_FLOW_TABLE,
	FS_TYPE_FLOW_GROUP,
	FS_TYPE_FLOW_ENTRY,
	FS_TYPE_FLOW_DEST
	};

	-enum fs_ft_type {
	+/**********************************************************************************************************/
	+
	+
	+#define fs_ft_type fs_flow_table_type
	+/************************************************************************************************************/
	+enum fs_flow_table_type {
	FS_FT_NIC_RX = 0x0,
	+ FS_FT_NIC_TX = 0x1,
	FS_FT_ESW_EGRESS_ACL = 0x2,
	FS_FT_ESW_INGRESS_ACL = 0x3,
	FS_FT_FDB = 0X4,
	- FS_FT_SNIFFER_RX = 0x5,
	- FS_FT_SNIFFER_TX = 0x6
	+ FS_FT_SNIFFER_RX = 0X5,
	+ FS_FT_SNIFFER_TX = 0X6,
	+ FS_FT_RDMA_RX = 0X7,
	+ FS_FT_RDMA_TX = 0X8,
	+ FS_FT_PORT_SEL = 0X9,
	+ FS_FT_MAX_TYPE = FS_FT_PORT_SEL,
	+};
	+
	+enum fs_flow_table_op_mod {
	+ FS_FT_OP_MOD_NORMAL,
	+ FS_FT_OP_MOD_LAG_DEMUX,
	};

	enum fs_fte_status {
	FS_FTE_STATUS_EXISTING = 1UL << 0,
	};

	-/* Should always be the first variable in the struct */
	-struct fs_base {
	- struct list_head list;
	- struct fs_base *parent;
	- enum fs_type type;
	- struct kref refcount;
	+enum mlx5_flow_steering_mode {
	+ MLX5_FLOW_STEERING_MODE_DMFS,
	+ MLX5_FLOW_STEERING_MODE_SMFS
	+};
	+
	+enum mlx5_flow_steering_capabilty {
	+ MLX5_FLOW_STEERING_CAP_VLAN_PUSH_ON_RX = 1UL << 0,
	+ MLX5_FLOW_STEERING_CAP_VLAN_POP_ON_TX = 1UL << 1,
	+ MLX5_FLOW_STEERING_CAP_MATCH_RANGES = 1UL << 2,
	+};
	+
	+struct mlx5_flow_steering {
	+ struct mlx5_core_dev *dev;
	+ enum mlx5_flow_steering_mode mode;
	+ struct kmem_cache *fgs_cache;
	+ struct kmem_cache *ftes_cache;
	+ struct mlx5_flow_root_namespace *root_ns;
	+ struct mlx5_flow_root_namespace *fdb_root_ns;
	+ struct mlx5_flow_namespace **fdb_sub_ns;
	+ struct mlx5_flow_root_namespace **esw_egress_root_ns;
	+ struct mlx5_flow_root_namespace **esw_ingress_root_ns;
	+ struct mlx5_flow_root_namespace *sniffer_tx_root_ns;
	+ struct mlx5_flow_root_namespace *sniffer_rx_root_ns;
	+ struct mlx5_flow_root_namespace *rdma_rx_root_ns;
	+ struct mlx5_flow_root_namespace *rdma_tx_root_ns;
	+ struct mlx5_flow_root_namespace *egress_root_ns;
	+ struct mlx5_flow_root_namespace *port_sel_root_ns;
	+ int esw_egress_acl_vports;
	+ int esw_ingress_acl_vports;
	+};
	+
	+struct fs_node {
	+ struct list_head list;
	+ struct list_head children;
	+ enum fs_node_type type;
	+ struct fs_node *parent;
	+ struct fs_node *root;
	/* lock the node for writing and traversing */
	- struct mutex lock;
	- struct completion complete;
	- atomic_t users_refcount;
	- const char *name;
	+ struct rw_semaphore lock;
	+ refcount_t refcount;
	+ bool active;
	+ void (del_hw_func)(struct fs_node );
	+ void (del_sw_func)(struct fs_node );
	+ atomic_t version;
	};

	struct mlx5_flow_rule {
	- struct fs_base base;
	+ struct fs_node node;
	+ struct mlx5_flow_table *ft;
	struct mlx5_flow_destination dest_attr;
	- struct list_head clients_data;
	- /protect clients lits/
	- struct mutex clients_lock;
	-};
	-
	-struct fs_fte {
	- struct fs_base base;
	- u32 val[MLX5_ST_SZ_DW(fte_match_param)];
	- uint32_t dests_size;
	- struct list_head dests;
	- uint32_t index; /* index in ft */
	- struct mlx5_flow_act flow_act;
	- u32 sw_action; /* enum mlx5_rule_fwd_action */
	- enum fs_fte_status status;
	+ /* next_ft should be accessed under chain_lock and only of
	+ * destination type is FWD_NEXT_fT.
	+ */
	+ struct list_head next_ft;
	+ u32 sw_action;
	};

	-struct fs_star_rule {
	- struct mlx5_flow_group *fg;
	- struct fs_fte *fte;
	+struct mlx5_flow_handle {
	+ int num_rules;
	+ struct mlx5_flow_rule *rule[];
	};

	+/* Type of children is mlx5_flow_group */
	struct mlx5_flow_table {
	- struct fs_base base;
	- /* sorted list by start_index */
	- struct list_head fgs;
	+ struct fs_node node;
	+ u32 id;
	+ u16 vport;
	+ unsigned int max_fte;
	+ unsigned int level;
	+ enum fs_flow_table_type type;
	+ enum fs_flow_table_op_mod op_mod;
	struct {
	bool active;
	- unsigned int max_types;
	+ unsigned int required_groups;
	unsigned int group_size;
	- unsigned int num_types;
	+ unsigned int num_groups;
	unsigned int max_fte;
	} autogroup;
	- unsigned int max_fte;
	- unsigned int level;
	- uint32_t id;
	- u16 vport;
	- enum fs_ft_type type;
	- struct fs_star_rule star_rule;
	- unsigned int shared_refcount;
	+ /* Protect fwd_rules */
	+ struct mutex lock;
	+ /* FWD rules that point on this flow table */
	+ struct list_head fwd_rules;
	+ u32 flags;
	+ struct xarray fgs_xa;
	+ enum mlx5_flow_table_miss_action def_miss_action;
	+ struct mlx5_flow_namespace *ns;
	};

	-enum fs_prio_flags {
	- MLX5_CORE_FS_PRIO_SHARED = 1
	+struct mlx5_ft_underlay_qp {
	+ struct list_head list;
	+ u32 qpn;
	};

	+#define MLX5_FTE_MATCH_PARAM_RESERVED reserved_at_e00
	+/* Calculate the fte_match_param length and without the reserved length.
	+ * Make sure the reserved field is the last.
	+ */
	+#define MLX5_ST_SZ_DW_MATCH_PARAM \
	+ ((MLX5_BYTE_OFF(fte_match_param, MLX5_FTE_MATCH_PARAM_RESERVED) / sizeof(u32)) + \
	+ BUILD_BUG_ON_ZERO(MLX5_ST_SZ_BYTES(fte_match_param) != \
	+ MLX5_FLD_SZ_BYTES(fte_match_param, \
	+ MLX5_FTE_MATCH_PARAM_RESERVED) +\
	+ MLX5_BYTE_OFF(fte_match_param, \
	+ MLX5_FTE_MATCH_PARAM_RESERVED)))
	+
	+/* Type of children is mlx5_flow_rule */
	+struct fs_fte {
	+ struct fs_node node;
	+ u32 val[MLX5_ST_SZ_DW_MATCH_PARAM];
	+ u32 dests_size;
	+ u32 fwd_dests;
	+ u32 index;
	+ struct mlx5_flow_context flow_context;
	+ struct mlx5_flow_act action;
	+ enum fs_fte_status status;
	+ struct mlx5_fc *counter;
	+ int modify_mask;
	+};
	+
	+/* Type of children is mlx5_flow_table/namespace */
	struct fs_prio {
	- struct fs_base base;
	- struct list_head objs; /* each object is a namespace or ft */
	- unsigned int max_ft;
	- unsigned int num_ft;
	- unsigned int max_ns;
	+ struct fs_node node;
	+ unsigned int num_levels;
	+ unsigned int start_level;
	unsigned int prio;
	- /When create shared flow table, this lock should be taken/
	- struct mutex shared_lock;
	- u8 flags;
	+ unsigned int num_ft;
	};

	+/* Type of children is fs_prio */
	struct mlx5_flow_namespace {
	/* parent == NULL => root ns */
	- struct fs_base base;
	- /* sorted by priority number */
	- struct list_head prios; /* list of fs_prios */
	- struct list_head list_notifiers;
	- struct rw_semaphore notifiers_rw_sem;
	- struct rw_semaphore dests_rw_sem;
	+ struct fs_node node;
	+ enum mlx5_flow_table_miss_action def_miss_action;
	+};
	+
	+struct mlx5_flow_group_mask {
	+ u8 match_criteria_enable;
	+ u32 match_criteria[MLX5_ST_SZ_DW_MATCH_PARAM];
	+};
	+
	+/* Type of children is fs_fte */
	+struct mlx5_flow_group {
	+ struct fs_node node;
	+ struct mlx5_flow_group_mask mask;
	+ u32 start_index;
	+ u32 max_ftes;
	+ struct ida fte_allocator;
	+ u32 id;
	+ struct xarray ftes_xa;
	};

	struct mlx5_flow_root_namespace {
	struct mlx5_flow_namespace ns;
	- struct mlx5_flow_table *ft_level_0;
	- enum fs_ft_type table_type;
	+ enum mlx5_flow_steering_mode mode;
	+ enum fs_flow_table_type table_type;
	struct mlx5_core_dev *dev;
	struct mlx5_flow_table *root_ft;
	- /* When chaining flow-tables, this lock should be taken */
	- struct mutex fs_chain_lock;
	+ /* Should be held when chaining flow tables */
	+ struct mutex chain_lock;
	+ struct list_head underlay_qpns;
	+ const struct mlx5_flow_cmds *cmds;
	};

	-struct mlx5_flow_group {
	- struct fs_base base;
	- struct list_head ftes;
	- struct mlx5_core_fs_mask mask;
	- uint32_t start_index;
	- uint32_t max_ftes;
	- uint32_t num_ftes;
	- uint32_t id;
	-};
	+int mlx5_init_fc_stats(struct mlx5_core_dev *dev);
	+void mlx5_cleanup_fc_stats(struct mlx5_core_dev *dev);
	+void mlx5_fc_queue_stats_work(struct mlx5_core_dev *dev,
	+ struct delayed_work *dwork,
	+ unsigned long delay);
	+void mlx5_fc_update_sampling_interval(struct mlx5_core_dev *dev,
	+ unsigned long interval);

	-struct mlx5_flow_handler {
	- struct list_head list;
	- rule_event_fn add_dst_cb;
	- rule_event_fn del_dst_cb;
	- void *client_context;
	- struct mlx5_flow_namespace *ns;
	-};
	+const struct mlx5_flow_cmds *mlx5_fs_cmd_get_fw_cmds(void);

	-struct fs_client_priv_data {
	- struct mlx5_flow_handler *fs_handler;
	- struct list_head list;
	- void *client_dst_data;
	-};
	+int mlx5_flow_namespace_set_peer(struct mlx5_flow_root_namespace *ns,
	+ struct mlx5_flow_root_namespace *peer_ns);

	-struct mlx5_modify_hdr {
	- enum mlx5_flow_namespace_type ns_type;
	- u32 id;
	-};
	+int mlx5_flow_namespace_set_mode(struct mlx5_flow_namespace *ns,
	+ enum mlx5_flow_steering_mode mode);

	-struct mlx5_pkt_reformat {
	- enum mlx5_flow_namespace_type ns_type;
	- int reformat_type; /* from mlx5_ifc */
	- u32 id;
	-};
	+int mlx5_fs_core_alloc(struct mlx5_core_dev *dev);
	+void mlx5_fs_core_free(struct mlx5_core_dev *dev);
	+int mlx5_fs_core_init(struct mlx5_core_dev *dev);
	+void mlx5_fs_core_cleanup(struct mlx5_core_dev *dev);

	-void _fs_remove_node(struct kref *kref);
	-#define fs_get_obj(v, _base) {v = container_of((_base), typeof(*v), base); }
	-#define fs_get_parent(v, child) {v = (child)->base.parent ? \
	- container_of((child)->base.parent, \
	- typeof(*v), base) : NULL; }
	+int mlx5_fs_egress_acls_init(struct mlx5_core_dev *dev, int total_vports);
	+void mlx5_fs_egress_acls_cleanup(struct mlx5_core_dev *dev);
	+int mlx5_fs_ingress_acls_init(struct mlx5_core_dev *dev, int total_vports);
	+void mlx5_fs_ingress_acls_cleanup(struct mlx5_core_dev *dev);

	-#define fs_list_for_each_entry(pos, cond, root) \
	- list_for_each_entry(pos, root, base.list) \
	- if (!(cond)) {} else
	+u32 mlx5_fs_get_capabilities(struct mlx5_core_dev *dev, enum mlx5_flow_namespace_type type);

	-#define fs_list_for_each_entry_continue(pos, cond, root) \
	- list_for_each_entry_continue(pos, root, base.list) \
	- if (!(cond)) {} else
	+struct mlx5_flow_root_namespace find_root(struct fs_node node);

	-#define fs_list_for_each_entry_reverse(pos, cond, root) \
	- list_for_each_entry_reverse(pos, root, base.list) \
	- if (!(cond)) {} else
	+#define fs_get_obj(v, _node) {v = container_of((_node), typeof(*v), node); }

	-#define fs_list_for_each_entry_continue_reverse(pos, cond, root) \
	- list_for_each_entry_continue_reverse(pos, root, base.list) \
	- if (!(cond)) {} else
	+#define fs_list_for_each_entry(pos, root) \
	+ list_for_each_entry(pos, root, node.list)

	-#define fs_for_each_ft(pos, prio) \
	- fs_list_for_each_entry(pos, (pos)->base.type == FS_TYPE_FLOW_TABLE, \
	- &(prio)->objs)
	+#define fs_list_for_each_entry_safe(pos, tmp, root) \
	+ list_for_each_entry_safe(pos, tmp, root, node.list)

	-#define fs_for_each_ft_reverse(pos, prio) \
	- fs_list_for_each_entry_reverse(pos, \
	- (pos)->base.type == FS_TYPE_FLOW_TABLE, \
	- &(prio)->objs)
	+#define fs_for_each_ns_or_ft_reverse(pos, prio) \
	+ list_for_each_entry_reverse(pos, &(prio)->node.children, list)

	-#define fs_for_each_ns(pos, prio) \
	- fs_list_for_each_entry(pos, \
	- (pos)->base.type == FS_TYPE_NAMESPACE, \
	- &(prio)->objs)
	-
	-#define fs_for_each_ns_or_ft_reverse(pos, prio) \
	- list_for_each_entry_reverse(pos, &(prio)->objs, list) \
	- if (!((pos)->type == FS_TYPE_NAMESPACE \|\| \
	- (pos)->type == FS_TYPE_FLOW_TABLE)) {} else
	-
	-#define fs_for_each_ns_or_ft(pos, prio) \
	- list_for_each_entry(pos, &(prio)->objs, list) \
	- if (!((pos)->type == FS_TYPE_NAMESPACE \|\| \
	- (pos)->type == FS_TYPE_FLOW_TABLE)) {} else
	-
	-#define fs_for_each_ns_or_ft_continue_reverse(pos, prio) \
	- list_for_each_entry_continue_reverse(pos, &(prio)->objs, list) \
	- if (!((pos)->type == FS_TYPE_NAMESPACE \|\| \
	- (pos)->type == FS_TYPE_FLOW_TABLE)) {} else
	-
	-#define fs_for_each_ns_or_ft_continue(pos, prio) \
	- list_for_each_entry_continue(pos, &(prio)->objs, list) \
	- if (!((pos)->type == FS_TYPE_NAMESPACE \|\| \
	- (pos)->type == FS_TYPE_FLOW_TABLE)) {} else
	+#define fs_for_each_ns_or_ft(pos, prio) \
	+ list_for_each_entry(pos, (&(prio)->node.children), list)

	#define fs_for_each_prio(pos, ns) \
	- fs_list_for_each_entry(pos, (pos)->base.type == FS_TYPE_PRIO, \
	- &(ns)->prios)
	+ fs_list_for_each_entry(pos, &(ns)->node.children)

	-#define fs_for_each_prio_reverse(pos, ns) \
	- fs_list_for_each_entry_reverse(pos, (pos)->base.type == FS_TYPE_PRIO, \
	- &(ns)->prios)
	+#define fs_for_each_ns(pos, prio) \
	+ fs_list_for_each_entry(pos, &(prio)->node.children)

	-#define fs_for_each_prio_continue(pos, ns) \
	- fs_list_for_each_entry_continue(pos, (pos)->base.type == FS_TYPE_PRIO, \
	- &(ns)->prios)
	+#define fs_for_each_ft(pos, prio) \
	+ fs_list_for_each_entry(pos, &(prio)->node.children)

	-#define fs_for_each_prio_continue_reverse(pos, ns) \
	- fs_list_for_each_entry_continue_reverse(pos, \
	- (pos)->base.type == FS_TYPE_PRIO, \
	- &(ns)->prios)
	+#define fs_for_each_ft_safe(pos, tmp, prio) \
	+ fs_list_for_each_entry_safe(pos, tmp, &(prio)->node.children)

	#define fs_for_each_fg(pos, ft) \
	- fs_list_for_each_entry(pos, (pos)->base.type == FS_TYPE_FLOW_GROUP, \
	- &(ft)->fgs)
	+ fs_list_for_each_entry(pos, &(ft)->node.children)

	#define fs_for_each_fte(pos, fg) \
	- fs_list_for_each_entry(pos, (pos)->base.type == FS_TYPE_FLOW_ENTRY, \
	- &(fg)->ftes)
	+ fs_list_for_each_entry(pos, &(fg)->node.children)
	+
	#define fs_for_each_dst(pos, fte) \
	- fs_list_for_each_entry(pos, (pos)->base.type == FS_TYPE_FLOW_DEST, \
	- &(fte)->dests)
	-
	-int mlx5_cmd_fs_create_ft(struct mlx5_core_dev *dev,
	- u16 vport, enum fs_ft_type type, unsigned int level,
	- unsigned int log_size, const char name, unsigned int table_id);
	-
	-int mlx5_cmd_fs_destroy_ft(struct mlx5_core_dev *dev,
	- u16 vport,
	- enum fs_ft_type type, unsigned int table_id);
	-
	-int mlx5_cmd_fs_create_fg(struct mlx5_core_dev *dev,
	- u32 *in,
	- u16 vport,
	- enum fs_ft_type type, unsigned int table_id,
	- unsigned int *group_id);
	-
	-int mlx5_cmd_fs_destroy_fg(struct mlx5_core_dev *dev,
	- u16 vport,
	- enum fs_ft_type type, unsigned int table_id,
	- unsigned int group_id);
	-
	-
	-int mlx5_cmd_fs_set_fte(struct mlx5_core_dev *dev,
	- u16 vport,
	- enum fs_fte_status *fte_status,
	- u32 *match_val,
	- enum fs_ft_type type, unsigned int table_id,
	- unsigned int index, unsigned int group_id,
	- struct mlx5_flow_act *flow_act,
	- u32 sw_action, int dest_size,
	- struct list_head dests); / mlx5_flow_desination */
	-
	-int mlx5_cmd_fs_delete_fte(struct mlx5_core_dev *dev,
	- u16 vport,
	- enum fs_fte_status *fte_status,
	- enum fs_ft_type type, unsigned int table_id,
	- unsigned int index);
	-
	-int mlx5_cmd_update_root_ft(struct mlx5_core_dev *dev,
	- enum fs_ft_type type,
	- unsigned int id);
	-
	-int mlx5_init_fs(struct mlx5_core_dev *dev);
	-void mlx5_cleanup_fs(struct mlx5_core_dev *dev);
	-void mlx5_fc_update_sampling_interval(struct mlx5_core_dev *dev,
	- unsigned long interval);
	+ fs_list_for_each_entry(pos, &(fte)->node.children)
	+
	+#define MLX5_CAP_FLOWTABLE_TYPE(mdev, cap, type) ( \
	+ (type == FS_FT_NIC_RX) ? MLX5_CAP_FLOWTABLE_NIC_RX(mdev, cap) : \
	+ (type == FS_FT_NIC_TX) ? MLX5_CAP_FLOWTABLE_NIC_TX(mdev, cap) : \
	+ (type == FS_FT_ESW_EGRESS_ACL) ? MLX5_CAP_ESW_EGRESS_ACL(mdev, cap) : \
	+ (type == FS_FT_ESW_INGRESS_ACL) ? MLX5_CAP_ESW_INGRESS_ACL(mdev, cap) : \
	+ (type == FS_FT_FDB) ? MLX5_CAP_ESW_FLOWTABLE_FDB(mdev, cap) : \
	+ (type == FS_FT_SNIFFER_RX) ? MLX5_CAP_FLOWTABLE_SNIFFER_RX(mdev, cap) : \
	+ (type == FS_FT_SNIFFER_TX) ? MLX5_CAP_FLOWTABLE_SNIFFER_TX(mdev, cap) : \
	+ (type == FS_FT_RDMA_RX) ? MLX5_CAP_FLOWTABLE_RDMA_RX(mdev, cap) : \
	+ (type == FS_FT_RDMA_TX) ? MLX5_CAP_FLOWTABLE_RDMA_TX(mdev, cap) : \
	+ (type == FS_FT_PORT_SEL) ? MLX5_CAP_FLOWTABLE_PORT_SELECTION(mdev, cap) : \
	+ (BUILD_BUG_ON_ZERO(FS_FT_PORT_SEL != FS_FT_MAX_TYPE))\
	+ )

	-int mlx5_cmd_modify_header_alloc(struct mlx5_core_dev *dev,
	- enum mlx5_flow_namespace_type namespace,
	- u8 num_actions,
	- void *modify_actions,
	- struct mlx5_modify_hdr *modify_hdr);
	-void mlx5_cmd_modify_header_dealloc(struct mlx5_core_dev *dev,
	- struct mlx5_modify_hdr *modify_hdr);
	-int mlx5_cmd_packet_reformat_alloc(struct mlx5_core_dev *dev,
	- struct mlx5_pkt_reformat_params *params,
	- enum mlx5_flow_namespace_type namespace,
	- struct mlx5_pkt_reformat *pkt_reformat);
	-void mlx5_cmd_packet_reformat_dealloc(struct mlx5_core_dev *dev,
	- struct mlx5_pkt_reformat *pkt_reformat);
	-int mlx5_init_fc_stats(struct mlx5_core_dev *dev);
	-void mlx5_cleanup_fc_stats(struct mlx5_core_dev *dev);
	-void mlx5_fc_queue_stats_work(struct mlx5_core_dev *dev,
	- struct delayed_work *dwork,
	- unsigned long delay);
	#endif
	diff --git a/sys/dev/mlx5/mlx5_core/fs_ft_pool.h b/sys/dev/mlx5/mlx5_core/fs_ft_pool.h
	new file mode 100644
	--- /dev/null
	+++ b/sys/dev/mlx5/mlx5_core/fs_ft_pool.h
	@@ -0,0 +1,23 @@
	+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
	+/* Copyright (c) 2021 Mellanox Technologies. */
	+
	+#ifndef __MLX5_FS_FT_POOL_H__
	+#define __MLX5_FS_FT_POOL_H__
	+
	+#include <linux/module.h>
	+#include <dev/mlx5/driver.h>
	+#include <dev/mlx5/mlx5_core/fs_core.h>
	+#include <linux/compiler.h>
	+
	+#define POOL_NEXT_SIZE 0
	+
	+int mlx5_ft_pool_init(struct mlx5_core_dev *dev);
	+void mlx5_ft_pool_destroy(struct mlx5_core_dev *dev);
	+
	+int
	+mlx5_ft_pool_get_avail_sz(struct mlx5_core_dev *dev, enum fs_flow_table_type table_type,
	+ int desired_size);
	+void
	+mlx5_ft_pool_put_sz(struct mlx5_core_dev *dev, int sz);
	+
	+#endif /* __MLX5_FS_FT_POOL_H__ */
	diff --git a/sys/dev/mlx5/mlx5_core/fs_tcp.h b/sys/dev/mlx5/mlx5_core/fs_tcp.h
	--- a/sys/dev/mlx5/mlx5_core/fs_tcp.h
	+++ b/sys/dev/mlx5/mlx5_core/fs_tcp.h
	@@ -27,15 +27,15 @@
	#define __MLX5E_ACCEL_FS_TCP_H__

	struct inpcb;
	-struct mlx5_flow_rule;
	+struct mlx5_flow_handle;
	struct mlx5e_priv;

	int mlx5e_accel_fs_tcp_create(struct mlx5e_priv *);
	void mlx5e_accel_fs_tcp_destroy(struct mlx5e_priv *);
	-struct mlx5_flow_rule *
	+struct mlx5_flow_handle *
	mlx5e_accel_fs_add_inpcb(struct mlx5e_priv *,
	struct inpcb *, uint32_t tirn, uint32_t flow_tag, uint16_t vlan_id);
	#define MLX5E_ACCEL_FS_ADD_INPCB_NO_VLAN 0xFFFF
	-void mlx5e_accel_fs_del_inpcb(struct mlx5_flow_rule *);
	+void mlx5e_accel_fs_del_inpcb(struct mlx5_flow_handle *);

	#endif /* __MLX5E_ACCEL_FS_TCP_H__ */
	diff --git a/sys/dev/mlx5/mlx5_core/mlx5_cmd.c b/sys/dev/mlx5/mlx5_core/mlx5_cmd.c
	--- a/sys/dev/mlx5/mlx5_core/mlx5_cmd.c
	+++ b/sys/dev/mlx5/mlx5_core/mlx5_cmd.c
	@@ -615,6 +615,9 @@
	MLX5_COMMAND_STR_CASE(MODIFY_GENERAL_OBJ);
	MLX5_COMMAND_STR_CASE(QUERY_GENERAL_OBJ);
	MLX5_COMMAND_STR_CASE(DESTROY_GENERAL_OBJ);
	+ MLX5_COMMAND_STR_CASE(ALLOC_FLOW_COUNTER);
	+ MLX5_COMMAND_STR_CASE(DEALLOC_FLOW_COUNTER);
	+ MLX5_COMMAND_STR_CASE(QUERY_FLOW_COUNTER);
	default: return "unknown command opcode";
	}
	}
	diff --git a/sys/dev/mlx5/mlx5_core/mlx5_core.h b/sys/dev/mlx5/mlx5_core/mlx5_core.h
	--- a/sys/dev/mlx5/mlx5_core/mlx5_core.h
	+++ b/sys/dev/mlx5/mlx5_core/mlx5_core.h
	@@ -165,4 +165,14 @@
	u8 mlx5_get_nic_state(struct mlx5_core_dev *dev);
	void mlx5_set_nic_state(struct mlx5_core_dev *dev, u8 state);

	+/********************************************** TESTTEST******************************************/
	+static inline int mlx5_init_fs(struct mlx5_core_dev *dev)
	+{
	+ return 0;
	+}
	+
	+static inline int mlx5_cleanup_fs(struct mlx5_core_dev *dev)
	+{
	+ return 0;
	+}
	#endif /* __MLX5_CORE_H__ */
	diff --git a/sys/dev/mlx5/mlx5_core/mlx5_crypto.c b/sys/dev/mlx5/mlx5_core/mlx5_crypto.c
	new file mode 100644
	--- /dev/null
	+++ b/sys/dev/mlx5/mlx5_core/mlx5_crypto.c
	@@ -0,0 +1,95 @@
	+/*-
	+ * Copyright (c) 2019-2021, Mellanox Technologies, Ltd. All rights reserved.
	+ *
	+ * Redistribution and use in source and binary forms, with or without
	+ * modification, are permitted provided that the following conditions
	+ * are met:
	+ * 1. Redistributions of source code must retain the above copyright
	+ * notice, this list of conditions and the following disclaimer.
	+ * 2. Redistributions in binary form must reproduce the above copyright
	+ * notice, this list of conditions and the following disclaimer in the
	+ * documentation and/or other materials provided with the distribution.
	+ *
	+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
	+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
	+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	+ * SUCH DAMAGE.
	+ *
	+ * $FreeBSD$
	+ */
	+
	+#include "opt_rss.h"
	+#include "opt_ratelimit.h"
	+
	+#include <linux/kernel.h>
	+#include <linux/module.h>
	+
	+#include <dev/mlx5/driver.h>
	+#include <dev/mlx5/crypto.h>
	+
	+int mlx5_encryption_key_create(struct mlx5_core_dev *mdev, u32 pdn, u32 key_type,
	+ const void p_key, u32 key_len, u32 p_obj_id)
	+{
	+ u32 in[MLX5_ST_SZ_DW(create_encryption_key_in)] = {};
	+ u32 out[MLX5_ST_SZ_DW(create_encryption_key_out)] = {};
	+ u64 general_obj_types;
	+ int err;
	+
	+ general_obj_types = MLX5_CAP_GEN_64(mdev, general_obj_types);
	+ if (!(general_obj_types & MLX5_HCA_CAP_GENERAL_OBJ_TYPES_ENCRYPTION_KEY))
	+ return -EINVAL;
	+
	+ switch (key_len) {
	+ case 128 / 8:
	+ memcpy(MLX5_ADDR_OF(create_encryption_key_in, in,
	+ encryption_key_object.key[4]), p_key, 128 / 8);
	+ MLX5_SET(create_encryption_key_in, in, encryption_key_object.pd, pdn);
	+ MLX5_SET(create_encryption_key_in, in, encryption_key_object.key_size,
	+ MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_KEY_SIZE_128);
	+ MLX5_SET(create_encryption_key_in, in, encryption_key_object.key_type,
	+ key_type);
	+ break;
	+ case 256 / 8:
	+ memcpy(MLX5_ADDR_OF(create_encryption_key_in, in,
	+ encryption_key_object.key[0]), p_key, 256 / 8);
	+ MLX5_SET(create_encryption_key_in, in, encryption_key_object.pd, pdn);
	+ MLX5_SET(create_encryption_key_in, in, encryption_key_object.key_size,
	+ MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_KEY_SIZE_256);
	+ MLX5_SET(create_encryption_key_in, in, encryption_key_object.key_type,
	+ key_type);
	+ break;
	+ default:
	+ return -EINVAL;
	+ }
	+
	+ MLX5_SET(create_encryption_key_in, in, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJ);
	+ MLX5_SET(create_encryption_key_in, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY);
	+
	+ err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
	+ if (err == 0)
	+ *p_obj_id = MLX5_GET(create_encryption_key_out, out, obj_id);
	+
	+ /* avoid leaking key on the stack */
	+ explicit_bzero(in, sizeof(in));
	+
	+ return err;
	+}
	+
	+int mlx5_encryption_key_destroy(struct mlx5_core_dev *mdev, u32 oid)
	+{
	+ u32 in[MLX5_ST_SZ_DW(destroy_encryption_key_in)] = {};
	+ u32 out[MLX5_ST_SZ_DW(destroy_encryption_key_out)] = {};
	+
	+ MLX5_SET(destroy_encryption_key_in, in, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJ);
	+ MLX5_SET(destroy_encryption_key_in, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY);
	+ MLX5_SET(destroy_encryption_key_in, in, obj_id, oid);
	+
	+ return mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
	+}
	diff --git a/sys/dev/mlx5/mlx5_core/mlx5_eq.c b/sys/dev/mlx5/mlx5_core/mlx5_eq.c
	--- a/sys/dev/mlx5/mlx5_core/mlx5_eq.c
	+++ b/sys/dev/mlx5/mlx5_core/mlx5_eq.c
	@@ -33,6 +33,7 @@
	#include <dev/mlx5/mlx5_ifc.h>
	#include <dev/mlx5/mlx5_core/mlx5_core.h>
	#include <dev/mlx5/mlx5_core/eswitch.h>
	+#include <dev/mlx5/mlx5_accel/ipsec.h>

	#ifdef RSS
	#include <net/rss_config.h>
	@@ -165,6 +166,8 @@
	return "MLX5_EVENT_TYPE_CODING_DCBX_CHANGE_EVENT";
	case MLX5_EVENT_TYPE_CODING_GENERAL_NOTIFICATION_EVENT:
	return "MLX5_EVENT_TYPE_CODING_GENERAL_NOTIFICATION_EVENT";
	+ case MLX5_EVENT_TYPE_OBJECT_CHANGE:
	+ return "MLX5_EVENT_TYPE_OBJECT_CHANGE";
	default:
	return "Unrecognized event";
	}
	@@ -366,6 +369,10 @@
	mlx5_temp_warning_event(dev, eqe);
	break;

	+ case MLX5_EVENT_TYPE_OBJECT_CHANGE:
	+ mlx5_object_change_event(dev, eqe);
	+ break;
	+
	default:
	mlx5_core_warn(dev, "Unhandled event 0x%x on EQ 0x%x\n",
	eqe->type, eq->eqn);
	@@ -567,6 +574,10 @@
	MLX5_EVENT_TYPE_CODING_GENERAL_NOTIFICATION_EVENT);
	}

	+ if (mlx5_ipsec_device_caps(dev) & MLX5_IPSEC_CAP_PACKET_OFFLOAD)
	+ async_event_mask \|=
	+ (1ull << MLX5_EVENT_TYPE_OBJECT_CHANGE);
	+
	err = mlx5_create_map_eq(dev, &table->cmd_eq, MLX5_EQ_VEC_CMD,
	MLX5_NUM_CMD_EQE, 1ull << MLX5_EVENT_TYPE_CMD);
	if (err) {
	diff --git a/sys/dev/mlx5/mlx5_core/mlx5_eswitch.c b/sys/dev/mlx5/mlx5_core/mlx5_eswitch.c
	--- a/sys/dev/mlx5/mlx5_core/mlx5_eswitch.c
	+++ b/sys/dev/mlx5/mlx5_core/mlx5_eswitch.c
	@@ -65,7 +65,7 @@
	/* E-Switch MC FDB table hash node */
	struct esw_mc_addr { /* SRIOV only */
	struct l2addr_node node;
	- struct mlx5_flow_rule uplink_rule; / Forward to uplink rule */
	+ struct mlx5_flow_handle uplink_rule; / Forward to uplink rule */
	u32 refcnt;
	};

	@@ -74,7 +74,7 @@
	struct l2addr_node node;
	u8 action;
	u32 vport;
	- struct mlx5_flow_rule flow_rule; / SRIOV only */
	+ struct mlx5_flow_handle flow_rule; / SRIOV only */
	};

	enum {
	@@ -216,59 +216,54 @@
	}

	/* E-Switch FDB */
	-static struct mlx5_flow_rule *
	+static struct mlx5_flow_handle *
	esw_fdb_set_vport_rule(struct mlx5_eswitch *esw, u8 mac[ETH_ALEN], u32 vport)
	{
	- int match_header = MLX5_MATCH_OUTER_HEADERS;
	struct mlx5_flow_destination dest;
	- struct mlx5_flow_rule *flow_rule = NULL;
	+ struct mlx5_flow_handle *flow_rule = NULL;
	struct mlx5_flow_act flow_act = {};
	- u32 *match_v;
	- u32 *match_c;
	+ struct mlx5_flow_spec *spec;
	u8 *dmac_v;
	u8 *dmac_c;

	- match_v = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), GFP_KERNEL);
	- match_c = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), GFP_KERNEL);
	- if (!match_v \|\| !match_c) {
	- printf("mlx5_core: WARN: ""FDB: Failed to alloc match parameters\n");
	+ spec = kzalloc(sizeof(*spec), GFP_KERNEL);
	+ if (!spec) {
	+ printf("mlx5_core: WARN: ""FDB: Failed to alloc flow spec\n");
	goto out;
	}
	- dmac_v = MLX5_ADDR_OF(fte_match_param, match_v,
	+ dmac_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
	outer_headers.dmac_47_16);
	- dmac_c = MLX5_ADDR_OF(fte_match_param, match_c,
	+ dmac_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
	outer_headers.dmac_47_16);

	+ spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
	ether_addr_copy(dmac_v, mac);
	/* Match criteria mask */
	memset(dmac_c, 0xff, 6);

	dest.type = MLX5_FLOW_CONTEXT_DEST_TYPE_VPORT;
	- dest.vport_num = vport;
	+ dest.vport.num = vport;

	esw_debug(esw->dev,
	"\tFDB add rule dmac_v(%pM) dmac_c(%pM) -> vport(%d)\n",
	dmac_v, dmac_c, vport);
	+ flow_act.action = MLX5_FLOW_RULE_FWD_ACTION_DEST;
	flow_rule =
	- mlx5_add_flow_rule(esw->fdb_table.fdb,
	- match_header,
	- match_c,
	- match_v,
	- MLX5_FLOW_RULE_FWD_ACTION_DEST,
	- &flow_act, &dest);
	+ mlx5_add_flow_rules(esw->fdb_table.fdb, spec,
	+ &flow_act, &dest, 1);
	if (IS_ERR_OR_NULL(flow_rule)) {
	printf("mlx5_core: WARN: ""FDB: Failed to add flow rule: dmac_v(%pM) dmac_c(%pM) -> vport(%d), err(%ld)\n", dmac_v, dmac_c, vport, PTR_ERR(flow_rule));
	flow_rule = NULL;
	}
	out:
	- kfree(match_v);
	- kfree(match_c);
	+ kfree(spec);
	return flow_rule;
	}

	static int esw_create_fdb_table(struct mlx5_eswitch *esw)
	{
	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
	+ struct mlx5_flow_table_attr ft_attr = {};
	struct mlx5_core_dev *dev = esw->dev;
	struct mlx5_flow_namespace *root_ns;
	struct mlx5_flow_table *fdb;
	@@ -296,7 +291,9 @@
	/* (-2) Since MaorG said so .. */
	table_size = BIT(MLX5_CAP_ESW_FLOWTABLE_FDB(dev, log_max_ft_size)) - 2;

	- fdb = mlx5_create_flow_table(root_ns, 0, "FDB", table_size);
	+ ft_attr.prio = 0;
	+ ft_attr.max_fte = table_size;
	+ fdb = mlx5_create_flow_table(root_ns, &ft_attr);
	if (IS_ERR_OR_NULL(fdb)) {
	err = PTR_ERR(fdb);
	esw_warn(dev, "Failed to create FDB Table err %d\n", err);
	@@ -398,7 +395,7 @@

	mlx5_mpfs_del_mac(esw->dev, esw_uc->table_index);

	- mlx5_del_flow_rule(&vaddr->flow_rule);
	+ mlx5_del_flow_rules(&vaddr->flow_rule);

	l2addr_hash_del(esw_uc);
	return 0;
	@@ -457,12 +454,12 @@
	vport, mac, vaddr->flow_rule, esw_mc->refcnt,
	esw_mc->uplink_rule);

	- mlx5_del_flow_rule(&vaddr->flow_rule);
	+ mlx5_del_flow_rules(&vaddr->flow_rule);

	if (--esw_mc->refcnt)
	return 0;

	- mlx5_del_flow_rule(&esw_mc->uplink_rule);
	+ mlx5_del_flow_rules(&esw_mc->uplink_rule);

	l2addr_hash_del(esw_mc);
	return 0;
	@@ -603,13 +600,13 @@
	struct mlx5_vport *vport)
	{
	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
	+ struct mlx5_flow_table_attr ft_attr = {};
	struct mlx5_flow_group *vlan_grp = NULL;
	struct mlx5_flow_group *drop_grp = NULL;
	struct mlx5_core_dev *dev = esw->dev;
	struct mlx5_flow_namespace *root_ns;
	struct mlx5_flow_table *acl;
	void *match_criteria;
	- char table_name[32];
	u32 *flow_group_in;
	int table_size = 2;
	int err = 0;
	@@ -630,8 +627,8 @@
	if (!flow_group_in)
	return;

	- snprintf(table_name, 32, "egress_%d", vport->vport);
	- acl = mlx5_create_vport_flow_table(root_ns, vport->vport, 0, table_name, table_size);
	+ ft_attr.max_fte = table_size;
	+ acl = mlx5_create_vport_flow_table(root_ns, &ft_attr, vport->vport);
	if (IS_ERR_OR_NULL(acl)) {
	err = PTR_ERR(acl);
	esw_warn(dev, "Failed to create E-Switch vport[%d] egress flow Table, err(%d)\n",
	@@ -679,8 +676,8 @@
	static void esw_vport_cleanup_egress_rules(struct mlx5_eswitch *esw,
	struct mlx5_vport *vport)
	{
	- mlx5_del_flow_rule(&vport->egress.allowed_vlan);
	- mlx5_del_flow_rule(&vport->egress.drop_rule);
	+ mlx5_del_flow_rules(&vport->egress.allowed_vlan);
	+ mlx5_del_flow_rules(&vport->egress.drop_rule);
	}

	static void esw_vport_disable_egress_acl(struct mlx5_eswitch *esw,
	@@ -704,12 +701,12 @@
	struct mlx5_vport *vport)
	{
	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
	+ struct mlx5_flow_table_attr ft_attr = {};
	struct mlx5_core_dev *dev = esw->dev;
	struct mlx5_flow_namespace *root_ns;
	struct mlx5_flow_table *acl;
	struct mlx5_flow_group *g;
	void *match_criteria;
	- char table_name[32];
	u32 *flow_group_in;
	int table_size = 1;
	int err = 0;
	@@ -730,8 +727,8 @@
	if (!flow_group_in)
	return;

	- snprintf(table_name, 32, "ingress_%d", vport->vport);
	- acl = mlx5_create_vport_flow_table(root_ns, vport->vport, 0, table_name, table_size);
	+ ft_attr.max_fte = table_size;
	+ acl = mlx5_create_vport_flow_table(root_ns, &ft_attr, vport->vport);
	if (IS_ERR_OR_NULL(acl)) {
	err = PTR_ERR(acl);
	esw_warn(dev, "Failed to create E-Switch vport[%d] ingress flow Table, err(%d)\n",
	@@ -764,7 +761,7 @@
	static void esw_vport_cleanup_ingress_rules(struct mlx5_eswitch *esw,
	struct mlx5_vport *vport)
	{
	- mlx5_del_flow_rule(&vport->ingress.drop_rule);
	+ mlx5_del_flow_rules(&vport->ingress.drop_rule);
	}

	static void esw_vport_disable_ingress_acl(struct mlx5_eswitch *esw,
	@@ -786,9 +783,7 @@
	struct mlx5_vport *vport)
	{
	struct mlx5_flow_act flow_act = {};
	- struct mlx5_flow_destination dest;
	- u32 *match_v;
	- u32 *match_c;
	+ struct mlx5_flow_spec *spec;
	int err = 0;

	if (IS_ERR_OR_NULL(vport->ingress.acl)) {
	@@ -807,35 +802,28 @@
	"vport[%d] configure ingress rules, vlan(%d) qos(%d)\n",
	vport->vport, vport->vlan, vport->qos);

	- match_v = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), GFP_KERNEL);
	- match_c = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), GFP_KERNEL);
	- if (!match_v \|\| !match_c) {
	+ spec = kzalloc(sizeof(*spec), GFP_KERNEL);
	+ if (!spec) {
	err = -ENOMEM;
	esw_warn(esw->dev, "vport[%d] configure ingress rules failed, err(%d)\n",
	vport->vport, err);
	goto out;
	}
	- MLX5_SET_TO_ONES(fte_match_param, match_c, outer_headers.cvlan_tag);
	- MLX5_SET_TO_ONES(fte_match_param, match_v, outer_headers.cvlan_tag);
	-
	- dest.type = MLX5_FLOW_CONTEXT_DEST_TYPE_VPORT;
	- dest.vport_num = vport->vport;
	+ MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.cvlan_tag);
	+ MLX5_SET_TO_ONES(fte_match_param, spec->match_value, outer_headers.cvlan_tag);

	+ flow_act.action = MLX5_FLOW_RULE_FWD_ACTION_DROP;
	+ spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
	vport->ingress.drop_rule =
	- mlx5_add_flow_rule(vport->ingress.acl,
	- MLX5_MATCH_OUTER_HEADERS,
	- match_c,
	- match_v,
	- MLX5_FLOW_RULE_FWD_ACTION_DROP,
	- &flow_act, &dest);
	+ mlx5_add_flow_rules(vport->ingress.acl, spec,
	+ &flow_act, NULL, 0);
	if (IS_ERR_OR_NULL(vport->ingress.drop_rule)) {
	err = PTR_ERR(vport->ingress.drop_rule);
	printf("mlx5_core: WARN: ""vport[%d] configure ingress rules, err(%d)\n", vport->vport, err);
	vport->ingress.drop_rule = NULL;
	}
	out:
	- kfree(match_v);
	- kfree(match_c);
	+ kfree(spec);
	return err;
	}

	@@ -843,9 +831,7 @@
	struct mlx5_vport *vport)
	{
	struct mlx5_flow_act flow_act = {};
	- struct mlx5_flow_destination dest;
	- u32 *match_v;
	- u32 *match_c;
	+ struct mlx5_flow_spec *spec;
	int err = 0;

	if (IS_ERR_OR_NULL(vport->egress.acl)) {
	@@ -863,9 +849,8 @@
	"vport[%d] configure egress rules, vlan(%d) qos(%d)\n",
	vport->vport, vport->vlan, vport->qos);

	- match_v = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), GFP_KERNEL);
	- match_c = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), GFP_KERNEL);
	- if (!match_v \|\| !match_c) {
	+ spec = kzalloc(sizeof(*spec), GFP_KERNEL);
	+ if (!spec) {
	err = -ENOMEM;
	esw_warn(esw->dev, "vport[%d] configure egress rules failed, err(%d)\n",
	vport->vport, err);
	@@ -873,21 +858,17 @@
	}

	/* Allowed vlan rule */
	- MLX5_SET_TO_ONES(fte_match_param, match_c, outer_headers.cvlan_tag);
	- MLX5_SET_TO_ONES(fte_match_param, match_v, outer_headers.cvlan_tag);
	- MLX5_SET_TO_ONES(fte_match_param, match_c, outer_headers.first_vid);
	- MLX5_SET(fte_match_param, match_v, outer_headers.first_vid, vport->vlan);
	+ MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.cvlan_tag);
	+ MLX5_SET_TO_ONES(fte_match_param, spec->match_value, outer_headers.cvlan_tag);
	+ MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.first_vid);
	+ MLX5_SET(fte_match_param, spec->match_value, outer_headers.first_vid, vport->vlan);

	- dest.type = MLX5_FLOW_CONTEXT_DEST_TYPE_VPORT;
	- dest.vport_num = vport->vport;
	+ spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
	+ flow_act.action = MLX5_FLOW_RULE_FWD_ACTION_ALLOW;

	vport->egress.allowed_vlan =
	- mlx5_add_flow_rule(vport->egress.acl,
	- MLX5_MATCH_OUTER_HEADERS,
	- match_c,
	- match_v,
	- MLX5_FLOW_RULE_FWD_ACTION_ALLOW,
	- &flow_act, &dest);
	+ mlx5_add_flow_rules(vport->egress.acl, spec,
	+ &flow_act, NULL, 0);
	if (IS_ERR_OR_NULL(vport->egress.allowed_vlan)) {
	err = PTR_ERR(vport->egress.allowed_vlan);
	printf("mlx5_core: WARN: ""vport[%d] configure egress allowed vlan rule failed, err(%d)\n", vport->vport, err);
	@@ -895,24 +876,17 @@
	goto out;
	}

	- /* Drop others rule (star rule) */
	- memset(match_c, 0, MLX5_ST_SZ_BYTES(fte_match_param));
	- memset(match_v, 0, MLX5_ST_SZ_BYTES(fte_match_param));
	+ flow_act.action = MLX5_FLOW_RULE_FWD_ACTION_DROP;
	vport->egress.drop_rule =
	- mlx5_add_flow_rule(vport->egress.acl,
	- 0,
	- match_c,
	- match_v,
	- MLX5_FLOW_RULE_FWD_ACTION_DROP,
	- &flow_act, &dest);
	+ mlx5_add_flow_rules(vport->egress.acl, NULL,
	+ &flow_act, NULL, 0);
	if (IS_ERR_OR_NULL(vport->egress.drop_rule)) {
	err = PTR_ERR(vport->egress.drop_rule);
	printf("mlx5_core: WARN: ""vport[%d] configure egress drop rule failed, err(%d)\n", vport->vport, err);
	vport->egress.drop_rule = NULL;
	}
	out:
	- kfree(match_v);
	- kfree(match_c);
	+ kfree(spec);
	return err;
	}

	diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fc_cmd.h b/sys/dev/mlx5/mlx5_core/mlx5_fc_cmd.h
	deleted file mode 100644
	--- a/sys/dev/mlx5/mlx5_core/mlx5_fc_cmd.h
	+++ /dev/null
	@@ -1,54 +0,0 @@
	-/*
	- * Copyright (c) 2023, NVIDIA Technologies. All rights reserved.
	- *
	- * This software is available to you under a choice of one of two
	- * licenses. You may choose to be licensed under the terms of the GNU
	- * General Public License (GPL) Version 2, available from the file
	- * COPYING in the main directory of this source tree, or the
	- * OpenIB.org BSD license below:
	- *
	- * Redistribution and use in source and binary forms, with or
	- * without modification, are permitted provided that the following
	- * conditions are met:
	- *
	- * - Redistributions of source code must retain the above
	- * copyright notice, this list of conditions and the following
	- * disclaimer.
	- *
	- * - Redistributions in binary form must reproduce the above
	- * copyright notice, this list of conditions and the following
	- * disclaimer in the documentation and/or other materials
	- * provided with the distribution.
	- *
	- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	- * SOFTWARE.
	- */
	-
	-#ifndef _MLX5_FC_CMD_
	-#define _MLX5_FC_CMD_
	-
	-#include "fs_core.h"
	-
	-int mlx5_cmd_fc_alloc(struct mlx5_core_dev dev, u32 id);
	-int mlx5_cmd_fc_bulk_alloc(struct mlx5_core_dev *dev,
	- enum mlx5_fc_bulk_alloc_bitmask alloc_bitmask,
	- u32 *id);
	-int mlx5_cmd_fc_free(struct mlx5_core_dev *dev, u32 id);
	-int mlx5_cmd_fc_query(struct mlx5_core_dev *dev, u32 id,
	- u64 packets, u64 bytes);
	-
	-int mlx5_cmd_fc_bulk_query(struct mlx5_core_dev *dev, u32 base_id, int bulk_len,
	- u32 *out);
	-static inline int mlx5_cmd_fc_get_bulk_query_out_len(int bulk_len)
	-{
	- return MLX5_ST_SZ_BYTES(query_flow_counter_out) +
	- MLX5_ST_SZ_BYTES(traffic_counter) * bulk_len;
	-}
	-
	-#endif
	diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fc_cmd.c b/sys/dev/mlx5/mlx5_core/mlx5_fc_cmd.c
	deleted file mode 100644
	--- a/sys/dev/mlx5/mlx5_core/mlx5_fc_cmd.c
	+++ /dev/null
	@@ -1,102 +0,0 @@
	-/*-
	- * Copyright (c) 2022 NVIDIA corporation & affiliates.
	- *
	- * Redistribution and use in source and binary forms, with or without
	- * modification, are permitted provided that the following conditions
	- * are met:
	- * 1. Redistributions of source code must retain the above copyright
	- * notice, this list of conditions and the following disclaimer.
	- * 2. Redistributions in binary form must reproduce the above copyright
	- * notice, this list of conditions and the following disclaimer in the
	- * documentation and/or other materials provided with the distribution.
	- *
	- * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
	- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	- * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
	- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	- * SUCH DAMAGE.
	- *
	- * $FreeBSD$
	- */
	-
	-#include <dev/mlx5/driver.h>
	-#include <dev/mlx5/device.h>
	-#include <dev/mlx5/mlx5_ifc.h>
	-#include <dev/mlx5/mlx5_core/mlx5_fc_cmd.h>
	-#include <dev/mlx5/mlx5_core/mlx5_core.h>
	-
	-int mlx5_cmd_fc_bulk_alloc(struct mlx5_core_dev *dev,
	- enum mlx5_fc_bulk_alloc_bitmask alloc_bitmask,
	- u32 *id)
	-{
	- u32 out[MLX5_ST_SZ_DW(alloc_flow_counter_out)] = {};
	- u32 in[MLX5_ST_SZ_DW(alloc_flow_counter_in)] = {};
	- int err;
	-
	- MLX5_SET(alloc_flow_counter_in, in, opcode,
	- MLX5_CMD_OP_ALLOC_FLOW_COUNTER);
	- MLX5_SET(alloc_flow_counter_in, in, flow_counter_bulk, alloc_bitmask);
	-
	- err = mlx5_cmd_exec_inout(dev, alloc_flow_counter, in, out);
	- if (!err)
	- *id = MLX5_GET(alloc_flow_counter_out, out, flow_counter_id);
	- return err;
	-}
	-
	-int mlx5_cmd_fc_alloc(struct mlx5_core_dev dev, u32 id)
	-{
	- return mlx5_cmd_fc_bulk_alloc(dev, 0, id);
	-}
	-
	-int mlx5_cmd_fc_free(struct mlx5_core_dev *dev, u32 id)
	-{
	- u32 in[MLX5_ST_SZ_DW(dealloc_flow_counter_in)] = {};
	-
	- MLX5_SET(dealloc_flow_counter_in, in, opcode,
	- MLX5_CMD_OP_DEALLOC_FLOW_COUNTER);
	- MLX5_SET(dealloc_flow_counter_in, in, flow_counter_id, id);
	- return mlx5_cmd_exec_in(dev, dealloc_flow_counter, in);
	-}
	-
	-int mlx5_cmd_fc_query(struct mlx5_core_dev *dev, u32 id,
	- u64 packets, u64 bytes)
	-{
	- u32 out[MLX5_ST_SZ_BYTES(query_flow_counter_out) +
	- MLX5_ST_SZ_BYTES(traffic_counter)] = {};
	- u32 in[MLX5_ST_SZ_DW(query_flow_counter_in)] = {};
	- void *stats;
	- int err = 0;
	-
	- MLX5_SET(query_flow_counter_in, in, opcode,
	- MLX5_CMD_OP_QUERY_FLOW_COUNTER);
	- MLX5_SET(query_flow_counter_in, in, op_mod, 0);
	- MLX5_SET(query_flow_counter_in, in, flow_counter_id, id);
	- err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
	- if (err)
	- return err;
	-
	- stats = MLX5_ADDR_OF(query_flow_counter_out, out, flow_statistics);
	- *packets = MLX5_GET64(traffic_counter, stats, packets);
	- *bytes = MLX5_GET64(traffic_counter, stats, octets);
	- return 0;
	-}
	-
	-int mlx5_cmd_fc_bulk_query(struct mlx5_core_dev *dev, u32 base_id, int bulk_len,
	- u32 *out)
	-{
	- int outlen = mlx5_cmd_fc_get_bulk_query_out_len(bulk_len);
	- u32 in[MLX5_ST_SZ_DW(query_flow_counter_in)] = {};
	-
	- MLX5_SET(query_flow_counter_in, in, opcode,
	- MLX5_CMD_OP_QUERY_FLOW_COUNTER);
	- MLX5_SET(query_flow_counter_in, in, flow_counter_id, base_id);
	- MLX5_SET(query_flow_counter_in, in, num_of_counters, bulk_len);
	- return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen);
	-}
	-
	diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fs_chains.c b/sys/dev/mlx5/mlx5_core/mlx5_fs_chains.c
	new file mode 100644
	--- /dev/null
	+++ b/sys/dev/mlx5/mlx5_core/mlx5_fs_chains.c
	@@ -0,0 +1,664 @@
	+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
	+// Copyright (c) 2020 Mellanox Technologies.
	+
	+#include <dev/mlx5/driver.h>
	+#include <dev/mlx5/mlx5_ifc.h>
	+#include <dev/mlx5/fs.h>
	+
	+#include "mlx5_core.h"
	+#include "fs_chains.h"
	+#include "fs_ft_pool.h"
	+#include "fs_core.h"
	+
	+#define chains_lock(chains) ((chains)->lock)
	+#define chains_xa(chains) ((chains)->chains_xa)
	+#define prios_xa(chains) ((chains)->prios_xa)
	+#define chains_default_ft(chains) ((chains)->chains_default_ft)
	+#define chains_end_ft(chains) ((chains)->chains_end_ft)
	+#define FT_TBL_SZ (64 * 1024)
	+
	+struct mlx5_fs_chains {
	+ struct mlx5_core_dev *dev;
	+
	+ struct xarray chains_xa;
	+ struct xarray prios_xa;
	+ /* Protects above chains_ht and prios_ht */
	+ struct mutex lock;
	+
	+ struct mlx5_flow_table *chains_default_ft;
	+ struct mlx5_flow_table *chains_end_ft;
	+
	+ enum mlx5_flow_namespace_type ns;
	+ u32 group_num;
	+ u32 flags;
	+ int fs_base_prio;
	+ int fs_base_level;
	+};
	+
	+struct fs_chain {
	+ u32 chain;
	+
	+ int ref;
	+ int id;
	+ uint32_t xa_idx;
	+
	+ struct mlx5_fs_chains *chains;
	+ struct list_head prios_list;
	+ struct mlx5_flow_handle *restore_rule;
	+ struct mlx5_modify_hdr *miss_modify_hdr;
	+};
	+
	+struct prio_key {
	+ u32 chain;
	+ u32 prio;
	+ u32 level;
	+};
	+
	+struct prio {
	+ struct list_head list;
	+
	+ struct prio_key key;
	+ uint32_t xa_idx;
	+
	+ int ref;
	+
	+ struct fs_chain *chain;
	+ struct mlx5_flow_table *ft;
	+ struct mlx5_flow_table *next_ft;
	+ struct mlx5_flow_group *miss_group;
	+ struct mlx5_flow_handle *miss_rule;
	+};
	+
	+/*
	+static const struct rhashtable_params chain_params = {
	+ .head_offset = offsetof(struct fs_chain, node),
	+ .key_offset = offsetof(struct fs_chain, chain),
	+ .key_len = sizeof_field(struct fs_chain, chain),
	+ .automatic_shrinking = true,
	+};
	+
	+static const struct rhashtable_params prio_params = {
	+ .head_offset = offsetof(struct prio, node),
	+ .key_offset = offsetof(struct prio, key),
	+ .key_len = sizeof_field(struct prio, key),
	+ .automatic_shrinking = true,
	+};
	+*/
	+
	+bool mlx5_chains_prios_supported(struct mlx5_fs_chains *chains)
	+{
	+ return chains->flags & MLX5_CHAINS_AND_PRIOS_SUPPORTED;
	+}
	+
	+bool mlx5_chains_ignore_flow_level_supported(struct mlx5_fs_chains *chains)
	+{
	+ return chains->flags & MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED;
	+}
	+
	+bool mlx5_chains_backwards_supported(struct mlx5_fs_chains *chains)
	+{
	+ return mlx5_chains_prios_supported(chains) &&
	+ mlx5_chains_ignore_flow_level_supported(chains);
	+}
	+
	+u32 mlx5_chains_get_chain_range(struct mlx5_fs_chains *chains)
	+{
	+ if (!mlx5_chains_prios_supported(chains))
	+ return 1;
	+
	+ if (mlx5_chains_ignore_flow_level_supported(chains))
	+ return UINT_MAX - 1;
	+
	+ /* We should get here only for eswitch case */
	+ return FDB_TC_MAX_CHAIN;
	+}
	+
	+u32 mlx5_chains_get_nf_ft_chain(struct mlx5_fs_chains *chains)
	+{
	+ return mlx5_chains_get_chain_range(chains) + 1;
	+}
	+
	+u32 mlx5_chains_get_prio_range(struct mlx5_fs_chains *chains)
	+{
	+ if (mlx5_chains_ignore_flow_level_supported(chains))
	+ return UINT_MAX;
	+
	+ if (!chains->dev->priv.eswitch)
	+ return 1;
	+
	+ /* We should get here only for eswitch case */
	+ return FDB_TC_MAX_PRIO;
	+}
	+
	+static unsigned int mlx5_chains_get_level_range(struct mlx5_fs_chains *chains)
	+{
	+ if (mlx5_chains_ignore_flow_level_supported(chains))
	+ return UINT_MAX;
	+
	+ /* Same value for FDB and NIC RX tables */
	+ return FDB_TC_LEVELS_PER_PRIO;
	+}
	+
	+void
	+mlx5_chains_set_end_ft(struct mlx5_fs_chains *chains,
	+ struct mlx5_flow_table *ft)
	+{
	+ chains_end_ft(chains) = ft;
	+}
	+
	+static struct mlx5_flow_table *
	+mlx5_chains_create_table(struct mlx5_fs_chains *chains,
	+ u32 chain, u32 prio, u32 level)
	+{
	+ struct mlx5_flow_table_attr ft_attr = {};
	+ struct mlx5_flow_namespace *ns;
	+ struct mlx5_flow_table *ft;
	+ int sz;
	+
	+ if (chains->flags & MLX5_CHAINS_FT_TUNNEL_SUPPORTED)
	+ ft_attr.flags \|= (MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT \|
	+ MLX5_FLOW_TABLE_TUNNEL_EN_DECAP);
	+
	+ sz = (chain == mlx5_chains_get_nf_ft_chain(chains)) ? FT_TBL_SZ : POOL_NEXT_SIZE;
	+ ft_attr.max_fte = sz;
	+
	+ /* We use chains_default_ft(chains) as the table's next_ft till
	+ * ignore_flow_level is allowed on FT creation and not just for FTEs.
	+ * Instead caller should add an explicit miss rule if needed.
	+ */
	+ ft_attr.next_ft = chains_default_ft(chains);
	+
	+ /* The root table(chain 0, prio 1, level 0) is required to be
	+ * connected to the previous fs_core managed prio.
	+ * We always create it, as a managed table, in order to align with
	+ * fs_core logic.
	+ */
	+ if (!mlx5_chains_ignore_flow_level_supported(chains) \|\|
	+ (chain == 0 && prio == 1 && level == 0)) {
	+ ft_attr.level = chains->fs_base_level;
	+ ft_attr.prio = chains->fs_base_prio;
	+ ns = (chains->ns == MLX5_FLOW_NAMESPACE_FDB) ?
	+ mlx5_get_fdb_sub_ns(chains->dev, chain) :
	+ mlx5_get_flow_namespace(chains->dev, chains->ns);
	+ } else {
	+ ft_attr.flags \|= MLX5_FLOW_TABLE_UNMANAGED;
	+ ft_attr.prio = chains->fs_base_prio;
	+ /* Firmware doesn't allow us to create another level 0 table,
	+ * so we create all unmanaged tables as level 1 (base + 1).
	+ *
	+ * To connect them, we use explicit miss rules with
	+ * ignore_flow_level. Caller is responsible to create
	+ * these rules (if needed).
	+ */
	+ ft_attr.level = chains->fs_base_level + 1;
	+ ns = mlx5_get_flow_namespace(chains->dev, chains->ns);
	+ }
	+
	+ ft_attr.autogroup.num_reserved_entries = 2;
	+ ft_attr.autogroup.max_num_groups = chains->group_num;
	+ ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr);
	+ if (IS_ERR(ft)) {
	+ mlx5_core_warn(chains->dev, "Failed to create chains table err %d (chain: %d, prio: %d, level: %d, size: %d)\n",
	+ (int)PTR_ERR(ft), chain, prio, level, sz);
	+ return ft;
	+ }
	+
	+ return ft;
	+}
	+
	+static struct fs_chain *
	+mlx5_chains_create_chain(struct mlx5_fs_chains *chains, u32 chain)
	+{
	+ struct fs_chain *chain_s = NULL;
	+ int err;
	+
	+ chain_s = kvzalloc(sizeof(*chain_s), GFP_KERNEL);
	+ if (!chain_s)
	+ return ERR_PTR(-ENOMEM);
	+
	+ chain_s->chains = chains;
	+ chain_s->chain = chain;
	+ INIT_LIST_HEAD(&chain_s->prios_list);
	+
	+ err = xa_alloc(&chains_xa(chains), &chain_s->xa_idx, chain_s,
	+ xa_limit_32b, GFP_KERNEL);
	+ if (err)
	+ goto err_insert;
	+
	+ return chain_s;
	+
	+err_insert:
	+ kvfree(chain_s);
	+ return ERR_PTR(err);
	+}
	+
	+static void
	+mlx5_chains_destroy_chain(struct fs_chain *chain)
	+{
	+ struct mlx5_fs_chains *chains = chain->chains;
	+
	+ xa_erase(&chains_xa(chains), chain->xa_idx);
	+ kvfree(chain);
	+}
	+
	+static struct fs_chain *
	+mlx5_chains_get_chain(struct mlx5_fs_chains *chains, u32 chain)
	+{
	+ struct fs_chain *chain_s = NULL;
	+ unsigned long idx;
	+
	+ xa_for_each(&chains_xa(chains), idx, chain_s) {
	+ if (chain_s->chain == chain)
	+ break;
	+ }
	+
	+ if (!chain_s) {
	+ chain_s = mlx5_chains_create_chain(chains, chain);
	+ if (IS_ERR(chain_s))
	+ return chain_s;
	+ }
	+
	+ chain_s->ref++;
	+
	+ return chain_s;
	+}
	+
	+static struct mlx5_flow_handle *
	+mlx5_chains_add_miss_rule(struct fs_chain *chain,
	+ struct mlx5_flow_table *ft,
	+ struct mlx5_flow_table *next_ft)
	+{
	+ struct mlx5_flow_destination dest = {};
	+ struct mlx5_flow_act act = {};
	+
	+ act.flags = FLOW_ACT_NO_APPEND;
	+ if (mlx5_chains_ignore_flow_level_supported(chain->chains))
	+ act.flags \|= FLOW_ACT_IGNORE_FLOW_LEVEL;
	+
	+ act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
	+ dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
	+ dest.ft = next_ft;
	+
	+ return mlx5_add_flow_rules(ft, NULL, &act, &dest, 1);
	+}
	+
	+static int
	+mlx5_chains_update_prio_prevs(struct prio *prio,
	+ struct mlx5_flow_table *next_ft)
	+{
	+ struct mlx5_flow_handle *miss_rules[FDB_TC_LEVELS_PER_PRIO + 1] = {};
	+ struct fs_chain *chain = prio->chain;
	+ struct prio *pos;
	+ int n = 0, err;
	+
	+ if (prio->key.level)
	+ return 0;
	+
	+ /* Iterate in reverse order until reaching the level 0 rule of
	+ * the previous priority, adding all the miss rules first, so we can
	+ * revert them if any of them fails.
	+ */
	+ pos = prio;
	+ list_for_each_entry_continue_reverse(pos,
	+ &chain->prios_list,
	+ list) {
	+ miss_rules[n] = mlx5_chains_add_miss_rule(chain,
	+ pos->ft,
	+ next_ft);
	+ if (IS_ERR(miss_rules[n])) {
	+ err = PTR_ERR(miss_rules[n]);
	+ goto err_prev_rule;
	+ }
	+
	+ n++;
	+ if (!pos->key.level)
	+ break;
	+ }
	+
	+ /* Success, delete old miss rules, and update the pointers. */
	+ n = 0;
	+ pos = prio;
	+ list_for_each_entry_continue_reverse(pos,
	+ &chain->prios_list,
	+ list) {
	+ mlx5_del_flow_rules(&pos->miss_rule);
	+
	+ pos->miss_rule = miss_rules[n];
	+ pos->next_ft = next_ft;
	+
	+ n++;
	+ if (!pos->key.level)
	+ break;
	+ }
	+
	+ return 0;
	+
	+err_prev_rule:
	+ while (--n >= 0)
	+ mlx5_del_flow_rules(&miss_rules[n]);
	+
	+ return err;
	+}
	+
	+static void
	+mlx5_chains_put_chain(struct fs_chain *chain)
	+{
	+ if (--chain->ref == 0)
	+ mlx5_chains_destroy_chain(chain);
	+}
	+
	+static struct prio *
	+mlx5_chains_create_prio(struct mlx5_fs_chains *chains,
	+ u32 chain, u32 prio, u32 level)
	+{
	+ int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
	+ struct mlx5_flow_handle *miss_rule;
	+ struct mlx5_flow_group *miss_group;
	+ struct mlx5_flow_table *next_ft;
	+ struct mlx5_flow_table *ft;
	+ struct fs_chain *chain_s;
	+ struct list_head *pos;
	+ struct prio *prio_s;
	+ u32 *flow_group_in;
	+ int err;
	+
	+ chain_s = mlx5_chains_get_chain(chains, chain);
	+ if (IS_ERR(chain_s))
	+ return ERR_CAST(chain_s);
	+
	+ prio_s = kvzalloc(sizeof(*prio_s), GFP_KERNEL);
	+ flow_group_in = kvzalloc(inlen, GFP_KERNEL);
	+ if (!prio_s \|\| !flow_group_in) {
	+ err = -ENOMEM;
	+ goto err_alloc;
	+ }
	+
	+ /* Chain's prio list is sorted by prio and level.
	+ * And all levels of some prio point to the next prio's level 0.
	+ * Example list (prio, level):
	+ * (3,0)->(3,1)->(5,0)->(5,1)->(6,1)->(7,0)
	+ * In hardware, we will we have the following pointers:
	+ * (3,0) -> (5,0) -> (7,0) -> Slow path
	+ * (3,1) -> (5,0)
	+ * (5,1) -> (7,0)
	+ * (6,1) -> (7,0)
	+ */
	+
	+ /* Default miss for each chain: */
	+ next_ft = (chain == mlx5_chains_get_nf_ft_chain(chains)) ?
	+ chains_default_ft(chains) :
	+ chains_end_ft(chains);
	+ list_for_each(pos, &chain_s->prios_list) {
	+ struct prio *p = list_entry(pos, struct prio, list);
	+
	+ /* exit on first pos that is larger */
	+ if (prio < p->key.prio \|\| (prio == p->key.prio &&
	+ level < p->key.level)) {
	+ /* Get next level 0 table */
	+ next_ft = p->key.level == 0 ? p->ft : p->next_ft;
	+ break;
	+ }
	+ }
	+
	+ ft = mlx5_chains_create_table(chains, chain, prio, level);
	+ if (IS_ERR(ft)) {
	+ err = PTR_ERR(ft);
	+ goto err_create;
	+ }
	+
	+ MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index,
	+ ft->max_fte - 2);
	+ MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
	+ ft->max_fte - 1);
	+ miss_group = mlx5_create_flow_group(ft, flow_group_in);
	+ if (IS_ERR(miss_group)) {
	+ err = PTR_ERR(miss_group);
	+ goto err_group;
	+ }
	+
	+ /* Add miss rule to next_ft */
	+ miss_rule = mlx5_chains_add_miss_rule(chain_s, ft, next_ft);
	+ if (IS_ERR(miss_rule)) {
	+ err = PTR_ERR(miss_rule);
	+ goto err_miss_rule;
	+ }
	+
	+ prio_s->miss_group = miss_group;
	+ prio_s->miss_rule = miss_rule;
	+ prio_s->next_ft = next_ft;
	+ prio_s->chain = chain_s;
	+ prio_s->key.chain = chain;
	+ prio_s->key.prio = prio;
	+ prio_s->key.level = level;
	+ prio_s->ft = ft;
	+
	+ err = xa_alloc(&prios_xa(chains), &prio_s->xa_idx, prio_s,
	+ xa_limit_32b, GFP_KERNEL);
	+ if (err)
	+ goto err_insert;
	+
	+ list_add(&prio_s->list, pos->prev);
	+
	+ /* Table is ready, connect it */
	+ err = mlx5_chains_update_prio_prevs(prio_s, ft);
	+ if (err)
	+ goto err_update;
	+
	+ kvfree(flow_group_in);
	+ return prio_s;
	+
	+err_update:
	+ list_del(&prio_s->list);
	+ xa_erase(&prios_xa(chains), prio_s->xa_idx);
	+err_insert:
	+ mlx5_del_flow_rules(&miss_rule);
	+err_miss_rule:
	+ mlx5_destroy_flow_group(miss_group);
	+err_group:
	+ mlx5_destroy_flow_table(ft);
	+err_create:
	+err_alloc:
	+ kvfree(prio_s);
	+ kvfree(flow_group_in);
	+ mlx5_chains_put_chain(chain_s);
	+ return ERR_PTR(err);
	+}
	+
	+static void
	+mlx5_chains_destroy_prio(struct mlx5_fs_chains *chains,
	+ struct prio *prio)
	+{
	+ struct fs_chain *chain = prio->chain;
	+
	+ WARN_ON(mlx5_chains_update_prio_prevs(prio,
	+ prio->next_ft));
	+
	+ list_del(&prio->list);
	+ xa_erase(&prios_xa(chains), prio->xa_idx);
	+ mlx5_del_flow_rules(&prio->miss_rule);
	+ mlx5_destroy_flow_group(prio->miss_group);
	+ mlx5_destroy_flow_table(prio->ft);
	+ mlx5_chains_put_chain(chain);
	+ kvfree(prio);
	+}
	+
	+struct mlx5_flow_table *
	+mlx5_chains_get_table(struct mlx5_fs_chains *chains, u32 chain, u32 prio,
	+ u32 level)
	+{
	+ struct mlx5_flow_table *prev_fts;
	+ struct prio *prio_s;
	+ unsigned long idx;
	+ int l = 0;
	+
	+ if ((chain > mlx5_chains_get_chain_range(chains) &&
	+ chain != mlx5_chains_get_nf_ft_chain(chains)) \|\|
	+ prio > mlx5_chains_get_prio_range(chains) \|\|
	+ level > mlx5_chains_get_level_range(chains))
	+ return ERR_PTR(-EOPNOTSUPP);
	+
	+ /* create earlier levels for correct fs_core lookup when
	+ * connecting tables.
	+ */
	+ for (l = 0; l < level; l++) {
	+ prev_fts = mlx5_chains_get_table(chains, chain, prio, l);
	+ if (IS_ERR(prev_fts)) {
	+ prio_s = ERR_CAST(prev_fts);
	+ goto err_get_prevs;
	+ }
	+ }
	+
	+ mutex_lock(&chains_lock(chains));
	+ xa_for_each(&prios_xa(chains), idx, prio_s) {
	+ if (chain == prio_s->key.chain &&
	+ prio == prio_s->key.prio &&
	+ level == prio_s->key.level)
	+ break;
	+ }
	+ if (!prio_s) {
	+ prio_s = mlx5_chains_create_prio(chains, chain,
	+ prio, level);
	+ if (IS_ERR(prio_s))
	+ goto err_create_prio;
	+ }
	+
	+ ++prio_s->ref;
	+ mutex_unlock(&chains_lock(chains));
	+
	+ return prio_s->ft;
	+
	+err_create_prio:
	+ mutex_unlock(&chains_lock(chains));
	+err_get_prevs:
	+ while (--l >= 0)
	+ mlx5_chains_put_table(chains, chain, prio, l);
	+ return ERR_CAST(prio_s);
	+}
	+
	+void
	+mlx5_chains_put_table(struct mlx5_fs_chains *chains, u32 chain, u32 prio,
	+ u32 level)
	+{
	+ struct prio *prio_s;
	+ unsigned long idx;
	+
	+ mutex_lock(&chains_lock(chains));
	+
	+ xa_for_each(&prios_xa(chains), idx, prio_s) {
	+ if (chain == prio_s->key.chain &&
	+ prio == prio_s->key.prio &&
	+ level == prio_s->key.level)
	+ break;
	+ }
	+ if (!prio_s)
	+ goto err_get_prio;
	+
	+ if (--prio_s->ref == 0)
	+ mlx5_chains_destroy_prio(chains, prio_s);
	+ mutex_unlock(&chains_lock(chains));
	+
	+ while (level-- > 0)
	+ mlx5_chains_put_table(chains, chain, prio, level);
	+
	+ return;
	+
	+err_get_prio:
	+ mutex_unlock(&chains_lock(chains));
	+ WARN_ONCE(1,
	+ "Couldn't find table: (chain: %d prio: %d level: %d)",
	+ chain, prio, level);
	+}
	+
	+struct mlx5_flow_table *
	+mlx5_chains_get_tc_end_ft(struct mlx5_fs_chains *chains)
	+{
	+ return chains_end_ft(chains);
	+}
	+
	+struct mlx5_flow_table *
	+mlx5_chains_create_global_table(struct mlx5_fs_chains *chains)
	+{
	+ u32 chain, prio, level;
	+ int err;
	+
	+ if (!mlx5_chains_ignore_flow_level_supported(chains)) {
	+ err = -EOPNOTSUPP;
	+
	+ mlx5_core_warn(chains->dev,
	+ "Couldn't create global flow table, ignore_flow_level not supported.");
	+ goto err_ignore;
	+ }
	+
	+ chain = mlx5_chains_get_chain_range(chains),
	+ prio = mlx5_chains_get_prio_range(chains);
	+ level = mlx5_chains_get_level_range(chains);
	+
	+ return mlx5_chains_create_table(chains, chain, prio, level);
	+
	+err_ignore:
	+ return ERR_PTR(err);
	+}
	+
	+void
	+mlx5_chains_destroy_global_table(struct mlx5_fs_chains *chains,
	+ struct mlx5_flow_table *ft)
	+{
	+ mlx5_destroy_flow_table(ft);
	+}
	+
	+static struct mlx5_fs_chains *
	+mlx5_chains_init(struct mlx5_core_dev dev, struct mlx5_chains_attr attr)
	+{
	+ struct mlx5_fs_chains *chains;
	+
	+ chains = kzalloc(sizeof(*chains), GFP_KERNEL);
	+ if (!chains)
	+ return ERR_PTR(-ENOMEM);
	+
	+ chains->dev = dev;
	+ chains->flags = attr->flags;
	+ chains->ns = attr->ns;
	+ chains->group_num = attr->max_grp_num;
	+ chains->fs_base_prio = attr->fs_base_prio;
	+ chains->fs_base_level = attr->fs_base_level;
	+ chains_default_ft(chains) = chains_end_ft(chains) = attr->default_ft;
	+
	+ xa_init(&chains_xa(chains));
	+ xa_init(&prios_xa(chains));
	+
	+ mutex_init(&chains_lock(chains));
	+
	+ return chains;
	+}
	+
	+static void
	+mlx5_chains_cleanup(struct mlx5_fs_chains *chains)
	+{
	+ mutex_destroy(&chains_lock(chains));
	+ xa_destroy(&prios_xa(chains));
	+ xa_destroy(&chains_xa(chains));
	+
	+ kfree(chains);
	+}
	+
	+struct mlx5_fs_chains *
	+mlx5_chains_create(struct mlx5_core_dev dev, struct mlx5_chains_attr attr)
	+{
	+ struct mlx5_fs_chains *chains;
	+
	+ chains = mlx5_chains_init(dev, attr);
	+
	+ return chains;
	+}
	+
	+void
	+mlx5_chains_destroy(struct mlx5_fs_chains *chains)
	+{
	+ mlx5_chains_cleanup(chains);
	+}
	+
	+void
	+mlx5_chains_print_info(struct mlx5_fs_chains *chains)
	+{
	+ mlx5_core_dbg(chains->dev, "Flow table chains groups(%d)\n", chains->group_num);
	+}
	diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fs_cmd.c b/sys/dev/mlx5/mlx5_core/mlx5_fs_cmd.c
	--- a/sys/dev/mlx5/mlx5_core/mlx5_fs_cmd.c
	+++ b/sys/dev/mlx5/mlx5_core/mlx5_fs_cmd.c
	@@ -1,469 +1,993 @@
	-/*-
	- * Copyright (c) 2013-2017, Mellanox Technologies, Ltd. All rights reserved.
	+/*
	+ * Copyright (c) 2015, Mellanox Technologies. All rights reserved.
	*
	- * Redistribution and use in source and binary forms, with or without
	- * modification, are permitted provided that the following conditions
	- * are met:
	- * 1. Redistributions of source code must retain the above copyright
	- * notice, this list of conditions and the following disclaimer.
	- * 2. Redistributions in binary form must reproduce the above copyright
	- * notice, this list of conditions and the following disclaimer in the
	- * documentation and/or other materials provided with the distribution.
	+ * This software is available to you under a choice of one of two
	+ * licenses. You may choose to be licensed under the terms of the GNU
	+ * General Public License (GPL) Version 2, available from the file
	+ * COPYING in the main directory of this source tree, or the
	+ * OpenIB.org BSD license below:
	*
	- * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
	- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	- * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
	- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	- * SUCH DAMAGE.
	+ * Redistribution and use in source and binary forms, with or
	+ * without modification, are permitted provided that the following
	+ * conditions are met:
	*
	- * $FreeBSD$
	+ * - Redistributions of source code must retain the above
	+ * copyright notice, this list of conditions and the following
	+ * disclaimer.
	+ *
	+ * - Redistributions in binary form must reproduce the above
	+ * copyright notice, this list of conditions and the following
	+ * disclaimer in the documentation and/or other materials
	+ * provided with the distribution.
	+ *
	+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	+ * SOFTWARE.
	*/

	-#include "opt_rss.h"
	-
	-#include <linux/types.h>
	-#include <linux/module.h>
	-#include <dev/mlx5/mlx5_ifc.h>
	+#include <dev/mlx5/driver.h>
	#include <dev/mlx5/device.h>
	-#include <dev/mlx5/fs.h>
	+#include <dev/mlx5/mlx5_ifc.h>

	-#include <dev/mlx5/mlx5_core/fs_core.h>
	-#include <dev/mlx5/mlx5_core/mlx5_core.h>
	+#include "fs_core.h"
	+#include "fs_cmd.h"
	+#include "fs_ft_pool.h"
	+#include "mlx5_core.h"
	+#include "eswitch.h"

	-int mlx5_cmd_update_root_ft(struct mlx5_core_dev *dev,
	- enum fs_ft_type type,
	- unsigned int id)
	+static int mlx5_cmd_stub_update_root_ft(struct mlx5_flow_root_namespace *ns,
	+ struct mlx5_flow_table *ft,
	+ u32 underlay_qpn,
	+ bool disconnect)
	{
	- u32 in[MLX5_ST_SZ_DW(set_flow_table_root_in)] = {0};
	- u32 out[MLX5_ST_SZ_DW(set_flow_table_root_out)] = {0};
	+ return 0;
	+}

	- if (!dev)
	- return -EINVAL;
	+static int mlx5_cmd_stub_create_flow_table(struct mlx5_flow_root_namespace *ns,
	+ struct mlx5_flow_table *ft,
	+ struct mlx5_flow_table_attr *ft_attr,
	+ struct mlx5_flow_table *next_ft)
	+{
	+ int max_fte = ft_attr->max_fte;
	+
	+ ft->max_fte = max_fte ? roundup_pow_of_two(max_fte) : 1;
	+
	+ return 0;
	+}
	+
	+static int mlx5_cmd_stub_destroy_flow_table(struct mlx5_flow_root_namespace *ns,
	+ struct mlx5_flow_table *ft)
	+{
	+ return 0;
	+}
	+
	+static int mlx5_cmd_stub_modify_flow_table(struct mlx5_flow_root_namespace *ns,
	+ struct mlx5_flow_table *ft,
	+ struct mlx5_flow_table *next_ft)
	+{
	+ return 0;
	+}
	+
	+static int mlx5_cmd_stub_create_flow_group(struct mlx5_flow_root_namespace *ns,
	+ struct mlx5_flow_table *ft,
	+ u32 *in,
	+ struct mlx5_flow_group *fg)
	+{
	+ return 0;
	+}
	+
	+static int mlx5_cmd_stub_destroy_flow_group(struct mlx5_flow_root_namespace *ns,
	+ struct mlx5_flow_table *ft,
	+ struct mlx5_flow_group *fg)
	+{
	+ return 0;
	+}
	+
	+static int mlx5_cmd_stub_create_fte(struct mlx5_flow_root_namespace *ns,
	+ struct mlx5_flow_table *ft,
	+ struct mlx5_flow_group *group,
	+ struct fs_fte *fte)
	+{
	+ return 0;
	+}
	+
	+static int mlx5_cmd_stub_update_fte(struct mlx5_flow_root_namespace *ns,
	+ struct mlx5_flow_table *ft,
	+ struct mlx5_flow_group *group,
	+ int modify_mask,
	+ struct fs_fte *fte)
	+{
	+ return -EOPNOTSUPP;
	+}
	+
	+static int mlx5_cmd_stub_delete_fte(struct mlx5_flow_root_namespace *ns,
	+ struct mlx5_flow_table *ft,
	+ struct fs_fte *fte)
	+{
	+ return 0;
	+}
	+
	+static int mlx5_cmd_stub_packet_reformat_alloc(struct mlx5_flow_root_namespace *ns,
	+ struct mlx5_pkt_reformat_params *params,
	+ enum mlx5_flow_namespace_type namespace,
	+ struct mlx5_pkt_reformat *pkt_reformat)
	+{
	+ return 0;
	+}
	+
	+static void mlx5_cmd_stub_packet_reformat_dealloc(struct mlx5_flow_root_namespace *ns,
	+ struct mlx5_pkt_reformat *pkt_reformat)
	+{
	+}
	+
	+static int mlx5_cmd_stub_modify_header_alloc(struct mlx5_flow_root_namespace *ns,
	+ u8 namespace, u8 num_actions,
	+ void *modify_actions,
	+ struct mlx5_modify_hdr *modify_hdr)
	+{
	+ return 0;
	+}
	+
	+static void mlx5_cmd_stub_modify_header_dealloc(struct mlx5_flow_root_namespace *ns,
	+ struct mlx5_modify_hdr *modify_hdr)
	+{
	+}
	+
	+static int mlx5_cmd_stub_set_peer(struct mlx5_flow_root_namespace *ns,
	+ struct mlx5_flow_root_namespace *peer_ns)
	+{
	+ return 0;
	+}
	+
	+static int mlx5_cmd_stub_create_ns(struct mlx5_flow_root_namespace *ns)
	+{
	+ return 0;
	+}
	+
	+static int mlx5_cmd_stub_destroy_ns(struct mlx5_flow_root_namespace *ns)
	+{
	+ return 0;
	+}
	+
	+static u32 mlx5_cmd_stub_get_capabilities(struct mlx5_flow_root_namespace *ns,
	+ enum fs_flow_table_type ft_type)
	+{
	+ return 0;
	+}
	+
	+static int mlx5_cmd_update_root_ft(struct mlx5_flow_root_namespace *ns,
	+ struct mlx5_flow_table *ft, u32 underlay_qpn,
	+ bool disconnect)
	+{
	+ u32 in[MLX5_ST_SZ_DW(set_flow_table_root_in)] = {};
	+ struct mlx5_core_dev *dev = ns->dev;
	+
	+ if ((MLX5_CAP_GEN(dev, port_type) == MLX5_CAP_PORT_TYPE_IB) &&
	+ underlay_qpn == 0)
	+ return 0;

	MLX5_SET(set_flow_table_root_in, in, opcode,
	MLX5_CMD_OP_SET_FLOW_TABLE_ROOT);
	- MLX5_SET(set_flow_table_root_in, in, table_type, type);
	- MLX5_SET(set_flow_table_root_in, in, table_id, id);
	+ MLX5_SET(set_flow_table_root_in, in, table_type, ft->type);
	+
	+ if (disconnect)
	+ MLX5_SET(set_flow_table_root_in, in, op_mod, 1);
	+ else
	+ MLX5_SET(set_flow_table_root_in, in, table_id, ft->id);

	- return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
	+ MLX5_SET(set_flow_table_root_in, in, underlay_qpn, underlay_qpn);
	+ MLX5_SET(set_flow_table_root_in, in, vport_number, ft->vport);
	+ MLX5_SET(set_flow_table_root_in, in, other_vport,
	+ !!(ft->flags & MLX5_FLOW_TABLE_OTHER_VPORT));
	+
	+ return mlx5_cmd_exec_in(dev, set_flow_table_root, in);
	}

	-int mlx5_cmd_fs_create_ft(struct mlx5_core_dev *dev,
	- u16 vport, enum fs_ft_type type, unsigned int level,
	- unsigned int log_size, const char name, unsigned int table_id)
	+static int mlx5_cmd_create_flow_table(struct mlx5_flow_root_namespace *ns,
	+ struct mlx5_flow_table *ft,
	+ struct mlx5_flow_table_attr *ft_attr,
	+ struct mlx5_flow_table *next_ft)
	{
	- u32 in[MLX5_ST_SZ_DW(create_flow_table_in)] = {0};
	- u32 out[MLX5_ST_SZ_DW(create_flow_table_out)] = {0};
	+ int en_encap = !!(ft->flags & MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT);
	+ int en_decap = !!(ft->flags & MLX5_FLOW_TABLE_TUNNEL_EN_DECAP);
	+ int term = !!(ft->flags & MLX5_FLOW_TABLE_TERMINATION);
	+ u32 out[MLX5_ST_SZ_DW(create_flow_table_out)] = {};
	+ u32 in[MLX5_ST_SZ_DW(create_flow_table_in)] = {};
	+ struct mlx5_core_dev *dev = ns->dev;
	+ unsigned int size;
	int err;

	- if (!dev)
	- return -EINVAL;
	+ size = mlx5_ft_pool_get_avail_sz(dev, ft->type, ft_attr->max_fte);
	+ if (!size)
	+ return -ENOSPC;

	MLX5_SET(create_flow_table_in, in, opcode,
	MLX5_CMD_OP_CREATE_FLOW_TABLE);

	- MLX5_SET(create_flow_table_in, in, table_type, type);
	- MLX5_SET(create_flow_table_in, in, flow_table_context.level, level);
	- MLX5_SET(create_flow_table_in, in, flow_table_context.log_size,
	- log_size);
	- if (strstr(name, FS_REFORMAT_KEYWORD) != NULL)
	- MLX5_SET(create_flow_table_in, in,
	- flow_table_context.reformat_en, 1);
	- if (vport) {
	- MLX5_SET(create_flow_table_in, in, vport_number, vport);
	- MLX5_SET(create_flow_table_in, in, other_vport, 1);
	+ MLX5_SET(create_flow_table_in, in, uid, ft_attr->uid);
	+ MLX5_SET(create_flow_table_in, in, table_type, ft->type);
	+ MLX5_SET(create_flow_table_in, in, flow_table_context.level, ft->level);
	+ MLX5_SET(create_flow_table_in, in, flow_table_context.log_size, size ? ilog2(size) : 0);
	+ MLX5_SET(create_flow_table_in, in, vport_number, ft->vport);
	+ MLX5_SET(create_flow_table_in, in, other_vport,
	+ !!(ft->flags & MLX5_FLOW_TABLE_OTHER_VPORT));
	+
	+ MLX5_SET(create_flow_table_in, in, flow_table_context.decap_en,
	+ en_decap);
	+ MLX5_SET(create_flow_table_in, in, flow_table_context.reformat_en,
	+ en_encap);
	+ MLX5_SET(create_flow_table_in, in, flow_table_context.termination_table,
	+ term);
	+
	+ switch (ft->op_mod) {
	+ case FS_FT_OP_MOD_NORMAL:
	+ if (next_ft) {
	+ MLX5_SET(create_flow_table_in, in,
	+ flow_table_context.table_miss_action,
	+ MLX5_FLOW_TABLE_MISS_ACTION_FWD);
	+ MLX5_SET(create_flow_table_in, in,
	+ flow_table_context.table_miss_id, next_ft->id);
	+ } else {
	+ MLX5_SET(create_flow_table_in, in,
	+ flow_table_context.table_miss_action,
	+ ft->def_miss_action);
	+ }
	+ break;
	+
	+ case FS_FT_OP_MOD_LAG_DEMUX:
	+ MLX5_SET(create_flow_table_in, in, op_mod, 0x1);
	+ if (next_ft)
	+ MLX5_SET(create_flow_table_in, in,
	+ flow_table_context.lag_master_next_table_id,
	+ next_ft->id);
	+ break;
	}

	- err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
	- if (!err)
	- *table_id = MLX5_GET(create_flow_table_out, out, table_id);
	+ err = mlx5_cmd_exec_inout(dev, create_flow_table, in, out);
	+ if (!err) {
	+ ft->id = MLX5_GET(create_flow_table_out, out,
	+ table_id);
	+ ft->max_fte = size;
	+ } else {
	+ mlx5_ft_pool_put_sz(ns->dev, size);
	+ }

	return err;
	}

	-int mlx5_cmd_fs_destroy_ft(struct mlx5_core_dev *dev,
	- u16 vport,
	- enum fs_ft_type type, unsigned int table_id)
	+static int mlx5_cmd_destroy_flow_table(struct mlx5_flow_root_namespace *ns,
	+ struct mlx5_flow_table *ft)
	{
	- u32 in[MLX5_ST_SZ_DW(destroy_flow_table_in)] = {0};
	- u32 out[MLX5_ST_SZ_DW(destroy_flow_table_out)] = {0};
	-
	- if (!dev)
	- return -EINVAL;
	+ u32 in[MLX5_ST_SZ_DW(destroy_flow_table_in)] = {};
	+ struct mlx5_core_dev *dev = ns->dev;
	+ int err;

	MLX5_SET(destroy_flow_table_in, in, opcode,
	MLX5_CMD_OP_DESTROY_FLOW_TABLE);
	- MLX5_SET(destroy_flow_table_in, in, table_type, type);
	- MLX5_SET(destroy_flow_table_in, in, table_id, table_id);
	- if (vport) {
	- MLX5_SET(destroy_flow_table_in, in, vport_number, vport);
	- MLX5_SET(destroy_flow_table_in, in, other_vport, 1);
	+ MLX5_SET(destroy_flow_table_in, in, table_type, ft->type);
	+ MLX5_SET(destroy_flow_table_in, in, table_id, ft->id);
	+ MLX5_SET(destroy_flow_table_in, in, vport_number, ft->vport);
	+ MLX5_SET(destroy_flow_table_in, in, other_vport,
	+ !!(ft->flags & MLX5_FLOW_TABLE_OTHER_VPORT));
	+
	+ err = mlx5_cmd_exec_in(dev, destroy_flow_table, in);
	+ if (!err)
	+ mlx5_ft_pool_put_sz(ns->dev, ft->max_fte);
	+
	+ return err;
	+}
	+
	+static int mlx5_cmd_modify_flow_table(struct mlx5_flow_root_namespace *ns,
	+ struct mlx5_flow_table *ft,
	+ struct mlx5_flow_table *next_ft)
	+{
	+ u32 in[MLX5_ST_SZ_DW(modify_flow_table_in)] = {};
	+ struct mlx5_core_dev *dev = ns->dev;
	+
	+ MLX5_SET(modify_flow_table_in, in, opcode,
	+ MLX5_CMD_OP_MODIFY_FLOW_TABLE);
	+ MLX5_SET(modify_flow_table_in, in, table_type, ft->type);
	+ MLX5_SET(modify_flow_table_in, in, table_id, ft->id);
	+
	+ if (ft->op_mod == FS_FT_OP_MOD_LAG_DEMUX) {
	+ MLX5_SET(modify_flow_table_in, in, modify_field_select,
	+ MLX5_MODIFY_FLOW_TABLE_LAG_NEXT_TABLE_ID);
	+ if (next_ft) {
	+ MLX5_SET(modify_flow_table_in, in,
	+ flow_table_context.lag_master_next_table_id, next_ft->id);
	+ } else {
	+ MLX5_SET(modify_flow_table_in, in,
	+ flow_table_context.lag_master_next_table_id, 0);
	+ }
	+ } else {
	+ MLX5_SET(modify_flow_table_in, in, vport_number, ft->vport);
	+ MLX5_SET(modify_flow_table_in, in, other_vport,
	+ !!(ft->flags & MLX5_FLOW_TABLE_OTHER_VPORT));
	+ MLX5_SET(modify_flow_table_in, in, modify_field_select,
	+ MLX5_MODIFY_FLOW_TABLE_MISS_TABLE_ID);
	+ if (next_ft) {
	+ MLX5_SET(modify_flow_table_in, in,
	+ flow_table_context.table_miss_action,
	+ MLX5_FLOW_TABLE_MISS_ACTION_FWD);
	+ MLX5_SET(modify_flow_table_in, in,
	+ flow_table_context.table_miss_id,
	+ next_ft->id);
	+ } else {
	+ MLX5_SET(modify_flow_table_in, in,
	+ flow_table_context.table_miss_action,
	+ ft->def_miss_action);
	+ }
	}

	- return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
	+ return mlx5_cmd_exec_in(dev, modify_flow_table, in);
	}

	-int mlx5_cmd_fs_create_fg(struct mlx5_core_dev *dev,
	- u32 *in,
	- u16 vport,
	- enum fs_ft_type type, unsigned int table_id,
	- unsigned int *group_id)
	+static int mlx5_cmd_create_flow_group(struct mlx5_flow_root_namespace *ns,
	+ struct mlx5_flow_table *ft,
	+ u32 *in,
	+ struct mlx5_flow_group *fg)
	{
	- u32 out[MLX5_ST_SZ_DW(create_flow_group_out)] = {0};
	+ u32 out[MLX5_ST_SZ_DW(create_flow_group_out)] = {};
	+ struct mlx5_core_dev *dev = ns->dev;
	int err;
	- int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
	- if (!dev)
	- return -EINVAL;

	MLX5_SET(create_flow_group_in, in, opcode,
	MLX5_CMD_OP_CREATE_FLOW_GROUP);
	- MLX5_SET(create_flow_group_in, in, table_type, type);
	- MLX5_SET(create_flow_group_in, in, table_id, table_id);
	- if (vport) {
	- MLX5_SET(create_flow_group_in, in, vport_number, vport);
	- MLX5_SET(create_flow_group_in, in, other_vport, 1);
	- }
	-
	- err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
	+ MLX5_SET(create_flow_group_in, in, table_type, ft->type);
	+ MLX5_SET(create_flow_group_in, in, table_id, ft->id);
	+ MLX5_SET(create_flow_group_in, in, vport_number, ft->vport);
	+ MLX5_SET(create_flow_group_in, in, other_vport,
	+ !!(ft->flags & MLX5_FLOW_TABLE_OTHER_VPORT));
	+ err = mlx5_cmd_exec_inout(dev, create_flow_group, in, out);
	if (!err)
	- *group_id = MLX5_GET(create_flow_group_out, out, group_id);
	-
	+ fg->id = MLX5_GET(create_flow_group_out, out,
	+ group_id);
	return err;
	}

	-int mlx5_cmd_fs_destroy_fg(struct mlx5_core_dev *dev,
	- u16 vport,
	- enum fs_ft_type type, unsigned int table_id,
	- unsigned int group_id)
	+static int mlx5_cmd_destroy_flow_group(struct mlx5_flow_root_namespace *ns,
	+ struct mlx5_flow_table *ft,
	+ struct mlx5_flow_group *fg)
	{
	- u32 in[MLX5_ST_SZ_DW(destroy_flow_group_in)] = {0};
	- u32 out[MLX5_ST_SZ_DW(destroy_flow_group_out)] = {0};
	-
	- if (!dev)
	- return -EINVAL;
	+ u32 in[MLX5_ST_SZ_DW(destroy_flow_group_in)] = {};
	+ struct mlx5_core_dev *dev = ns->dev;

	MLX5_SET(destroy_flow_group_in, in, opcode,
	MLX5_CMD_OP_DESTROY_FLOW_GROUP);
	- MLX5_SET(destroy_flow_group_in, in, table_type, type);
	- MLX5_SET(destroy_flow_group_in, in, table_id, table_id);
	- MLX5_SET(destroy_flow_group_in, in, group_id, group_id);
	- if (vport) {
	- MLX5_SET(destroy_flow_group_in, in, vport_number, vport);
	- MLX5_SET(destroy_flow_group_in, in, other_vport, 1);
	- }
	-
	- return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
	+ MLX5_SET(destroy_flow_group_in, in, table_type, ft->type);
	+ MLX5_SET(destroy_flow_group_in, in, table_id, ft->id);
	+ MLX5_SET(destroy_flow_group_in, in, group_id, fg->id);
	+ MLX5_SET(destroy_flow_group_in, in, vport_number, ft->vport);
	+ MLX5_SET(destroy_flow_group_in, in, other_vport,
	+ !!(ft->flags & MLX5_FLOW_TABLE_OTHER_VPORT));
	+ return mlx5_cmd_exec_in(dev, destroy_flow_group, in);
	}

	-int mlx5_cmd_fs_set_fte(struct mlx5_core_dev *dev,
	- u16 vport,
	- enum fs_fte_status *fte_status,
	- u32 *match_val,
	- enum fs_ft_type type, unsigned int table_id,
	- unsigned int index, unsigned int group_id,
	- struct mlx5_flow_act *flow_act,
	- u32 sw_action, int dest_size,
	- struct list_head dests) / mlx5_flow_desination */
	+static int mlx5_set_extended_dest(struct mlx5_core_dev *dev,
	+ struct fs_fte fte, bool extended_dest)
	{
	- u32 out[MLX5_ST_SZ_DW(set_fte_out)] = {0};
	- u32 *in;
	- unsigned int inlen;
	+ int fw_log_max_fdb_encap_uplink =
	+ MLX5_CAP_ESW(dev, log_max_fdb_encap_uplink);
	+ int num_fwd_destinations = 0;
	struct mlx5_flow_rule *dst;
	- void *in_flow_context;
	- void *in_match_value;
	- void *in_dests;
	- int err;
	- int opmod = 0;
	- int modify_mask = 0;
	- int atomic_mod_cap;
	- u32 prm_action = 0;
	- int count_list = 0;
	+ int num_encap = 0;

	- if (sw_action != MLX5_FLOW_RULE_FWD_ACTION_DEST)
	- dest_size = 0;
	+ *extended_dest = false;
	+ if (!(fte->action.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST))
	+ return 0;

	- if (sw_action & MLX5_FLOW_RULE_FWD_ACTION_ALLOW)
	- prm_action \|= MLX5_FLOW_CONTEXT_ACTION_ALLOW;
	+ list_for_each_entry(dst, &fte->node.children, node.list) {
	+ if (dst->dest_attr.type == MLX5_FLOW_DESTINATION_TYPE_COUNTER \|\|
	+ dst->dest_attr.type == MLX5_FLOW_DESTINATION_TYPE_NONE)
	+ continue;
	+ if ((dst->dest_attr.type == MLX5_FLOW_DESTINATION_TYPE_VPORT \|\|
	+ dst->dest_attr.type == MLX5_FLOW_DESTINATION_TYPE_UPLINK) &&
	+ dst->dest_attr.vport.flags & MLX5_FLOW_DEST_VPORT_REFORMAT_ID)
	+ num_encap++;
	+ num_fwd_destinations++;
	+ }
	+ if (num_fwd_destinations > 1 && num_encap > 0)
	+ *extended_dest = true;

	- if (sw_action & MLX5_FLOW_RULE_FWD_ACTION_DROP)
	- prm_action \|= MLX5_FLOW_CONTEXT_ACTION_DROP;
	+ if (*extended_dest && !fw_log_max_fdb_encap_uplink) {
	+ mlx5_core_warn(dev, "FW does not support extended destination");
	+ return -EOPNOTSUPP;
	+ }
	+ if (num_encap > (1 << fw_log_max_fdb_encap_uplink)) {
	+ mlx5_core_warn(dev, "FW does not support more than %d encaps",
	+ 1 << fw_log_max_fdb_encap_uplink);
	+ return -EOPNOTSUPP;
	+ }

	- if (sw_action & MLX5_FLOW_RULE_FWD_ACTION_DEST)
	- prm_action \|= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
	+ return 0;
	+}

	- if (flow_act->actions & MLX5_FLOW_ACT_ACTIONS_COUNT) {
	- prm_action \|= MLX5_FLOW_CONTEXT_ACTION_COUNT;
	- count_list = 1;
	- }
	+static void
	+mlx5_cmd_set_fte_flow_meter(struct fs_fte fte, void in_flow_context)
	+{
	+ void *exe_aso_ctrl;
	+ void *execute_aso;
	+
	+ execute_aso = MLX5_ADDR_OF(flow_context, in_flow_context,
	+ execute_aso[0]);
	+ MLX5_SET(execute_aso, execute_aso, valid, 1);
	+ MLX5_SET(execute_aso, execute_aso, aso_object_id,
	+ fte->action.exe_aso.object_id);
	+
	+ exe_aso_ctrl = MLX5_ADDR_OF(execute_aso, execute_aso, exe_aso_ctrl);
	+ MLX5_SET(exe_aso_ctrl_flow_meter, exe_aso_ctrl, return_reg_id,
	+ fte->action.exe_aso.return_reg_id);
	+ MLX5_SET(exe_aso_ctrl_flow_meter, exe_aso_ctrl, aso_type,
	+ fte->action.exe_aso.type);
	+ MLX5_SET(exe_aso_ctrl_flow_meter, exe_aso_ctrl, init_color,
	+ fte->action.exe_aso.flow_meter.init_color);
	+ MLX5_SET(exe_aso_ctrl_flow_meter, exe_aso_ctrl, meter_id,
	+ fte->action.exe_aso.flow_meter.meter_idx);
	+}

	- inlen = MLX5_ST_SZ_BYTES(set_fte_in) +
	- (dest_size + count_list) * MLX5_ST_SZ_BYTES(dest_format_struct);
	+static int mlx5_cmd_set_fte(struct mlx5_core_dev *dev,
	+ int opmod, int modify_mask,
	+ struct mlx5_flow_table *ft,
	+ unsigned group_id,
	+ struct fs_fte *fte)
	+{
	+ u32 out[MLX5_ST_SZ_DW(set_fte_out)] = {0};
	+ bool extended_dest = false;
	+ struct mlx5_flow_rule *dst;
	+ void in_flow_context, vlan;
	+ void *in_match_value;
	+ unsigned int inlen;
	+ int dst_cnt_size;
	+ void *in_dests;
	+ u32 *in;
	+ int err;

	- if (!dev)
	- return -EINVAL;
	+ if (mlx5_set_extended_dest(dev, fte, &extended_dest))
	+ return -EOPNOTSUPP;

	- if (*fte_status & FS_FTE_STATUS_EXISTING) {
	- atomic_mod_cap = MLX5_CAP_FLOWTABLE(dev,
	- flow_table_properties_nic_receive.
	- flow_modify_en);
	- if (!atomic_mod_cap)
	- return -ENOTSUPP;
	- opmod = 1;
	- modify_mask = 1 <<
	- MLX5_SET_FTE_MODIFY_ENABLE_MASK_DESTINATION_LIST;
	- }
	+ if (!extended_dest)
	+ dst_cnt_size = MLX5_ST_SZ_BYTES(dest_format_struct);
	+ else
	+ dst_cnt_size = MLX5_ST_SZ_BYTES(extended_dest_format);

	- in = mlx5_vzalloc(inlen);
	- if (!in) {
	- mlx5_core_warn(dev, "failed to allocate inbox\n");
	+ inlen = MLX5_ST_SZ_BYTES(set_fte_in) + fte->dests_size * dst_cnt_size;
	+ in = kvzalloc(inlen, GFP_KERNEL);
	+ if (!in)
	return -ENOMEM;
	- }

	MLX5_SET(set_fte_in, in, opcode, MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY);
	MLX5_SET(set_fte_in, in, op_mod, opmod);
	MLX5_SET(set_fte_in, in, modify_enable_mask, modify_mask);
	- MLX5_SET(set_fte_in, in, table_type, type);
	- MLX5_SET(set_fte_in, in, table_id, table_id);
	- MLX5_SET(set_fte_in, in, flow_index, index);
	- if (vport) {
	- MLX5_SET(set_fte_in, in, vport_number, vport);
	- MLX5_SET(set_fte_in, in, other_vport, 1);
	- }
	+ MLX5_SET(set_fte_in, in, table_type, ft->type);
	+ MLX5_SET(set_fte_in, in, table_id, ft->id);
	+ MLX5_SET(set_fte_in, in, flow_index, fte->index);
	+ MLX5_SET(set_fte_in, in, ignore_flow_level,
	+ !!(fte->action.flags & FLOW_ACT_IGNORE_FLOW_LEVEL));
	+
	+ MLX5_SET(set_fte_in, in, vport_number, ft->vport);
	+ MLX5_SET(set_fte_in, in, other_vport,
	+ !!(ft->flags & MLX5_FLOW_TABLE_OTHER_VPORT));

	in_flow_context = MLX5_ADDR_OF(set_fte_in, in, flow_context);
	MLX5_SET(flow_context, in_flow_context, group_id, group_id);
	- if (flow_act->actions & MLX5_FLOW_ACT_ACTIONS_FLOW_TAG)
	- MLX5_SET(flow_context, in_flow_context, flow_tag, flow_act->flow_tag);
	- if (flow_act->actions & MLX5_FLOW_ACT_ACTIONS_MODIFY_HDR) {
	- MLX5_SET(flow_context, in_flow_context, modify_header_id,
	- flow_act->modify_hdr->id);
	- prm_action \|= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
	- }
	- if (flow_act->actions & MLX5_FLOW_ACT_ACTIONS_PACKET_REFORMAT) {
	- MLX5_SET(flow_context, in_flow_context, packet_reformat_id,
	- flow_act->pkt_reformat->id);
	- prm_action \|= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
	+
	+ MLX5_SET(flow_context, in_flow_context, flow_tag,
	+ fte->flow_context.flow_tag);
	+ MLX5_SET(flow_context, in_flow_context, flow_source,
	+ fte->flow_context.flow_source);
	+
	+ MLX5_SET(flow_context, in_flow_context, extended_destination,
	+ extended_dest);
	+ if (extended_dest) {
	+ u32 action;
	+
	+ action = fte->action.action &
	+ ~MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
	+ MLX5_SET(flow_context, in_flow_context, action, action);
	+ } else {
	+ MLX5_SET(flow_context, in_flow_context, action,
	+ fte->action.action);
	+ if (fte->action.pkt_reformat)
	+ MLX5_SET(flow_context, in_flow_context, packet_reformat_id,
	+ fte->action.pkt_reformat->id);
	}
	- MLX5_SET(flow_context, in_flow_context, destination_list_size,
	- dest_size);
	+ if (fte->action.modify_hdr)
	+ MLX5_SET(flow_context, in_flow_context, modify_header_id,
	+ fte->action.modify_hdr->id);
	+
	+ MLX5_SET(flow_context, in_flow_context, encrypt_decrypt_type,
	+ fte->action.crypto.type);
	+ MLX5_SET(flow_context, in_flow_context, encrypt_decrypt_obj_id,
	+ fte->action.crypto.obj_id);
	+
	+ vlan = MLX5_ADDR_OF(flow_context, in_flow_context, push_vlan);
	+
	+ MLX5_SET(vlan, vlan, ethtype, fte->action.vlan[0].ethtype);
	+ MLX5_SET(vlan, vlan, vid, fte->action.vlan[0].vid);
	+ MLX5_SET(vlan, vlan, prio, fte->action.vlan[0].prio);
	+
	+ vlan = MLX5_ADDR_OF(flow_context, in_flow_context, push_vlan_2);
	+
	+ MLX5_SET(vlan, vlan, ethtype, fte->action.vlan[1].ethtype);
	+ MLX5_SET(vlan, vlan, vid, fte->action.vlan[1].vid);
	+ MLX5_SET(vlan, vlan, prio, fte->action.vlan[1].prio);
	+
	in_match_value = MLX5_ADDR_OF(flow_context, in_flow_context,
	match_value);
	- memcpy(in_match_value, match_val, MLX5_ST_SZ_BYTES(fte_match_param));
	+ memcpy(in_match_value, &fte->val, sizeof(fte->val));

	in_dests = MLX5_ADDR_OF(flow_context, in_flow_context, destination);
	+ if (fte->action.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) {
	+ int list_size = 0;

	- if (dest_size) {
	- list_for_each_entry(dst, dests, base.list) {
	+ list_for_each_entry(dst, &fte->node.children, node.list) {
	+ enum mlx5_flow_destination_type type = dst->dest_attr.type;
	+ enum mlx5_ifc_flow_destination_type ifc_type;
	unsigned int id;

	- MLX5_SET(dest_format_struct, in_dests, destination_type,
	- dst->dest_attr.type);
	- if (dst->dest_attr.type ==
	- MLX5_FLOW_CONTEXT_DEST_TYPE_FLOW_TABLE)
	+ if (type == MLX5_FLOW_DESTINATION_TYPE_COUNTER)
	+ continue;
	+
	+ switch (type) {
	+ case MLX5_FLOW_DESTINATION_TYPE_NONE:
	+ continue;
	+ case MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM:
	+ id = dst->dest_attr.ft_num;
	+ ifc_type = MLX5_IFC_FLOW_DESTINATION_TYPE_FLOW_TABLE;
	+ break;
	+ case MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE:
	id = dst->dest_attr.ft->id;
	- else
	+ ifc_type = MLX5_IFC_FLOW_DESTINATION_TYPE_FLOW_TABLE;
	+ break;
	+ case MLX5_FLOW_DESTINATION_TYPE_UPLINK:
	+ case MLX5_FLOW_DESTINATION_TYPE_VPORT:
	+ MLX5_SET(dest_format_struct, in_dests,
	+ destination_eswitch_owner_vhca_id_valid,
	+ !!(dst->dest_attr.vport.flags &
	+ MLX5_FLOW_DEST_VPORT_VHCA_ID));
	+ MLX5_SET(dest_format_struct, in_dests,
	+ destination_eswitch_owner_vhca_id,
	+ dst->dest_attr.vport.vhca_id);
	+ if (type == MLX5_FLOW_DESTINATION_TYPE_UPLINK) {
	+ /* destination_id is reserved */
	+ id = 0;
	+ ifc_type = MLX5_IFC_FLOW_DESTINATION_TYPE_UPLINK;
	+ break;
	+ }
	+ ifc_type = MLX5_IFC_FLOW_DESTINATION_TYPE_VPORT;
	+ id = dst->dest_attr.vport.num;
	+ if (extended_dest &&
	+ dst->dest_attr.vport.pkt_reformat) {
	+ MLX5_SET(dest_format_struct, in_dests,
	+ packet_reformat,
	+ !!(dst->dest_attr.vport.flags &
	+ MLX5_FLOW_DEST_VPORT_REFORMAT_ID));
	+ MLX5_SET(extended_dest_format, in_dests,
	+ packet_reformat_id,
	+ dst->dest_attr.vport.pkt_reformat->id);
	+ }
	+ break;
	+ case MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER:
	+ id = dst->dest_attr.sampler_id;
	+ ifc_type = MLX5_IFC_FLOW_DESTINATION_TYPE_FLOW_SAMPLER;
	+ break;
	+ case MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE:
	+ MLX5_SET(dest_format_struct, in_dests,
	+ destination_table_type, dst->dest_attr.ft->type);
	+ id = dst->dest_attr.ft->id;
	+ ifc_type = MLX5_IFC_FLOW_DESTINATION_TYPE_TABLE_TYPE;
	+ break;
	+ default:
	id = dst->dest_attr.tir_num;
	+ ifc_type = MLX5_IFC_FLOW_DESTINATION_TYPE_TIR;
	+ }
	+
	+ MLX5_SET(dest_format_struct, in_dests, destination_type,
	+ ifc_type);
	MLX5_SET(dest_format_struct, in_dests, destination_id, id);
	- in_dests += MLX5_ST_SZ_BYTES(dest_format_struct);
	+ in_dests += dst_cnt_size;
	+ list_size++;
	}
	+
	+ MLX5_SET(flow_context, in_flow_context, destination_list_size,
	+ list_size);
	}

	- if (flow_act->actions & MLX5_FLOW_ACT_ACTIONS_COUNT) {
	- MLX5_SET(dest_format_struct, in_dests, destination_id,
	- mlx5_fc_id(flow_act->counter));
	- in_dests += MLX5_ST_SZ_BYTES(dest_format_struct);
	- MLX5_SET(flow_context, in_flow_context, flow_counter_list_size, 1);
	+ if (fte->action.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
	+ int max_list_size = BIT(MLX5_CAP_FLOWTABLE_TYPE(dev,
	+ log_max_flow_counter,
	+ ft->type));
	+ int list_size = 0;
	+
	+ list_for_each_entry(dst, &fte->node.children, node.list) {
	+ if (dst->dest_attr.type !=
	+ MLX5_FLOW_DESTINATION_TYPE_COUNTER)
	+ continue;
	+
	+ MLX5_SET(flow_counter_list, in_dests, flow_counter_id,
	+ dst->dest_attr.counter_id);
	+ in_dests += dst_cnt_size;
	+ list_size++;
	+ }
	+ if (list_size > max_list_size) {
	+ err = -EINVAL;
	+ goto err_out;
	+ }
	+
	+ MLX5_SET(flow_context, in_flow_context, flow_counter_list_size,
	+ list_size);
	}

	- MLX5_SET(flow_context, in_flow_context, action, prm_action);
	- err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
	- if (!err)
	- *fte_status \|= FS_FTE_STATUS_EXISTING;
	+ if (fte->action.action & MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO) {
	+ if (fte->action.exe_aso.type == MLX5_EXE_ASO_FLOW_METER) {
	+ mlx5_cmd_set_fte_flow_meter(fte, in_flow_context);
	+ } else {
	+ err = -EOPNOTSUPP;
	+ goto err_out;
	+ }
	+ }

	+ err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
	+err_out:
	kvfree(in);
	+ return err;
	+}
	+
	+static int mlx5_cmd_create_fte(struct mlx5_flow_root_namespace *ns,
	+ struct mlx5_flow_table *ft,
	+ struct mlx5_flow_group *group,
	+ struct fs_fte *fte)
	+{
	+ struct mlx5_core_dev *dev = ns->dev;
	+ unsigned int group_id = group->id;
	+
	+ return mlx5_cmd_set_fte(dev, 0, 0, ft, group_id, fte);
	+}
	+
	+static int mlx5_cmd_update_fte(struct mlx5_flow_root_namespace *ns,
	+ struct mlx5_flow_table *ft,
	+ struct mlx5_flow_group *fg,
	+ int modify_mask,
	+ struct fs_fte *fte)
	+{
	+ int opmod;
	+ struct mlx5_core_dev *dev = ns->dev;
	+ int atomic_mod_cap = MLX5_CAP_FLOWTABLE(dev,
	+ flow_table_properties_nic_receive.
	+ flow_modify_en);
	+ if (!atomic_mod_cap)
	+ return -EOPNOTSUPP;
	+ opmod = 1;
	+
	+ return mlx5_cmd_set_fte(dev, opmod, modify_mask, ft, fg->id, fte);
	+}
	+
	+static int mlx5_cmd_delete_fte(struct mlx5_flow_root_namespace *ns,
	+ struct mlx5_flow_table *ft,
	+ struct fs_fte *fte)
	+{
	+ u32 in[MLX5_ST_SZ_DW(delete_fte_in)] = {};
	+ struct mlx5_core_dev *dev = ns->dev;

	+ MLX5_SET(delete_fte_in, in, opcode, MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY);
	+ MLX5_SET(delete_fte_in, in, table_type, ft->type);
	+ MLX5_SET(delete_fte_in, in, table_id, ft->id);
	+ MLX5_SET(delete_fte_in, in, flow_index, fte->index);
	+ MLX5_SET(delete_fte_in, in, vport_number, ft->vport);
	+ MLX5_SET(delete_fte_in, in, other_vport,
	+ !!(ft->flags & MLX5_FLOW_TABLE_OTHER_VPORT));
	+
	+ return mlx5_cmd_exec_in(dev, delete_fte, in);
	+}
	+
	+int mlx5_cmd_fc_bulk_alloc(struct mlx5_core_dev *dev,
	+ enum mlx5_fc_bulk_alloc_bitmask alloc_bitmask,
	+ u32 *id)
	+{
	+ u32 out[MLX5_ST_SZ_DW(alloc_flow_counter_out)] = {};
	+ u32 in[MLX5_ST_SZ_DW(alloc_flow_counter_in)] = {};
	+ int err;
	+
	+ MLX5_SET(alloc_flow_counter_in, in, opcode,
	+ MLX5_CMD_OP_ALLOC_FLOW_COUNTER);
	+ MLX5_SET(alloc_flow_counter_in, in, flow_counter_bulk, alloc_bitmask);
	+
	+ err = mlx5_cmd_exec_inout(dev, alloc_flow_counter, in, out);
	+ if (!err)
	+ *id = MLX5_GET(alloc_flow_counter_out, out, flow_counter_id);
	return err;
	}

	-int mlx5_cmd_fs_delete_fte(struct mlx5_core_dev *dev,
	- u16 vport,
	- enum fs_fte_status *fte_status,
	- enum fs_ft_type type, unsigned int table_id,
	- unsigned int index)
	+int mlx5_cmd_fc_alloc(struct mlx5_core_dev dev, u32 id)
	+{
	+ return mlx5_cmd_fc_bulk_alloc(dev, 0, id);
	+}
	+
	+int mlx5_cmd_fc_free(struct mlx5_core_dev *dev, u32 id)
	+{
	+ u32 in[MLX5_ST_SZ_DW(dealloc_flow_counter_in)] = {};
	+
	+ MLX5_SET(dealloc_flow_counter_in, in, opcode,
	+ MLX5_CMD_OP_DEALLOC_FLOW_COUNTER);
	+ MLX5_SET(dealloc_flow_counter_in, in, flow_counter_id, id);
	+ return mlx5_cmd_exec_in(dev, dealloc_flow_counter, in);
	+}
	+
	+int mlx5_cmd_fc_query(struct mlx5_core_dev *dev, u32 id,
	+ u64 packets, u64 bytes)
	+{
	+ u32 out[MLX5_ST_SZ_BYTES(query_flow_counter_out) +
	+ MLX5_ST_SZ_BYTES(traffic_counter)] = {};
	+ u32 in[MLX5_ST_SZ_DW(query_flow_counter_in)] = {};
	+ void *stats;
	+ int err = 0;
	+
	+ MLX5_SET(query_flow_counter_in, in, opcode,
	+ MLX5_CMD_OP_QUERY_FLOW_COUNTER);
	+ MLX5_SET(query_flow_counter_in, in, op_mod, 0);
	+ MLX5_SET(query_flow_counter_in, in, flow_counter_id, id);
	+ err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
	+ if (err)
	+ return err;
	+
	+ stats = MLX5_ADDR_OF(query_flow_counter_out, out, flow_statistics);
	+ *packets = MLX5_GET64(traffic_counter, stats, packets);
	+ *bytes = MLX5_GET64(traffic_counter, stats, octets);
	+ return 0;
	+}
	+
	+int mlx5_cmd_fc_get_bulk_query_out_len(int bulk_len)
	{
	- u32 in[MLX5_ST_SZ_DW(delete_fte_in)] = {0};
	- u32 out[MLX5_ST_SZ_DW(delete_fte_out)] = {0};
	+ return MLX5_ST_SZ_BYTES(query_flow_counter_out) +
	+ MLX5_ST_SZ_BYTES(traffic_counter) * bulk_len;
	+}
	+
	+int mlx5_cmd_fc_bulk_query(struct mlx5_core_dev *dev, u32 base_id, int bulk_len,
	+ u32 *out)
	+{
	+ int outlen = mlx5_cmd_fc_get_bulk_query_out_len(bulk_len);
	+ u32 in[MLX5_ST_SZ_DW(query_flow_counter_in)] = {};
	+
	+ MLX5_SET(query_flow_counter_in, in, opcode,
	+ MLX5_CMD_OP_QUERY_FLOW_COUNTER);
	+ MLX5_SET(query_flow_counter_in, in, flow_counter_id, base_id);
	+ MLX5_SET(query_flow_counter_in, in, num_of_counters, bulk_len);
	+ return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen);
	+}
	+
	+static int mlx5_cmd_packet_reformat_alloc(struct mlx5_flow_root_namespace *ns,
	+ struct mlx5_pkt_reformat_params *params,
	+ enum mlx5_flow_namespace_type namespace,
	+ struct mlx5_pkt_reformat *pkt_reformat)
	+{
	+ u32 out[MLX5_ST_SZ_DW(alloc_packet_reformat_context_out)] = {};
	+ struct mlx5_core_dev *dev = ns->dev;
	+ void *packet_reformat_context_in;
	+ int max_encap_size;
	+ void *reformat;
	+ int inlen;
	int err;
	+ u32 *in;

	- if (!(*fte_status & FS_FTE_STATUS_EXISTING))
	- return 0;
	+ if (namespace == MLX5_FLOW_NAMESPACE_FDB \|\|
	+ namespace == MLX5_FLOW_NAMESPACE_FDB_BYPASS)
	+ max_encap_size = MLX5_CAP_ESW(dev, max_encap_header_size);
	+ else
	+ max_encap_size = MLX5_CAP_FLOWTABLE(dev, max_encap_header_size);

	- if (!dev)
	+ if (params->size > max_encap_size) {
	+ mlx5_core_warn(dev, "encap size %zd too big, max supported is %d\n",
	+ params->size, max_encap_size);
	return -EINVAL;
	+ }

	- MLX5_SET(delete_fte_in, in, opcode, MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY);
	- MLX5_SET(delete_fte_in, in, table_type, type);
	- MLX5_SET(delete_fte_in, in, table_id, table_id);
	- MLX5_SET(delete_fte_in, in, flow_index, index);
	- if (vport) {
	- MLX5_SET(delete_fte_in, in, vport_number, vport);
	- MLX5_SET(delete_fte_in, in, other_vport, 1);
	+ in = kzalloc(MLX5_ST_SZ_BYTES(alloc_packet_reformat_context_in) +
	+ params->size, GFP_KERNEL);
	+ if (!in)
	+ return -ENOMEM;
	+
	+ packet_reformat_context_in = MLX5_ADDR_OF(alloc_packet_reformat_context_in,
	+ in, packet_reformat_context);
	+ reformat = MLX5_ADDR_OF(packet_reformat_context_in,
	+ packet_reformat_context_in,
	+ reformat_data);
	+ inlen = reformat - (void *)in + params->size;
	+
	+ MLX5_SET(alloc_packet_reformat_context_in, in, opcode,
	+ MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT);
	+ MLX5_SET(packet_reformat_context_in, packet_reformat_context_in,
	+ reformat_data_size, params->size);
	+ MLX5_SET(packet_reformat_context_in, packet_reformat_context_in,
	+ reformat_type, params->type);
	+ MLX5_SET(packet_reformat_context_in, packet_reformat_context_in,
	+ reformat_param_0, params->param_0);
	+ MLX5_SET(packet_reformat_context_in, packet_reformat_context_in,
	+ reformat_param_1, params->param_1);
	+ if (params->data && params->size)
	+ memcpy(reformat, params->data, params->size);
	+
	+ err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
	+
	+ pkt_reformat->id = MLX5_GET(alloc_packet_reformat_context_out,
	+ out, packet_reformat_id);
	+ kfree(in);
	+ return err;
	+}
	+
	+static void mlx5_cmd_packet_reformat_dealloc(struct mlx5_flow_root_namespace *ns,
	+ struct mlx5_pkt_reformat *pkt_reformat)
	+{
	+ u32 in[MLX5_ST_SZ_DW(dealloc_packet_reformat_context_in)] = {};
	+ struct mlx5_core_dev *dev = ns->dev;
	+
	+ MLX5_SET(dealloc_packet_reformat_context_in, in, opcode,
	+ MLX5_CMD_OP_DEALLOC_PACKET_REFORMAT_CONTEXT);
	+ MLX5_SET(dealloc_packet_reformat_context_in, in, packet_reformat_id,
	+ pkt_reformat->id);
	+
	+ mlx5_cmd_exec_in(dev, dealloc_packet_reformat_context, in);
	+}
	+
	+static int mlx5_cmd_modify_header_alloc(struct mlx5_flow_root_namespace *ns,
	+ u8 namespace, u8 num_actions,
	+ void *modify_actions,
	+ struct mlx5_modify_hdr *modify_hdr)
	+{
	+ u32 out[MLX5_ST_SZ_DW(alloc_modify_header_context_out)] = {};
	+ int max_actions, actions_size, inlen, err;
	+ struct mlx5_core_dev *dev = ns->dev;
	+ void *actions_in;
	+ u8 table_type;
	+ u32 *in;
	+
	+ switch (namespace) {
	+ case MLX5_FLOW_NAMESPACE_FDB:
	+ case MLX5_FLOW_NAMESPACE_FDB_BYPASS:
	+ max_actions = MLX5_CAP_ESW_FLOWTABLE_FDB(dev, max_modify_header_actions);
	+ table_type = FS_FT_FDB;
	+ break;
	+ case MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC:
	+ case MLX5_FLOW_NAMESPACE_KERNEL:
	+ case MLX5_FLOW_NAMESPACE_BYPASS:
	+ max_actions = MLX5_CAP_FLOWTABLE_NIC_RX(dev, max_modify_header_actions);
	+ table_type = FS_FT_NIC_RX;
	+ break;
	+ case MLX5_FLOW_NAMESPACE_EGRESS:
	+ case MLX5_FLOW_NAMESPACE_EGRESS_IPSEC:
	+ case MLX5_FLOW_NAMESPACE_EGRESS_MACSEC:
	+ max_actions = MLX5_CAP_FLOWTABLE_NIC_TX(dev, max_modify_header_actions);
	+ table_type = FS_FT_NIC_TX;
	+ break;
	+ case MLX5_FLOW_NAMESPACE_ESW_INGRESS:
	+ max_actions = MLX5_CAP_ESW_INGRESS_ACL(dev, max_modify_header_actions);
	+ table_type = FS_FT_ESW_INGRESS_ACL;
	+ break;
	+ case MLX5_FLOW_NAMESPACE_RDMA_TX:
	+ max_actions = MLX5_CAP_FLOWTABLE_RDMA_TX(dev, max_modify_header_actions);
	+ table_type = FS_FT_RDMA_TX;
	+ break;
	+ default:
	+ return -EOPNOTSUPP;
	}

	- err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
	- if (!err)
	- *fte_status = 0;
	+ if (num_actions > max_actions) {
	+ mlx5_core_warn(dev, "too many modify header actions %d, max supported %d\n",
	+ num_actions, max_actions);
	+ return -EOPNOTSUPP;
	+ }
	+
	+ actions_size = MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto) * num_actions;
	+ inlen = MLX5_ST_SZ_BYTES(alloc_modify_header_context_in) + actions_size;
	+
	+ in = kzalloc(inlen, GFP_KERNEL);
	+ if (!in)
	+ return -ENOMEM;
	+
	+ MLX5_SET(alloc_modify_header_context_in, in, opcode,
	+ MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT);
	+ MLX5_SET(alloc_modify_header_context_in, in, table_type, table_type);
	+ MLX5_SET(alloc_modify_header_context_in, in, num_of_actions, num_actions);
	+
	+ actions_in = MLX5_ADDR_OF(alloc_modify_header_context_in, in, actions);
	+ memcpy(actions_in, modify_actions, actions_size);

	+ err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
	+
	+ modify_hdr->id = MLX5_GET(alloc_modify_header_context_out, out, modify_header_id);
	+ kfree(in);
	return err;
	}

	-int mlx5_cmd_modify_header_alloc(struct mlx5_core_dev *dev,
	- enum mlx5_flow_namespace_type namespace,
	- u8 num_actions,
	- void *modify_actions,
	- struct mlx5_modify_hdr *modify_hdr)
	-{
	- u32 out[MLX5_ST_SZ_DW(alloc_modify_header_context_out)] = {};
	- int max_actions, actions_size, inlen, err;
	- void *actions_in;
	- u8 table_type;
	- u32 *in;
	-
	- switch (namespace) {
	- case MLX5_FLOW_NAMESPACE_FDB:
	- max_actions = MLX5_CAP_ESW_FLOWTABLE_FDB(dev, max_modify_header_actions);
	- table_type = FS_FT_FDB;
	- break;
	- case MLX5_FLOW_NAMESPACE_KERNEL:
	- case MLX5_FLOW_NAMESPACE_BYPASS:
	- max_actions = MLX5_CAP_FLOWTABLE_NIC_RX(dev, max_modify_header_actions);
	- table_type = FS_FT_NIC_RX;
	- break;
	- case MLX5_FLOW_NAMESPACE_ESW_INGRESS:
	- max_actions = MLX5_CAP_ESW_INGRESS_ACL(dev, max_modify_header_actions);
	- table_type = FS_FT_ESW_INGRESS_ACL;
	- break;
	- default:
	- return -EOPNOTSUPP;
	- }
	-
	- if (num_actions > max_actions) {
	- mlx5_core_warn(dev, "too many modify header actions %d, max supported %d\n",
	- num_actions, max_actions);
	- return -EOPNOTSUPP;
	- }
	-
	- actions_size = MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto) * num_actions;
	- inlen = MLX5_ST_SZ_BYTES(alloc_modify_header_context_in) + actions_size;
	-
	- in = kzalloc(inlen, GFP_KERNEL);
	- if (!in)
	- return -ENOMEM;
	-
	- MLX5_SET(alloc_modify_header_context_in, in, opcode,
	- MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT);
	- MLX5_SET(alloc_modify_header_context_in, in, table_type, table_type);
	- MLX5_SET(alloc_modify_header_context_in, in, num_of_actions, num_actions);
	-
	- actions_in = MLX5_ADDR_OF(alloc_modify_header_context_in, in, actions);
	- memcpy(actions_in, modify_actions, actions_size);
	-
	- err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
	-
	- modify_hdr->id = MLX5_GET(alloc_modify_header_context_out, out, modify_header_id);
	- kfree(in);
	-
	- return err;
	-}
	-
	-void mlx5_cmd_modify_header_dealloc(struct mlx5_core_dev *dev,
	- struct mlx5_modify_hdr *modify_hdr)
	-{
	- u32 out[MLX5_ST_SZ_DW(dealloc_modify_header_context_out)] = {};
	- u32 in[MLX5_ST_SZ_DW(dealloc_modify_header_context_in)] = {};
	-
	- MLX5_SET(dealloc_modify_header_context_in, in, opcode,
	- MLX5_CMD_OP_DEALLOC_MODIFY_HEADER_CONTEXT);
	- MLX5_SET(dealloc_modify_header_context_in, in, modify_header_id,
	- modify_hdr->id);
	-
	- mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
	-}
	-
	-int mlx5_cmd_packet_reformat_alloc(struct mlx5_core_dev *dev,
	- struct mlx5_pkt_reformat_params *params,
	- enum mlx5_flow_namespace_type namespace,
	- struct mlx5_pkt_reformat *pkt_reformat)
	-{
	- u32 out[MLX5_ST_SZ_DW(alloc_packet_reformat_context_out)] = {};
	- void *packet_reformat_context_in;
	- int max_encap_size;
	- void *reformat;
	- int inlen;
	- int err;
	- u32 *in;
	-
	- if (namespace == MLX5_FLOW_NAMESPACE_FDB)
	- max_encap_size = MLX5_CAP_ESW(dev, max_encap_header_size);
	- else
	- max_encap_size = MLX5_CAP_FLOWTABLE(dev, max_encap_header_size);
	-
	- if (params->size > max_encap_size) {
	- mlx5_core_warn(dev, "encap size %zd too big, max supported is %d\n",
	- params->size, max_encap_size);
	- return -EINVAL;
	- }
	-
	- in = kzalloc(MLX5_ST_SZ_BYTES(alloc_packet_reformat_context_in) +
	- params->size, GFP_KERNEL);
	- if (!in)
	- return -ENOMEM;
	-
	- packet_reformat_context_in = MLX5_ADDR_OF(alloc_packet_reformat_context_in,
	- in, packet_reformat_context);
	- reformat = MLX5_ADDR_OF(packet_reformat_context_in,
	- packet_reformat_context_in,
	- reformat_data);
	- inlen = reformat - (void *)in + params->size;
	-
	- MLX5_SET(alloc_packet_reformat_context_in, in, opcode,
	- MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT);
	- MLX5_SET(packet_reformat_context_in, packet_reformat_context_in,
	- reformat_data_size, params->size);
	- MLX5_SET(packet_reformat_context_in, packet_reformat_context_in,
	- reformat_type, params->type);
	- MLX5_SET(packet_reformat_context_in, packet_reformat_context_in,
	- reformat_param_0, params->param_0);
	- MLX5_SET(packet_reformat_context_in, packet_reformat_context_in,
	- reformat_param_1, params->param_1);
	- if (params->data && params->size)
	- memcpy(reformat, params->data, params->size);
	-
	- err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
	-
	- pkt_reformat->id = MLX5_GET(alloc_packet_reformat_context_out,
	- out, packet_reformat_id);
	- kfree(in);
	+static void mlx5_cmd_modify_header_dealloc(struct mlx5_flow_root_namespace *ns,
	+ struct mlx5_modify_hdr *modify_hdr)
	+{
	+ u32 in[MLX5_ST_SZ_DW(dealloc_modify_header_context_in)] = {};
	+ struct mlx5_core_dev *dev = ns->dev;
	+
	+ MLX5_SET(dealloc_modify_header_context_in, in, opcode,
	+ MLX5_CMD_OP_DEALLOC_MODIFY_HEADER_CONTEXT);
	+ MLX5_SET(dealloc_modify_header_context_in, in, modify_header_id,
	+ modify_hdr->id);

	- return err;
	+ mlx5_cmd_exec_in(dev, dealloc_modify_header_context, in);
	}

	-void mlx5_cmd_packet_reformat_dealloc(struct mlx5_core_dev *dev,
	- struct mlx5_pkt_reformat *pkt_reformat)
	+static u32 mlx5_cmd_get_capabilities(struct mlx5_flow_root_namespace *ns,
	+ enum fs_flow_table_type ft_type)
	{
	- u32 out[MLX5_ST_SZ_DW(dealloc_packet_reformat_context_out)] = {};
	- u32 in[MLX5_ST_SZ_DW(dealloc_packet_reformat_context_in)] = {};
	+ return 0;
	+}

	- MLX5_SET(dealloc_packet_reformat_context_in, in, opcode,
	- MLX5_CMD_OP_DEALLOC_PACKET_REFORMAT_CONTEXT);
	- MLX5_SET(dealloc_packet_reformat_context_in, in, packet_reformat_id,
	- pkt_reformat->id);
	+static const struct mlx5_flow_cmds mlx5_flow_cmds = {
	+ .create_flow_table = mlx5_cmd_create_flow_table,
	+ .destroy_flow_table = mlx5_cmd_destroy_flow_table,
	+ .modify_flow_table = mlx5_cmd_modify_flow_table,
	+ .create_flow_group = mlx5_cmd_create_flow_group,
	+ .destroy_flow_group = mlx5_cmd_destroy_flow_group,
	+ .create_fte = mlx5_cmd_create_fte,
	+ .update_fte = mlx5_cmd_update_fte,
	+ .delete_fte = mlx5_cmd_delete_fte,
	+ .update_root_ft = mlx5_cmd_update_root_ft,
	+ .packet_reformat_alloc = mlx5_cmd_packet_reformat_alloc,
	+ .packet_reformat_dealloc = mlx5_cmd_packet_reformat_dealloc,
	+ .modify_header_alloc = mlx5_cmd_modify_header_alloc,
	+ .modify_header_dealloc = mlx5_cmd_modify_header_dealloc,
	+ .set_peer = mlx5_cmd_stub_set_peer,
	+ .create_ns = mlx5_cmd_stub_create_ns,
	+ .destroy_ns = mlx5_cmd_stub_destroy_ns,
	+ .get_capabilities = mlx5_cmd_get_capabilities,
	+};
	+
	+static const struct mlx5_flow_cmds mlx5_flow_cmd_stubs = {
	+ .create_flow_table = mlx5_cmd_stub_create_flow_table,
	+ .destroy_flow_table = mlx5_cmd_stub_destroy_flow_table,
	+ .modify_flow_table = mlx5_cmd_stub_modify_flow_table,
	+ .create_flow_group = mlx5_cmd_stub_create_flow_group,
	+ .destroy_flow_group = mlx5_cmd_stub_destroy_flow_group,
	+ .create_fte = mlx5_cmd_stub_create_fte,
	+ .update_fte = mlx5_cmd_stub_update_fte,
	+ .delete_fte = mlx5_cmd_stub_delete_fte,
	+ .update_root_ft = mlx5_cmd_stub_update_root_ft,
	+ .packet_reformat_alloc = mlx5_cmd_stub_packet_reformat_alloc,
	+ .packet_reformat_dealloc = mlx5_cmd_stub_packet_reformat_dealloc,
	+ .modify_header_alloc = mlx5_cmd_stub_modify_header_alloc,
	+ .modify_header_dealloc = mlx5_cmd_stub_modify_header_dealloc,
	+ .set_peer = mlx5_cmd_stub_set_peer,
	+ .create_ns = mlx5_cmd_stub_create_ns,
	+ .destroy_ns = mlx5_cmd_stub_destroy_ns,
	+ .get_capabilities = mlx5_cmd_stub_get_capabilities,
	+};
	+
	+const struct mlx5_flow_cmds *mlx5_fs_cmd_get_fw_cmds(void)
	+{
	+ return &mlx5_flow_cmds;
	+}
	+
	+static const struct mlx5_flow_cmds *mlx5_fs_cmd_get_stub_cmds(void)
	+{
	+ return &mlx5_flow_cmd_stubs;
	+}

	- mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
	+const struct mlx5_flow_cmds *mlx5_fs_cmd_get_default(enum fs_flow_table_type type)
	+{
	+ switch (type) {
	+ case FS_FT_NIC_RX:
	+ case FS_FT_ESW_EGRESS_ACL:
	+ case FS_FT_ESW_INGRESS_ACL:
	+ case FS_FT_FDB:
	+ case FS_FT_SNIFFER_RX:
	+ case FS_FT_SNIFFER_TX:
	+ case FS_FT_NIC_TX:
	+ case FS_FT_RDMA_RX:
	+ case FS_FT_RDMA_TX:
	+ case FS_FT_PORT_SEL:
	+ return mlx5_fs_cmd_get_fw_cmds();
	+ default:
	+ return mlx5_fs_cmd_get_stub_cmds();
	+ }
	}
	diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fs_core.c b/sys/dev/mlx5/mlx5_core/mlx5_fs_core.c
	new file mode 100644
	--- /dev/null
	+++ b/sys/dev/mlx5/mlx5_core/mlx5_fs_core.c
	@@ -0,0 +1,3514 @@
	+/*
	+ * Copyright (c) 2015, Mellanox Technologies. All rights reserved.
	+ *
	+ * This software is available to you under a choice of one of two
	+ * licenses. You may choose to be licensed under the terms of the GNU
	+ * General Public License (GPL) Version 2, available from the file
	+ * COPYING in the main directory of this source tree, or the
	+ * OpenIB.org BSD license below:
	+ *
	+ * Redistribution and use in source and binary forms, with or
	+ * without modification, are permitted provided that the following
	+ * conditions are met:
	+ *
	+ * - Redistributions of source code must retain the above
	+ * copyright notice, this list of conditions and the following
	+ * disclaimer.
	+ *
	+ * - Redistributions in binary form must reproduce the above
	+ * copyright notice, this list of conditions and the following
	+ * disclaimer in the documentation and/or other materials
	+ * provided with the distribution.
	+ *
	+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	+ * SOFTWARE.
	+ */
	+
	+#include <linux/module.h>
	+#include <dev/mlx5/driver.h>
	+#include <dev/mlx5/mlx5_core/mlx5_core.h>
	+#include <dev/mlx5/mlx5_core/fs_core.h>
	+#include <linux/string.h>
	+#include <linux/compiler.h>
	+
	+#include "eswitch.h"
	+#include "fs_ft_pool.h"
	+#include "fs_cmd.h"
	+
	+#define down_write_nested(a, b) down_write(a)
	+
	+#define INIT_TREE_NODE_ARRAY_SIZE(...) (sizeof((struct init_tree_node[]){__VA_ARGS__}) /\
	+ sizeof(struct init_tree_node))
	+
	+#define ADD_PRIO(num_prios_val, min_level_val, num_levels_val, caps_val,\
	+ ...) {.type = FS_TYPE_PRIO,\
	+ .min_ft_level = min_level_val,\
	+ .num_levels = num_levels_val,\
	+ .num_leaf_prios = num_prios_val,\
	+ .caps = caps_val,\
	+ .children = (struct init_tree_node[]) {__VA_ARGS__},\
	+ .ar_size = INIT_TREE_NODE_ARRAY_SIZE(__VA_ARGS__) \
	+}
	+
	+#define ADD_MULTIPLE_PRIO(num_prios_val, num_levels_val, ...)\
	+ ADD_PRIO(num_prios_val, 0, num_levels_val, {},\
	+ __VA_ARGS__)\
	+
	+#define ADD_NS(def_miss_act, ...) {.type = FS_TYPE_NAMESPACE, \
	+ .def_miss_action = def_miss_act,\
	+ .children = (struct init_tree_node[]) {__VA_ARGS__},\
	+ .ar_size = INIT_TREE_NODE_ARRAY_SIZE(__VA_ARGS__) \
	+}
	+
	+#define INIT_CAPS_ARRAY_SIZE(...) (sizeof((long[]){__VA_ARGS__}) /\
	+ sizeof(long))
	+
	+#define FS_CAP(cap) (__mlx5_bit_off(flow_table_nic_cap, cap))
	+
	+#define FS_REQUIRED_CAPS(...) {.arr_sz = INIT_CAPS_ARRAY_SIZE(__VA_ARGS__), \
	+ .caps = (long[]) {__VA_ARGS__} }
	+
	+#define FS_CHAINING_CAPS FS_REQUIRED_CAPS(FS_CAP(flow_table_properties_nic_receive.flow_modify_en), \
	+ FS_CAP(flow_table_properties_nic_receive.modify_root), \
	+ FS_CAP(flow_table_properties_nic_receive.identified_miss_table_mode), \
	+ FS_CAP(flow_table_properties_nic_receive.flow_table_modify))
	+
	+#define FS_CHAINING_CAPS_EGRESS \
	+ FS_REQUIRED_CAPS( \
	+ FS_CAP(flow_table_properties_nic_transmit.flow_modify_en), \
	+ FS_CAP(flow_table_properties_nic_transmit.modify_root), \
	+ FS_CAP(flow_table_properties_nic_transmit \
	+ .identified_miss_table_mode), \
	+ FS_CAP(flow_table_properties_nic_transmit.flow_table_modify))
	+
	+#define FS_CHAINING_CAPS_RDMA_TX \
	+ FS_REQUIRED_CAPS( \
	+ FS_CAP(flow_table_properties_nic_transmit_rdma.flow_modify_en), \
	+ FS_CAP(flow_table_properties_nic_transmit_rdma.modify_root), \
	+ FS_CAP(flow_table_properties_nic_transmit_rdma \
	+ .identified_miss_table_mode), \
	+ FS_CAP(flow_table_properties_nic_transmit_rdma \
	+ .flow_table_modify))
	+
	+#define LEFTOVERS_NUM_LEVELS 1
	+#define LEFTOVERS_NUM_PRIOS 1
	+
	+#define RDMA_RX_COUNTERS_PRIO_NUM_LEVELS 1
	+#define RDMA_TX_COUNTERS_PRIO_NUM_LEVELS 1
	+
	+#define BY_PASS_PRIO_NUM_LEVELS 1
	+#define BY_PASS_MIN_LEVEL (ETHTOOL_MIN_LEVEL + MLX5_BY_PASS_NUM_PRIOS +\
	+ LEFTOVERS_NUM_PRIOS)
	+
	+#define KERNEL_RX_MACSEC_NUM_PRIOS 1
	+#define KERNEL_RX_MACSEC_NUM_LEVELS 2
	+#define KERNEL_RX_MACSEC_MIN_LEVEL (BY_PASS_MIN_LEVEL + KERNEL_RX_MACSEC_NUM_PRIOS)
	+
	+#define ETHTOOL_PRIO_NUM_LEVELS 1
	+#define ETHTOOL_NUM_PRIOS 11
	+#define ETHTOOL_MIN_LEVEL (KERNEL_MIN_LEVEL + ETHTOOL_NUM_PRIOS)
	+/* Promiscuous, Vlan, mac, ttc, inner ttc, {UDP/ANY/aRFS/accel/{esp, esp_err}}, IPsec policy,
	+ * IPsec RoCE policy
	+ */
	+#define KERNEL_NIC_PRIO_NUM_LEVELS 9
	+#define KERNEL_NIC_NUM_PRIOS 1
	+/* One more level for tc */
	+#define KERNEL_MIN_LEVEL (KERNEL_NIC_PRIO_NUM_LEVELS + 1)
	+
	+#define KERNEL_NIC_TC_NUM_PRIOS 1
	+#define KERNEL_NIC_TC_NUM_LEVELS 13
	+
	+#define ANCHOR_NUM_LEVELS 1
	+#define ANCHOR_NUM_PRIOS 1
	+#define ANCHOR_MIN_LEVEL (BY_PASS_MIN_LEVEL + 1)
	+
	+#define OFFLOADS_MAX_FT 2
	+#define OFFLOADS_NUM_PRIOS 1
	+#define OFFLOADS_MIN_LEVEL (ANCHOR_MIN_LEVEL + OFFLOADS_NUM_PRIOS)
	+
	+#define LAG_PRIO_NUM_LEVELS 1
	+#define LAG_NUM_PRIOS 1
	+#define LAG_MIN_LEVEL (OFFLOADS_MIN_LEVEL + KERNEL_RX_MACSEC_MIN_LEVEL + 1)
	+
	+#define KERNEL_TX_IPSEC_NUM_PRIOS 1
	+#define KERNEL_TX_IPSEC_NUM_LEVELS 3
	+#define KERNEL_TX_IPSEC_MIN_LEVEL (KERNEL_TX_IPSEC_NUM_LEVELS)
	+
	+#define KERNEL_TX_MACSEC_NUM_PRIOS 1
	+#define KERNEL_TX_MACSEC_NUM_LEVELS 2
	+#define KERNEL_TX_MACSEC_MIN_LEVEL (KERNEL_TX_IPSEC_MIN_LEVEL + KERNEL_TX_MACSEC_NUM_PRIOS)
	+
	+struct node_caps {
	+ size_t arr_sz;
	+ long *caps;
	+};
	+
	+static struct init_tree_node {
	+ enum fs_node_type type;
	+ struct init_tree_node *children;
	+ int ar_size;
	+ struct node_caps caps;
	+ int min_ft_level;
	+ int num_leaf_prios;
	+ int prio;
	+ int num_levels;
	+ enum mlx5_flow_table_miss_action def_miss_action;
	+} root_fs = {
	+ .type = FS_TYPE_NAMESPACE,
	+ .ar_size = 8,
	+ .children = (struct init_tree_node[]){
	+ ADD_PRIO(0, BY_PASS_MIN_LEVEL, 0, FS_CHAINING_CAPS,
	+ ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
	+ ADD_MULTIPLE_PRIO(MLX5_BY_PASS_NUM_PRIOS,
	+ BY_PASS_PRIO_NUM_LEVELS))),
	+ ADD_PRIO(0, KERNEL_RX_MACSEC_MIN_LEVEL, 0, FS_CHAINING_CAPS,
	+ ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
	+ ADD_MULTIPLE_PRIO(KERNEL_RX_MACSEC_NUM_PRIOS,
	+ KERNEL_RX_MACSEC_NUM_LEVELS))),
	+ ADD_PRIO(0, LAG_MIN_LEVEL, 0, FS_CHAINING_CAPS,
	+ ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
	+ ADD_MULTIPLE_PRIO(LAG_NUM_PRIOS,
	+ LAG_PRIO_NUM_LEVELS))),
	+ ADD_PRIO(0, OFFLOADS_MIN_LEVEL, 0, FS_CHAINING_CAPS,
	+ ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
	+ ADD_MULTIPLE_PRIO(OFFLOADS_NUM_PRIOS,
	+ OFFLOADS_MAX_FT))),
	+ ADD_PRIO(0, ETHTOOL_MIN_LEVEL, 0, FS_CHAINING_CAPS,
	+ ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
	+ ADD_MULTIPLE_PRIO(ETHTOOL_NUM_PRIOS,
	+ ETHTOOL_PRIO_NUM_LEVELS))),
	+ ADD_PRIO(0, KERNEL_MIN_LEVEL, 0, {},
	+ ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
	+ ADD_MULTIPLE_PRIO(KERNEL_NIC_TC_NUM_PRIOS,
	+ KERNEL_NIC_TC_NUM_LEVELS),
	+ ADD_MULTIPLE_PRIO(KERNEL_NIC_NUM_PRIOS,
	+ KERNEL_NIC_PRIO_NUM_LEVELS))),
	+ ADD_PRIO(0, BY_PASS_MIN_LEVEL, 0, FS_CHAINING_CAPS,
	+ ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
	+ ADD_MULTIPLE_PRIO(LEFTOVERS_NUM_PRIOS,
	+ LEFTOVERS_NUM_LEVELS))),
	+ ADD_PRIO(0, ANCHOR_MIN_LEVEL, 0, {},
	+ ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
	+ ADD_MULTIPLE_PRIO(ANCHOR_NUM_PRIOS,
	+ ANCHOR_NUM_LEVELS))),
	+ }
	+};
	+
	+static struct init_tree_node egress_root_fs = {
	+ .type = FS_TYPE_NAMESPACE,
	+ .ar_size = 3,
	+ .children = (struct init_tree_node[]) {
	+ ADD_PRIO(0, MLX5_BY_PASS_NUM_PRIOS, 0,
	+ FS_CHAINING_CAPS_EGRESS,
	+ ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
	+ ADD_MULTIPLE_PRIO(MLX5_BY_PASS_NUM_PRIOS,
	+ BY_PASS_PRIO_NUM_LEVELS))),
	+ ADD_PRIO(0, KERNEL_TX_IPSEC_MIN_LEVEL, 0,
	+ FS_CHAINING_CAPS_EGRESS,
	+ ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
	+ ADD_MULTIPLE_PRIO(KERNEL_TX_IPSEC_NUM_PRIOS,
	+ KERNEL_TX_IPSEC_NUM_LEVELS))),
	+ ADD_PRIO(0, KERNEL_TX_MACSEC_MIN_LEVEL, 0,
	+ FS_CHAINING_CAPS_EGRESS,
	+ ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
	+ ADD_MULTIPLE_PRIO(KERNEL_TX_MACSEC_NUM_PRIOS,
	+ KERNEL_TX_MACSEC_NUM_LEVELS))),
	+ }
	+};
	+
	+enum {
	+ RDMA_RX_IPSEC_PRIO,
	+ RDMA_RX_COUNTERS_PRIO,
	+ RDMA_RX_BYPASS_PRIO,
	+ RDMA_RX_KERNEL_PRIO,
	+};
	+
	+#define RDMA_RX_IPSEC_NUM_PRIOS 1
	+#define RDMA_RX_IPSEC_NUM_LEVELS 2
	+#define RDMA_RX_IPSEC_MIN_LEVEL (RDMA_RX_IPSEC_NUM_LEVELS)
	+
	+#define RDMA_RX_BYPASS_MIN_LEVEL MLX5_BY_PASS_NUM_REGULAR_PRIOS
	+#define RDMA_RX_KERNEL_MIN_LEVEL (RDMA_RX_BYPASS_MIN_LEVEL + 1)
	+#define RDMA_RX_COUNTERS_MIN_LEVEL (RDMA_RX_KERNEL_MIN_LEVEL + 2)
	+
	+static struct init_tree_node rdma_rx_root_fs = {
	+ .type = FS_TYPE_NAMESPACE,
	+ .ar_size = 4,
	+ .children = (struct init_tree_node[]) {
	+ [RDMA_RX_IPSEC_PRIO] =
	+ ADD_PRIO(0, RDMA_RX_IPSEC_MIN_LEVEL, 0,
	+ FS_CHAINING_CAPS,
	+ ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
	+ ADD_MULTIPLE_PRIO(RDMA_RX_IPSEC_NUM_PRIOS,
	+ RDMA_RX_IPSEC_NUM_LEVELS))),
	+ [RDMA_RX_COUNTERS_PRIO] =
	+ ADD_PRIO(0, RDMA_RX_COUNTERS_MIN_LEVEL, 0,
	+ FS_CHAINING_CAPS,
	+ ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
	+ ADD_MULTIPLE_PRIO(MLX5_RDMA_RX_NUM_COUNTERS_PRIOS,
	+ RDMA_RX_COUNTERS_PRIO_NUM_LEVELS))),
	+ [RDMA_RX_BYPASS_PRIO] =
	+ ADD_PRIO(0, RDMA_RX_BYPASS_MIN_LEVEL, 0,
	+ FS_CHAINING_CAPS,
	+ ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
	+ ADD_MULTIPLE_PRIO(MLX5_BY_PASS_NUM_REGULAR_PRIOS,
	+ BY_PASS_PRIO_NUM_LEVELS))),
	+ [RDMA_RX_KERNEL_PRIO] =
	+ ADD_PRIO(0, RDMA_RX_KERNEL_MIN_LEVEL, 0,
	+ FS_CHAINING_CAPS,
	+ ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_SWITCH_DOMAIN,
	+ ADD_MULTIPLE_PRIO(1, 1))),
	+ }
	+};
	+
	+enum {
	+ RDMA_TX_COUNTERS_PRIO,
	+ RDMA_TX_IPSEC_PRIO,
	+ RDMA_TX_BYPASS_PRIO,
	+};
	+
	+#define RDMA_TX_BYPASS_MIN_LEVEL MLX5_BY_PASS_NUM_PRIOS
	+#define RDMA_TX_COUNTERS_MIN_LEVEL (RDMA_TX_BYPASS_MIN_LEVEL + 1)
	+
	+#define RDMA_TX_IPSEC_NUM_PRIOS 1
	+#define RDMA_TX_IPSEC_PRIO_NUM_LEVELS 1
	+#define RDMA_TX_IPSEC_MIN_LEVEL (RDMA_TX_COUNTERS_MIN_LEVEL + RDMA_TX_IPSEC_NUM_PRIOS)
	+
	+static struct init_tree_node rdma_tx_root_fs = {
	+ .type = FS_TYPE_NAMESPACE,
	+ .ar_size = 3,
	+ .children = (struct init_tree_node[]) {
	+ [RDMA_TX_COUNTERS_PRIO] =
	+ ADD_PRIO(0, RDMA_TX_COUNTERS_MIN_LEVEL, 0,
	+ FS_CHAINING_CAPS,
	+ ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
	+ ADD_MULTIPLE_PRIO(MLX5_RDMA_TX_NUM_COUNTERS_PRIOS,
	+ RDMA_TX_COUNTERS_PRIO_NUM_LEVELS))),
	+ [RDMA_TX_IPSEC_PRIO] =
	+ ADD_PRIO(0, RDMA_TX_IPSEC_MIN_LEVEL, 0,
	+ FS_CHAINING_CAPS,
	+ ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
	+ ADD_MULTIPLE_PRIO(RDMA_TX_IPSEC_NUM_PRIOS,
	+ RDMA_TX_IPSEC_PRIO_NUM_LEVELS))),
	+
	+ [RDMA_TX_BYPASS_PRIO] =
	+ ADD_PRIO(0, RDMA_TX_BYPASS_MIN_LEVEL, 0,
	+ FS_CHAINING_CAPS_RDMA_TX,
	+ ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
	+ ADD_MULTIPLE_PRIO(RDMA_TX_BYPASS_MIN_LEVEL,
	+ BY_PASS_PRIO_NUM_LEVELS))),
	+ }
	+};
	+
	+enum fs_i_lock_class {
	+ FS_LOCK_GRANDPARENT,
	+ FS_LOCK_PARENT,
	+ FS_LOCK_CHILD
	+};
	+
	+static void del_hw_flow_table(struct fs_node *node);
	+static void del_hw_flow_group(struct fs_node *node);
	+static void del_hw_fte(struct fs_node *node);
	+static void del_sw_flow_table(struct fs_node *node);
	+static void del_sw_flow_group(struct fs_node *node);
	+static void del_sw_fte(struct fs_node *node);
	+static void del_sw_prio(struct fs_node *node);
	+static void del_sw_ns(struct fs_node *node);
	+/* Delete rule (destination) is special case that
	+ * requires to lock the FTE for all the deletion process.
	+ */
	+static void del_sw_hw_rule(struct fs_node *node);
	+static bool mlx5_flow_dests_cmp(struct mlx5_flow_destination *d1,
	+ struct mlx5_flow_destination *d2);
	+static void cleanup_root_ns(struct mlx5_flow_root_namespace *root_ns);
	+static struct mlx5_flow_rule *
	+find_flow_rule(struct fs_fte *fte,
	+ struct mlx5_flow_destination *dest);
	+
	+static void tree_init_node(struct fs_node *node,
	+ void (del_hw_func)(struct fs_node ),
	+ void (del_sw_func)(struct fs_node ))
	+{
	+ refcount_set(&node->refcount, 1);
	+ INIT_LIST_HEAD(&node->list);
	+ INIT_LIST_HEAD(&node->children);
	+ init_rwsem(&node->lock);
	+ node->del_hw_func = del_hw_func;
	+ node->del_sw_func = del_sw_func;
	+ node->active = false;
	+}
	+
	+static void tree_add_node(struct fs_node node, struct fs_node parent)
	+{
	+ if (parent)
	+ refcount_inc(&parent->refcount);
	+ node->parent = parent;
	+
	+ /* Parent is the root */
	+ if (!parent)
	+ node->root = node;
	+ else
	+ node->root = parent->root;
	+}
	+
	+static int tree_get_node(struct fs_node *node)
	+{
	+ return refcount_inc_not_zero(&node->refcount);
	+}
	+
	+static void nested_down_read_ref_node(struct fs_node *node,
	+ enum fs_i_lock_class class)
	+{
	+ if (node) {
	+ down_read_nested(&node->lock, class);
	+ refcount_inc(&node->refcount);
	+ }
	+}
	+
	+static void nested_down_write_ref_node(struct fs_node *node,
	+ enum fs_i_lock_class class)
	+{
	+ if (node) {
	+ down_write_nested(&node->lock, class);
	+ refcount_inc(&node->refcount);
	+ }
	+}
	+
	+static void down_write_ref_node(struct fs_node *node, bool locked)
	+{
	+ if (node) {
	+ if (!locked)
	+ down_write(&node->lock);
	+ refcount_inc(&node->refcount);
	+ }
	+}
	+
	+static void up_read_ref_node(struct fs_node *node)
	+{
	+ refcount_dec(&node->refcount);
	+ up_read(&node->lock);
	+}
	+
	+static void up_write_ref_node(struct fs_node *node, bool locked)
	+{
	+ refcount_dec(&node->refcount);
	+ if (!locked)
	+ up_write(&node->lock);
	+}
	+
	+static void tree_put_node(struct fs_node *node, bool locked)
	+{
	+ struct fs_node *parent_node = node->parent;
	+
	+ if (refcount_dec_and_test(&node->refcount.value)) {
	+ if (node->del_hw_func)
	+ node->del_hw_func(node);
	+ if (parent_node) {
	+ down_write_ref_node(parent_node, locked);
	+ list_del_init(&node->list);
	+ }
	+ node->del_sw_func(node);
	+ if (parent_node)
	+ up_write_ref_node(parent_node, locked);
	+ node = NULL;
	+ }
	+ if (!node && parent_node)
	+ tree_put_node(parent_node, locked);
	+}
	+
	+static int tree_remove_node(struct fs_node *node, bool locked)
	+{
	+ if (refcount_read(&node->refcount) > 1) {
	+ refcount_dec(&node->refcount);
	+ return -EEXIST;
	+ }
	+ tree_put_node(node, locked);
	+ return 0;
	+}
	+
	+static struct fs_prio find_prio(struct mlx5_flow_namespace ns,
	+ unsigned int prio)
	+{
	+ struct fs_prio *iter_prio;
	+
	+ fs_for_each_prio(iter_prio, ns) {
	+ if (iter_prio->prio == prio)
	+ return iter_prio;
	+ }
	+
	+ return NULL;
	+}
	+
	+static bool is_fwd_next_action(u32 action)
	+{
	+ return action & (MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO \|
	+ MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS);
	+}
	+
	+static bool is_fwd_dest_type(enum mlx5_flow_destination_type type)
	+{
	+ return type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM \|\|
	+ type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE \|\|
	+ type == MLX5_FLOW_DESTINATION_TYPE_UPLINK \|\|
	+ type == MLX5_FLOW_DESTINATION_TYPE_VPORT \|\|
	+ type == MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER \|\|
	+ type == MLX5_FLOW_DESTINATION_TYPE_TIR \|\|
	+ type == MLX5_FLOW_DESTINATION_TYPE_RANGE \|\|
	+ type == MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE;
	+}
	+
	+static bool check_valid_spec(const struct mlx5_flow_spec *spec)
	+{
	+ int i;
	+
	+ for (i = 0; i < MLX5_ST_SZ_DW_MATCH_PARAM; i++)
	+ if (spec->match_value[i] & ~spec->match_criteria[i]) {
	+ pr_warn("mlx5_core: match_value differs from match_criteria\n");
	+ return false;
	+ }
	+
	+ return true;
	+}
	+
	+struct mlx5_flow_root_namespace find_root(struct fs_node node)
	+{
	+ struct fs_node *root;
	+ struct mlx5_flow_namespace *ns;
	+
	+ root = node->root;
	+
	+ if (WARN_ON(root->type != FS_TYPE_NAMESPACE)) {
	+ pr_warn("mlx5: flow steering node is not in tree or garbaged\n");
	+ return NULL;
	+ }
	+
	+ ns = container_of(root, struct mlx5_flow_namespace, node);
	+ return container_of(ns, struct mlx5_flow_root_namespace, ns);
	+}
	+
	+static inline struct mlx5_flow_steering get_steering(struct fs_node node)
	+{
	+ struct mlx5_flow_root_namespace *root = find_root(node);
	+
	+ if (root)
	+ return root->dev->priv.steering;
	+ return NULL;
	+}
	+
	+static inline struct mlx5_core_dev get_dev(struct fs_node node)
	+{
	+ struct mlx5_flow_root_namespace *root = find_root(node);
	+
	+ if (root)
	+ return root->dev;
	+ return NULL;
	+}
	+
	+static void del_sw_ns(struct fs_node *node)
	+{
	+ kfree(node);
	+}
	+
	+static void del_sw_prio(struct fs_node *node)
	+{
	+ kfree(node);
	+}
	+
	+static void del_hw_flow_table(struct fs_node *node)
	+{
	+ struct mlx5_flow_root_namespace *root;
	+ struct mlx5_flow_table *ft;
	+ struct mlx5_core_dev *dev;
	+ int err;
	+
	+ fs_get_obj(ft, node);
	+ dev = get_dev(&ft->node);
	+ root = find_root(&ft->node);
	+
	+ if (node->active) {
	+ err = root->cmds->destroy_flow_table(root, ft);
	+ if (err)
	+ mlx5_core_warn(dev, "flow steering can't destroy ft\n");
	+ }
	+}
	+
	+static void del_sw_flow_table(struct fs_node *node)
	+{
	+ struct mlx5_flow_table *ft;
	+ struct fs_prio *prio;
	+
	+ fs_get_obj(ft, node);
	+
	+ xa_destroy(&ft->fgs_xa);
	+ if (ft->node.parent) {
	+ fs_get_obj(prio, ft->node.parent);
	+ prio->num_ft--;
	+ }
	+ kfree(ft);
	+}
	+
	+static void modify_fte(struct fs_fte *fte)
	+{
	+ struct mlx5_flow_root_namespace *root;
	+ struct mlx5_flow_table *ft;
	+ struct mlx5_flow_group *fg;
	+ struct mlx5_core_dev *dev;
	+ int err;
	+
	+ fs_get_obj(fg, fte->node.parent);
	+ fs_get_obj(ft, fg->node.parent);
	+ dev = get_dev(&fte->node);
	+
	+ root = find_root(&ft->node);
	+ err = root->cmds->update_fte(root, ft, fg, fte->modify_mask, fte);
	+ if (err)
	+ mlx5_core_warn(dev,
	+ "%s can't del rule fg id=%d fte_index=%d\n",
	+ __func__, fg->id, fte->index);
	+ fte->modify_mask = 0;
	+}
	+
	+static void del_sw_hw_rule(struct fs_node *node)
	+{
	+ struct mlx5_flow_rule *rule;
	+ struct fs_fte *fte;
	+
	+ fs_get_obj(rule, node);
	+ fs_get_obj(fte, rule->node.parent);
	+ if (is_fwd_next_action(rule->sw_action)) {
	+ mutex_lock(&rule->dest_attr.ft->lock);
	+ list_del(&rule->next_ft);
	+ mutex_unlock(&rule->dest_attr.ft->lock);
	+ }
	+
	+ if (rule->dest_attr.type == MLX5_FLOW_DESTINATION_TYPE_COUNTER) {
	+ --fte->dests_size;
	+ fte->modify_mask \|=
	+ BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_ACTION) \|
	+ BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_FLOW_COUNTERS);
	+ fte->action.action &= ~MLX5_FLOW_CONTEXT_ACTION_COUNT;
	+ goto out;
	+ }
	+
	+ if (rule->dest_attr.type == MLX5_FLOW_DESTINATION_TYPE_PORT) {
	+ --fte->dests_size;
	+ fte->modify_mask \|= BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_ACTION);
	+ fte->action.action &= ~MLX5_FLOW_CONTEXT_ACTION_ALLOW;
	+ goto out;
	+ }
	+
	+ if (is_fwd_dest_type(rule->dest_attr.type)) {
	+ --fte->dests_size;
	+ --fte->fwd_dests;
	+
	+ if (!fte->fwd_dests)
	+ fte->action.action &=
	+ ~MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
	+ fte->modify_mask \|=
	+ BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_DESTINATION_LIST);
	+ goto out;
	+ }
	+out:
	+ kfree(rule);
	+}
	+
	+static void del_hw_fte(struct fs_node *node)
	+{
	+ struct mlx5_flow_root_namespace *root;
	+ struct mlx5_flow_table *ft;
	+ struct mlx5_flow_group *fg;
	+ struct mlx5_core_dev *dev;
	+ struct fs_fte *fte;
	+ int err;
	+
	+ fs_get_obj(fte, node);
	+ fs_get_obj(fg, fte->node.parent);
	+ fs_get_obj(ft, fg->node.parent);
	+
	+ WARN_ON(fte->dests_size);
	+ dev = get_dev(&ft->node);
	+ root = find_root(&ft->node);
	+ if (node->active) {
	+ err = root->cmds->delete_fte(root, ft, fte);
	+ if (err)
	+ mlx5_core_warn(dev,
	+ "flow steering can't delete fte in index %d of flow group id %d\n",
	+ fte->index, fg->id);
	+ node->active = false;
	+ }
	+}
	+
	+static void del_sw_fte(struct fs_node *node)
	+{
	+ struct mlx5_flow_steering *steering = get_steering(node);
	+ struct mlx5_flow_group *fg;
	+ struct fs_fte *deleted_fte;
	+ struct fs_fte *fte;
	+
	+ fs_get_obj(fte, node);
	+ fs_get_obj(fg, fte->node.parent);
	+
	+ deleted_fte = xa_erase(&fg->ftes_xa, fte->index);
	+ WARN_ON(deleted_fte != fte);
	+ ida_free(&fg->fte_allocator, fte->index - fg->start_index);
	+ kmem_cache_free(steering->ftes_cache, fte);
	+}
	+
	+static void del_hw_flow_group(struct fs_node *node)
	+{
	+ struct mlx5_flow_root_namespace *root;
	+ struct mlx5_flow_group *fg;
	+ struct mlx5_flow_table *ft;
	+ struct mlx5_core_dev *dev;
	+
	+ fs_get_obj(fg, node);
	+ fs_get_obj(ft, fg->node.parent);
	+ dev = get_dev(&ft->node);
	+
	+ root = find_root(&ft->node);
	+ if (fg->node.active && root->cmds->destroy_flow_group(root, ft, fg))
	+ mlx5_core_warn(dev, "flow steering can't destroy fg %d of ft %d\n",
	+ fg->id, ft->id);
	+}
	+
	+static void del_sw_flow_group(struct fs_node *node)
	+{
	+ struct mlx5_flow_steering *steering = get_steering(node);
	+ struct mlx5_flow_group *deleted_fg;
	+ struct mlx5_flow_group *fg;
	+ struct mlx5_flow_table *ft;
	+
	+ fs_get_obj(fg, node);
	+ fs_get_obj(ft, fg->node.parent);
	+
	+ xa_destroy(&fg->ftes_xa);
	+ ida_destroy(&fg->fte_allocator);
	+ if (ft->autogroup.active &&
	+ fg->max_ftes == ft->autogroup.group_size &&
	+ fg->start_index < ft->autogroup.max_fte)
	+ ft->autogroup.num_groups--;
	+ deleted_fg = xa_erase(&ft->fgs_xa, fg->start_index);
	+ WARN_ON(deleted_fg != fg);
	+ kmem_cache_free(steering->fgs_cache, fg);
	+}
	+
	+static int insert_fte(struct mlx5_flow_group fg, struct fs_fte fte)
	+{
	+ int index;
	+ int ret;
	+
	+ index = ida_alloc_max(&fg->fte_allocator, fg->max_ftes, GFP_KERNEL);
	+ if (index < 0)
	+ return index;
	+
	+ fte->index = index + fg->start_index;
	+ ret = xa_insert(&fg->ftes_xa, fte->index, fte, GFP_KERNEL);
	+ if (ret)
	+ goto err_ida_remove;
	+
	+ tree_add_node(&fte->node, &fg->node);
	+ list_add_tail(&fte->node.list, &fg->node.children);
	+ return 0;
	+
	+err_ida_remove:
	+ ida_free(&fg->fte_allocator, index);
	+ return ret;
	+}
	+
	+static struct fs_fte alloc_fte(struct mlx5_flow_table ft,
	+ const struct mlx5_flow_spec *spec,
	+ struct mlx5_flow_act *flow_act)
	+{
	+ struct mlx5_flow_steering *steering = get_steering(&ft->node);
	+ struct fs_fte *fte;
	+
	+ fte = kmem_cache_zalloc(steering->ftes_cache, GFP_KERNEL);
	+ if (!fte)
	+ return ERR_PTR(-ENOMEM);
	+
	+ memcpy(fte->val, &spec->match_value, sizeof(fte->val));
	+ fte->node.type = FS_TYPE_FLOW_ENTRY;
	+ fte->action = *flow_act;
	+ fte->flow_context = spec->flow_context;
	+
	+ tree_init_node(&fte->node, del_hw_fte, del_sw_fte);
	+
	+ return fte;
	+}
	+
	+static void dealloc_flow_group(struct mlx5_flow_steering *steering,
	+ struct mlx5_flow_group *fg)
	+{
	+ xa_destroy(&fg->ftes_xa);
	+ kmem_cache_free(steering->fgs_cache, fg);
	+}
	+
	+static struct mlx5_flow_group alloc_flow_group(struct mlx5_flow_steering steering,
	+ u8 match_criteria_enable,
	+ const void *match_criteria,
	+ int start_index,
	+ int end_index)
	+{
	+ struct mlx5_flow_group *fg;
	+
	+ fg = kmem_cache_zalloc(steering->fgs_cache, GFP_KERNEL);
	+ if (!fg)
	+ return ERR_PTR(-ENOMEM);
	+
	+ xa_init(&fg->ftes_xa);
	+
	+ ida_init(&fg->fte_allocator);
	+ fg->mask.match_criteria_enable = match_criteria_enable;
	+ memcpy(&fg->mask.match_criteria, match_criteria,
	+ sizeof(fg->mask.match_criteria));
	+ fg->node.type = FS_TYPE_FLOW_GROUP;
	+ fg->start_index = start_index;
	+ fg->max_ftes = end_index - start_index + 1;
	+
	+ return fg;
	+}
	+
	+static struct mlx5_flow_group alloc_insert_flow_group(struct mlx5_flow_table ft,
	+ u8 match_criteria_enable,
	+ const void *match_criteria,
	+ int start_index,
	+ int end_index,
	+ struct list_head *prev)
	+{
	+ struct mlx5_flow_steering *steering = get_steering(&ft->node);
	+ struct mlx5_flow_group *fg;
	+ int ret;
	+
	+ fg = alloc_flow_group(steering, match_criteria_enable, match_criteria,
	+ start_index, end_index);
	+ if (IS_ERR(fg))
	+ return fg;
	+
	+ /* initialize refcnt, add to parent list */
	+ ret = xa_insert(&ft->fgs_xa, fg->start_index, fg, GFP_KERNEL);
	+ if (ret) {
	+ dealloc_flow_group(steering, fg);
	+ return ERR_PTR(ret);
	+ }
	+
	+ tree_init_node(&fg->node, del_hw_flow_group, del_sw_flow_group);
	+ tree_add_node(&fg->node, &ft->node);
	+ /* Add node to group list */
	+ list_add(&fg->node.list, prev);
	+ atomic_inc(&ft->node.version);
	+
	+ return fg;
	+}
	+
	+static struct mlx5_flow_table *alloc_flow_table(int level, u16 vport,
	+ enum fs_flow_table_type table_type,
	+ enum fs_flow_table_op_mod op_mod,
	+ u32 flags)
	+{
	+ struct mlx5_flow_table *ft;
	+
	+ ft = kzalloc(sizeof(*ft), GFP_KERNEL);
	+ if (!ft)
	+ return ERR_PTR(-ENOMEM);
	+
	+ xa_init(&ft->fgs_xa);
	+
	+ ft->level = level;
	+ ft->node.type = FS_TYPE_FLOW_TABLE;
	+ ft->op_mod = op_mod;
	+ ft->type = table_type;
	+ ft->vport = vport;
	+ ft->flags = flags;
	+ INIT_LIST_HEAD(&ft->fwd_rules);
	+ mutex_init(&ft->lock);
	+
	+ return ft;
	+}
	+
	+/* If reverse is false, then we search for the first flow table in the
	+ * root sub-tree from start(closest from right), else we search for the
	+ * last flow table in the root sub-tree till start(closest from left).
	+ */
	+static struct mlx5_flow_table find_closest_ft_recursive(struct fs_node root,
	+ struct list_head *start,
	+ bool reverse)
	+{
	+#define list_advance_entry(pos, reverse) \
	+ ((reverse) ? list_prev_entry(pos, list) : list_next_entry(pos, list))
	+
	+#define list_for_each_advance_continue(pos, head, reverse) \
	+ for (pos = list_advance_entry(pos, reverse); \
	+ &pos->list != (head); \
	+ pos = list_advance_entry(pos, reverse))
	+
	+ struct fs_node *iter = list_entry(start, struct fs_node, list);
	+ struct mlx5_flow_table *ft = NULL;
	+
	+ if (!root \|\| root->type == FS_TYPE_PRIO_CHAINS)
	+ return NULL;
	+
	+ list_for_each_advance_continue(iter, &root->children, reverse) {
	+ if (iter->type == FS_TYPE_FLOW_TABLE) {
	+ fs_get_obj(ft, iter);
	+ return ft;
	+ }
	+ ft = find_closest_ft_recursive(iter, &iter->children, reverse);
	+ if (ft)
	+ return ft;
	+ }
	+
	+ return ft;
	+}
	+
	+/* If reverse is false then return the first flow table in next priority of
	+ * prio in the tree, else return the last flow table in the previous priority
	+ * of prio in the tree.
	+ */
	+static struct mlx5_flow_table find_closest_ft(struct fs_prio prio, bool reverse)
	+{
	+ struct mlx5_flow_table *ft = NULL;
	+ struct fs_node *curr_node;
	+ struct fs_node *parent;
	+
	+ parent = prio->node.parent;
	+ curr_node = &prio->node;
	+ while (!ft && parent) {
	+ ft = find_closest_ft_recursive(parent, &curr_node->list, reverse);
	+ curr_node = parent;
	+ parent = curr_node->parent;
	+ }
	+ return ft;
	+}
	+
	+/* Assuming all the tree is locked by mutex chain lock */
	+static struct mlx5_flow_table find_next_chained_ft(struct fs_prio prio)
	+{
	+ return find_closest_ft(prio, false);
	+}
	+
	+/* Assuming all the tree is locked by mutex chain lock */
	+static struct mlx5_flow_table find_prev_chained_ft(struct fs_prio prio)
	+{
	+ return find_closest_ft(prio, true);
	+}
	+
	+static struct mlx5_flow_table find_next_fwd_ft(struct mlx5_flow_table ft,
	+ struct mlx5_flow_act *flow_act)
	+{
	+ struct fs_prio *prio;
	+ bool next_ns;
	+
	+ next_ns = flow_act->action & MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS;
	+ fs_get_obj(prio, next_ns ? ft->ns->node.parent : ft->node.parent);
	+
	+ return find_next_chained_ft(prio);
	+}
	+
	+static int connect_fts_in_prio(struct mlx5_core_dev *dev,
	+ struct fs_prio *prio,
	+ struct mlx5_flow_table *ft)
	+{
	+ struct mlx5_flow_root_namespace *root = find_root(&prio->node);
	+ struct mlx5_flow_table *iter;
	+ int err;
	+
	+ fs_for_each_ft(iter, prio) {
	+ err = root->cmds->modify_flow_table(root, iter, ft);
	+ if (err) {
	+ mlx5_core_err(dev,
	+ "Failed to modify flow table id %d, type %d, err %d\n",
	+ iter->id, iter->type, err);
	+ /* The driver is out of sync with the FW */
	+ return err;
	+ }
	+ }
	+ return 0;
	+}
	+
	+/* Connect flow tables from previous priority of prio to ft */
	+static int connect_prev_fts(struct mlx5_core_dev *dev,
	+ struct mlx5_flow_table *ft,
	+ struct fs_prio *prio)
	+{
	+ struct mlx5_flow_table *prev_ft;
	+
	+ prev_ft = find_prev_chained_ft(prio);
	+ if (prev_ft) {
	+ struct fs_prio *prev_prio;
	+
	+ fs_get_obj(prev_prio, prev_ft->node.parent);
	+ return connect_fts_in_prio(dev, prev_prio, ft);
	+ }
	+ return 0;
	+}
	+
	+static int update_root_ft_create(struct mlx5_flow_table *ft, struct fs_prio
	+ *prio)
	+{
	+ struct mlx5_flow_root_namespace *root = find_root(&prio->node);
	+ struct mlx5_ft_underlay_qp *uqp;
	+ int min_level = INT_MAX;
	+ int err = 0;
	+ u32 qpn;
	+
	+ if (root->root_ft)
	+ min_level = root->root_ft->level;
	+
	+ if (ft->level >= min_level)
	+ return 0;
	+
	+ if (list_empty(&root->underlay_qpns)) {
	+ /* Don't set any QPN (zero) in case QPN list is empty */
	+ qpn = 0;
	+ err = root->cmds->update_root_ft(root, ft, qpn, false);
	+ } else {
	+ list_for_each_entry(uqp, &root->underlay_qpns, list) {
	+ qpn = uqp->qpn;
	+ err = root->cmds->update_root_ft(root, ft,
	+ qpn, false);
	+ if (err)
	+ break;
	+ }
	+ }
	+
	+ if (err)
	+ mlx5_core_warn(root->dev,
	+ "Update root flow table of id(%u) qpn(%d) failed\n",
	+ ft->id, qpn);
	+ else
	+ root->root_ft = ft;
	+
	+ return err;
	+}
	+
	+static int _mlx5_modify_rule_destination(struct mlx5_flow_rule *rule,
	+ struct mlx5_flow_destination *dest)
	+{
	+ struct mlx5_flow_root_namespace *root;
	+ struct mlx5_flow_table *ft;
	+ struct mlx5_flow_group *fg;
	+ struct fs_fte *fte;
	+ int modify_mask = BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_DESTINATION_LIST);
	+ int err = 0;
	+
	+ fs_get_obj(fte, rule->node.parent);
	+ if (!(fte->action.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST))
	+ return -EINVAL;
	+ down_write_ref_node(&fte->node, false);
	+ fs_get_obj(fg, fte->node.parent);
	+ fs_get_obj(ft, fg->node.parent);
	+
	+ memcpy(&rule->dest_attr, dest, sizeof(*dest));
	+ root = find_root(&ft->node);
	+ err = root->cmds->update_fte(root, ft, fg,
	+ modify_mask, fte);
	+ up_write_ref_node(&fte->node, false);
	+
	+ return err;
	+}
	+
	+int mlx5_modify_rule_destination(struct mlx5_flow_handle *handle,
	+ struct mlx5_flow_destination *new_dest,
	+ struct mlx5_flow_destination *old_dest)
	+{
	+ int i;
	+
	+ if (!old_dest) {
	+ if (handle->num_rules != 1)
	+ return -EINVAL;
	+ return _mlx5_modify_rule_destination(handle->rule[0],
	+ new_dest);
	+ }
	+
	+ for (i = 0; i < handle->num_rules; i++) {
	+ if (mlx5_flow_dests_cmp(new_dest, &handle->rule[i]->dest_attr))
	+ return _mlx5_modify_rule_destination(handle->rule[i],
	+ new_dest);
	+ }
	+
	+ return -EINVAL;
	+}
	+
	+/* Modify/set FWD rules that point on old_next_ft to point on new_next_ft */
	+static int connect_fwd_rules(struct mlx5_core_dev *dev,
	+ struct mlx5_flow_table *new_next_ft,
	+ struct mlx5_flow_table *old_next_ft)
	+{
	+ struct mlx5_flow_destination dest = {};
	+ struct mlx5_flow_rule *iter;
	+ int err = 0;
	+
	+ /* new_next_ft and old_next_ft could be NULL only
	+ * when we create/destroy the anchor flow table.
	+ */
	+ if (!new_next_ft \|\| !old_next_ft)
	+ return 0;
	+
	+ dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
	+ dest.ft = new_next_ft;
	+
	+ mutex_lock(&old_next_ft->lock);
	+ list_splice_init(&old_next_ft->fwd_rules, &new_next_ft->fwd_rules);
	+ mutex_unlock(&old_next_ft->lock);
	+ list_for_each_entry(iter, &new_next_ft->fwd_rules, next_ft) {
	+ if ((iter->sw_action & MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS) &&
	+ iter->ft->ns == new_next_ft->ns)
	+ continue;
	+
	+ err = _mlx5_modify_rule_destination(iter, &dest);
	+ if (err)
	+ pr_err("mlx5_core: failed to modify rule to point on flow table %d\n",
	+ new_next_ft->id);
	+ }
	+ return 0;
	+}
	+
	+static int connect_flow_table(struct mlx5_core_dev dev, struct mlx5_flow_table ft,
	+ struct fs_prio *prio)
	+{
	+ struct mlx5_flow_table next_ft, first_ft;
	+ int err = 0;
	+
	+ /* Connect_prev_fts and update_root_ft_create are mutually exclusive */
	+
	+ first_ft = list_first_entry_or_null(&prio->node.children,
	+ struct mlx5_flow_table, node.list);
	+ if (!first_ft \|\| first_ft->level > ft->level) {
	+ err = connect_prev_fts(dev, ft, prio);
	+ if (err)
	+ return err;
	+
	+ next_ft = first_ft ? first_ft : find_next_chained_ft(prio);
	+ err = connect_fwd_rules(dev, ft, next_ft);
	+ if (err)
	+ return err;
	+ }
	+
	+ if (MLX5_CAP_FLOWTABLE(dev,
	+ flow_table_properties_nic_receive.modify_root))
	+ err = update_root_ft_create(ft, prio);
	+ return err;
	+}
	+
	+static void list_add_flow_table(struct mlx5_flow_table *ft,
	+ struct fs_prio *prio)
	+{
	+ struct list_head *prev = &prio->node.children;
	+ struct mlx5_flow_table *iter;
	+
	+ fs_for_each_ft(iter, prio) {
	+ if (iter->level > ft->level)
	+ break;
	+ prev = &iter->node.list;
	+ }
	+ list_add(&ft->node.list, prev);
	+}
	+
	+static struct mlx5_flow_table __mlx5_create_flow_table(struct mlx5_flow_namespace ns,
	+ struct mlx5_flow_table_attr *ft_attr,
	+ enum fs_flow_table_op_mod op_mod,
	+ u16 vport)
	+{
	+ struct mlx5_flow_root_namespace *root = find_root(&ns->node);
	+ bool unmanaged = ft_attr->flags & MLX5_FLOW_TABLE_UNMANAGED;
	+ struct mlx5_flow_table *next_ft;
	+ struct fs_prio *fs_prio = NULL;
	+ struct mlx5_flow_table *ft;
	+ int err;
	+
	+ if (!root) {
	+ pr_err("mlx5: flow steering failed to find root of namespace\n");
	+ return ERR_PTR(-ENODEV);
	+ }
	+
	+ mutex_lock(&root->chain_lock);
	+ fs_prio = find_prio(ns, ft_attr->prio);
	+ if (!fs_prio) {
	+ err = -EINVAL;
	+ goto unlock_root;
	+ }
	+ if (!unmanaged) {
	+ /* The level is related to the
	+ * priority level range.
	+ */
	+ if (ft_attr->level >= fs_prio->num_levels) {
	+ err = -ENOSPC;
	+ goto unlock_root;
	+ }
	+
	+ ft_attr->level += fs_prio->start_level;
	+ }
	+
	+ /* The level is related to the
	+ * priority level range.
	+ */
	+ ft = alloc_flow_table(ft_attr->level,
	+ vport,
	+ root->table_type,
	+ op_mod, ft_attr->flags);
	+ if (IS_ERR(ft)) {
	+ err = PTR_ERR(ft);
	+ goto unlock_root;
	+ }
	+
	+ tree_init_node(&ft->node, del_hw_flow_table, del_sw_flow_table);
	+ next_ft = unmanaged ? ft_attr->next_ft :
	+ find_next_chained_ft(fs_prio);
	+ ft->def_miss_action = ns->def_miss_action;
	+ ft->ns = ns;
	+ err = root->cmds->create_flow_table(root, ft, ft_attr, next_ft);
	+ if (err)
	+ goto free_ft;
	+
	+ if (!unmanaged) {
	+ err = connect_flow_table(root->dev, ft, fs_prio);
	+ if (err)
	+ goto destroy_ft;
	+ }
	+
	+ ft->node.active = true;
	+ down_write_ref_node(&fs_prio->node, false);
	+ if (!unmanaged) {
	+ tree_add_node(&ft->node, &fs_prio->node);
	+ list_add_flow_table(ft, fs_prio);
	+ } else {
	+ ft->node.root = fs_prio->node.root;
	+ }
	+ fs_prio->num_ft++;
	+ up_write_ref_node(&fs_prio->node, false);
	+ mutex_unlock(&root->chain_lock);
	+ return ft;
	+destroy_ft:
	+ root->cmds->destroy_flow_table(root, ft);
	+free_ft:
	+ xa_destroy(&ft->fgs_xa);
	+ kfree(ft);
	+unlock_root:
	+ mutex_unlock(&root->chain_lock);
	+ return ERR_PTR(err);
	+}
	+
	+struct mlx5_flow_table mlx5_create_flow_table(struct mlx5_flow_namespace ns,
	+ struct mlx5_flow_table_attr *ft_attr)
	+{
	+ return __mlx5_create_flow_table(ns, ft_attr, FS_FT_OP_MOD_NORMAL, 0);
	+}
	+EXPORT_SYMBOL(mlx5_create_flow_table);
	+
	+u32 mlx5_flow_table_id(struct mlx5_flow_table *ft)
	+{
	+ return ft->id;
	+}
	+EXPORT_SYMBOL(mlx5_flow_table_id);
	+
	+struct mlx5_flow_table *
	+mlx5_create_vport_flow_table(struct mlx5_flow_namespace *ns,
	+ struct mlx5_flow_table_attr *ft_attr, u16 vport)
	+{
	+ return __mlx5_create_flow_table(ns, ft_attr, FS_FT_OP_MOD_NORMAL, vport);
	+}
	+
	+struct mlx5_flow_table*
	+mlx5_create_lag_demux_flow_table(struct mlx5_flow_namespace *ns,
	+ int prio, u32 level)
	+{
	+ struct mlx5_flow_table_attr ft_attr = {};
	+
	+ ft_attr.level = level;
	+ ft_attr.prio = prio;
	+ ft_attr.max_fte = 1;
	+
	+ return __mlx5_create_flow_table(ns, &ft_attr, FS_FT_OP_MOD_LAG_DEMUX, 0);
	+}
	+EXPORT_SYMBOL(mlx5_create_lag_demux_flow_table);
	+
	+#define MAX_FLOW_GROUP_SIZE BIT(24)
	+struct mlx5_flow_table*
	+mlx5_create_auto_grouped_flow_table(struct mlx5_flow_namespace *ns,
	+ struct mlx5_flow_table_attr *ft_attr)
	+{
	+ int num_reserved_entries = ft_attr->autogroup.num_reserved_entries;
	+ int max_num_groups = ft_attr->autogroup.max_num_groups;
	+ struct mlx5_flow_table *ft;
	+ int autogroups_max_fte;
	+
	+ ft = mlx5_create_flow_table(ns, ft_attr);
	+ if (IS_ERR(ft))
	+ return ft;
	+
	+ autogroups_max_fte = ft->max_fte - num_reserved_entries;
	+ if (max_num_groups > autogroups_max_fte)
	+ goto err_validate;
	+ if (num_reserved_entries > ft->max_fte)
	+ goto err_validate;
	+
	+ /* Align the number of groups according to the largest group size */
	+ if (autogroups_max_fte / (max_num_groups + 1) > MAX_FLOW_GROUP_SIZE)
	+ max_num_groups = (autogroups_max_fte / MAX_FLOW_GROUP_SIZE) - 1;
	+
	+ ft->autogroup.active = true;
	+ ft->autogroup.required_groups = max_num_groups;
	+ ft->autogroup.max_fte = autogroups_max_fte;
	+ /* We save place for flow groups in addition to max types */
	+ ft->autogroup.group_size = autogroups_max_fte / (max_num_groups + 1);
	+
	+ return ft;
	+
	+err_validate:
	+ mlx5_destroy_flow_table(ft);
	+ return ERR_PTR(-ENOSPC);
	+}
	+EXPORT_SYMBOL(mlx5_create_auto_grouped_flow_table);
	+
	+struct mlx5_flow_group mlx5_create_flow_group(struct mlx5_flow_table ft,
	+ u32 *fg_in)
	+{
	+ struct mlx5_flow_root_namespace *root = find_root(&ft->node);
	+ void *match_criteria = MLX5_ADDR_OF(create_flow_group_in,
	+ fg_in, match_criteria);
	+ u8 match_criteria_enable = MLX5_GET(create_flow_group_in,
	+ fg_in,
	+ match_criteria_enable);
	+ int start_index = MLX5_GET(create_flow_group_in, fg_in,
	+ start_flow_index);
	+ int end_index = MLX5_GET(create_flow_group_in, fg_in,
	+ end_flow_index);
	+ struct mlx5_flow_group *fg;
	+ int err;
	+
	+ if (ft->autogroup.active && start_index < ft->autogroup.max_fte)
	+ return ERR_PTR(-EPERM);
	+
	+ down_write_ref_node(&ft->node, false);
	+ fg = alloc_insert_flow_group(ft, match_criteria_enable, match_criteria,
	+ start_index, end_index,
	+ ft->node.children.prev);
	+ up_write_ref_node(&ft->node, false);
	+ if (IS_ERR(fg))
	+ return fg;
	+
	+ err = root->cmds->create_flow_group(root, ft, fg_in, fg);
	+ if (err) {
	+ tree_put_node(&fg->node, false);
	+ return ERR_PTR(err);
	+ }
	+ fg->node.active = true;
	+
	+ return fg;
	+}
	+EXPORT_SYMBOL(mlx5_create_flow_group);
	+
	+static struct mlx5_flow_rule alloc_rule(struct mlx5_flow_destination dest)
	+{
	+ struct mlx5_flow_rule *rule;
	+
	+ rule = kzalloc(sizeof(*rule), GFP_KERNEL);
	+ if (!rule)
	+ return NULL;
	+
	+ INIT_LIST_HEAD(&rule->next_ft);
	+ rule->node.type = FS_TYPE_FLOW_DEST;
	+ if (dest)
	+ memcpy(&rule->dest_attr, dest, sizeof(*dest));
	+ else
	+ rule->dest_attr.type = MLX5_FLOW_DESTINATION_TYPE_NONE;
	+
	+ return rule;
	+}
	+
	+static struct mlx5_flow_handle *alloc_handle(int num_rules)
	+{
	+ struct mlx5_flow_handle *handle;
	+
	+ handle = kzalloc(struct_size(handle, rule, num_rules), GFP_KERNEL);
	+ if (!handle)
	+ return NULL;
	+
	+ handle->num_rules = num_rules;
	+
	+ return handle;
	+}
	+
	+static void destroy_flow_handle(struct fs_fte *fte,
	+ struct mlx5_flow_handle *handle,
	+ struct mlx5_flow_destination *dest,
	+ int i)
	+{
	+ for (; --i >= 0;) {
	+ if (refcount_dec_and_test(&handle->rule[i]->node.refcount.value)) {
	+ fte->dests_size--;
	+ list_del(&handle->rule[i]->node.list);
	+ kfree(handle->rule[i]);
	+ }
	+ }
	+ kfree(handle);
	+}
	+
	+static struct mlx5_flow_handle *
	+create_flow_handle(struct fs_fte *fte,
	+ struct mlx5_flow_destination *dest,
	+ int dest_num,
	+ int *modify_mask,
	+ bool *new_rule)
	+{
	+ struct mlx5_flow_handle *handle;
	+ struct mlx5_flow_rule *rule = NULL;
	+ static int count = BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_FLOW_COUNTERS);
	+ static int dst = BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_DESTINATION_LIST);
	+ int type;
	+ int i = 0;
	+
	+ handle = alloc_handle((dest_num) ? dest_num : 1);
	+ if (!handle)
	+ return ERR_PTR(-ENOMEM);
	+
	+ do {
	+ if (dest) {
	+ rule = find_flow_rule(fte, dest + i);
	+ if (rule) {
	+ refcount_inc(&rule->node.refcount);
	+ goto rule_found;
	+ }
	+ }
	+
	+ *new_rule = true;
	+ rule = alloc_rule(dest + i);
	+ if (!rule)
	+ goto free_rules;
	+
	+ /* Add dest to dests list- we need flow tables to be in the
	+ * end of the list for forward to next prio rules.
	+ */
	+ tree_init_node(&rule->node, NULL, del_sw_hw_rule);
	+ if (dest &&
	+ dest[i].type != MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE)
	+ list_add(&rule->node.list, &fte->node.children);
	+ else
	+ list_add_tail(&rule->node.list, &fte->node.children);
	+ if (dest) {
	+ fte->dests_size++;
	+
	+ if (is_fwd_dest_type(dest[i].type))
	+ fte->fwd_dests++;
	+
	+ type = dest[i].type ==
	+ MLX5_FLOW_DESTINATION_TYPE_COUNTER;
	+ *modify_mask \|= type ? count : dst;
	+ }
	+rule_found:
	+ handle->rule[i] = rule;
	+ } while (++i < dest_num);
	+
	+ return handle;
	+
	+free_rules:
	+ destroy_flow_handle(fte, handle, dest, i);
	+ return ERR_PTR(-ENOMEM);
	+}
	+
	+/* fte should not be deleted while calling this function */
	+static struct mlx5_flow_handle *
	+add_rule_fte(struct fs_fte *fte,
	+ struct mlx5_flow_group *fg,
	+ struct mlx5_flow_destination *dest,
	+ int dest_num,
	+ bool update_action)
	+{
	+ struct mlx5_flow_root_namespace *root;
	+ struct mlx5_flow_handle *handle;
	+ struct mlx5_flow_table *ft;
	+ int modify_mask = 0;
	+ int err;
	+ bool new_rule = false;
	+
	+ handle = create_flow_handle(fte, dest, dest_num, &modify_mask,
	+ &new_rule);
	+ if (IS_ERR(handle) \|\| !new_rule)
	+ goto out;
	+
	+ if (update_action)
	+ modify_mask \|= BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_ACTION);
	+
	+ fs_get_obj(ft, fg->node.parent);
	+ root = find_root(&fg->node);
	+ if (!(fte->status & FS_FTE_STATUS_EXISTING))
	+ err = root->cmds->create_fte(root, ft, fg, fte);
	+ else
	+ err = root->cmds->update_fte(root, ft, fg, modify_mask, fte);
	+ if (err)
	+ goto free_handle;
	+
	+ fte->node.active = true;
	+ fte->status \|= FS_FTE_STATUS_EXISTING;
	+ atomic_inc(&fg->node.version);
	+
	+out:
	+ return handle;
	+
	+free_handle:
	+ destroy_flow_handle(fte, handle, dest, handle->num_rules);
	+ return ERR_PTR(err);
	+}
	+
	+static struct mlx5_flow_group alloc_auto_flow_group(struct mlx5_flow_table ft,
	+ const struct mlx5_flow_spec *spec)
	+{
	+ struct list_head *prev = &ft->node.children;
	+ u32 max_fte = ft->autogroup.max_fte;
	+ unsigned int candidate_index = 0;
	+ unsigned int group_size = 0;
	+ struct mlx5_flow_group *fg;
	+
	+ if (!ft->autogroup.active)
	+ return ERR_PTR(-ENOENT);
	+
	+ if (ft->autogroup.num_groups < ft->autogroup.required_groups)
	+ group_size = ft->autogroup.group_size;
	+
	+ /* max_fte == ft->autogroup.max_types */
	+ if (group_size == 0)
	+ group_size = 1;
	+
	+ /* sorted by start_index */
	+ fs_for_each_fg(fg, ft) {
	+ if (candidate_index + group_size > fg->start_index)
	+ candidate_index = fg->start_index + fg->max_ftes;
	+ else
	+ break;
	+ prev = &fg->node.list;
	+ }
	+
	+ if (candidate_index + group_size > max_fte)
	+ return ERR_PTR(-ENOSPC);
	+
	+ fg = alloc_insert_flow_group(ft,
	+ spec->match_criteria_enable,
	+ spec->match_criteria,
	+ candidate_index,
	+ candidate_index + group_size - 1,
	+ prev);
	+ if (IS_ERR(fg))
	+ goto out;
	+
	+ if (group_size == ft->autogroup.group_size)
	+ ft->autogroup.num_groups++;
	+
	+out:
	+ return fg;
	+}
	+
	+static int create_auto_flow_group(struct mlx5_flow_table *ft,
	+ struct mlx5_flow_group *fg)
	+{
	+ struct mlx5_flow_root_namespace *root = find_root(&ft->node);
	+ int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
	+ void *match_criteria_addr;
	+ u8 src_esw_owner_mask_on;
	+ void *misc;
	+ int err;
	+ u32 *in;
	+
	+ in = kvzalloc(inlen, GFP_KERNEL);
	+ if (!in)
	+ return -ENOMEM;
	+
	+ MLX5_SET(create_flow_group_in, in, match_criteria_enable,
	+ fg->mask.match_criteria_enable);
	+ MLX5_SET(create_flow_group_in, in, start_flow_index, fg->start_index);
	+ MLX5_SET(create_flow_group_in, in, end_flow_index, fg->start_index +
	+ fg->max_ftes - 1);
	+
	+ misc = MLX5_ADDR_OF(fte_match_param, fg->mask.match_criteria,
	+ misc_parameters);
	+ src_esw_owner_mask_on = !!MLX5_GET(fte_match_set_misc, misc,
	+ source_eswitch_owner_vhca_id);
	+ MLX5_SET(create_flow_group_in, in,
	+ source_eswitch_owner_vhca_id_valid, src_esw_owner_mask_on);
	+
	+ match_criteria_addr = MLX5_ADDR_OF(create_flow_group_in,
	+ in, match_criteria);
	+ memcpy(match_criteria_addr, fg->mask.match_criteria,
	+ sizeof(fg->mask.match_criteria));
	+
	+ err = root->cmds->create_flow_group(root, ft, in, fg);
	+ if (!err) {
	+ fg->node.active = true;
	+ }
	+
	+ kvfree(in);
	+ return err;
	+}
	+
	+static bool mlx5_flow_dests_cmp(struct mlx5_flow_destination *d1,
	+ struct mlx5_flow_destination *d2)
	+{
	+ if (d1->type == d2->type) {
	+ if (((d1->type == MLX5_FLOW_DESTINATION_TYPE_VPORT \|\|
	+ d1->type == MLX5_FLOW_DESTINATION_TYPE_UPLINK) &&
	+ d1->vport.num == d2->vport.num &&
	+ d1->vport.flags == d2->vport.flags &&
	+ ((d1->vport.flags & MLX5_FLOW_DEST_VPORT_VHCA_ID) ?
	+ (d1->vport.vhca_id == d2->vport.vhca_id) : true) &&
	+ ((d1->vport.flags & MLX5_FLOW_DEST_VPORT_REFORMAT_ID) ?
	+ (d1->vport.pkt_reformat->id ==
	+ d2->vport.pkt_reformat->id) : true)) \|\|
	+ (d1->type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE &&
	+ d1->ft == d2->ft) \|\|
	+ (d1->type == MLX5_FLOW_DESTINATION_TYPE_TIR &&
	+ d1->tir_num == d2->tir_num) \|\|
	+ (d1->type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM &&
	+ d1->ft_num == d2->ft_num) \|\|
	+ (d1->type == MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER &&
	+ d1->sampler_id == d2->sampler_id) \|\|
	+ (d1->type == MLX5_FLOW_DESTINATION_TYPE_RANGE &&
	+ d1->range.field == d2->range.field &&
	+ d1->range.hit_ft == d2->range.hit_ft &&
	+ d1->range.miss_ft == d2->range.miss_ft &&
	+ d1->range.min == d2->range.min &&
	+ d1->range.max == d2->range.max))
	+ return true;
	+ }
	+
	+ return false;
	+}
	+
	+static struct mlx5_flow_rule find_flow_rule(struct fs_fte fte,
	+ struct mlx5_flow_destination *dest)
	+{
	+ struct mlx5_flow_rule *rule;
	+
	+ list_for_each_entry(rule, &fte->node.children, node.list) {
	+ if (mlx5_flow_dests_cmp(&rule->dest_attr, dest))
	+ return rule;
	+ }
	+ return NULL;
	+}
	+
	+static bool check_conflicting_actions_vlan(const struct mlx5_fs_vlan *vlan0,
	+ const struct mlx5_fs_vlan *vlan1)
	+{
	+ return vlan0->ethtype != vlan1->ethtype \|\|
	+ vlan0->vid != vlan1->vid \|\|
	+ vlan0->prio != vlan1->prio;
	+}
	+
	+static bool check_conflicting_actions(const struct mlx5_flow_act *act1,
	+ const struct mlx5_flow_act *act2)
	+{
	+ u32 action1 = act1->action;
	+ u32 action2 = act2->action;
	+ u32 xored_actions;
	+
	+ xored_actions = action1 ^ action2;
	+
	+ /* if one rule only wants to count, it's ok */
	+ if (action1 == MLX5_FLOW_CONTEXT_ACTION_COUNT \|\|
	+ action2 == MLX5_FLOW_CONTEXT_ACTION_COUNT)
	+ return false;
	+
	+ if (xored_actions & (MLX5_FLOW_CONTEXT_ACTION_DROP \|
	+ MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT \|
	+ MLX5_FLOW_CONTEXT_ACTION_DECAP \|
	+ MLX5_FLOW_CONTEXT_ACTION_MOD_HDR \|
	+ MLX5_FLOW_CONTEXT_ACTION_VLAN_POP \|
	+ MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH \|
	+ MLX5_FLOW_CONTEXT_ACTION_VLAN_POP_2 \|
	+ MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2))
	+ return true;
	+
	+ if (action1 & MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT &&
	+ act1->pkt_reformat != act2->pkt_reformat)
	+ return true;
	+
	+ if (action1 & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR &&
	+ act1->modify_hdr != act2->modify_hdr)
	+ return true;
	+
	+ if (action1 & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH &&
	+ check_conflicting_actions_vlan(&act1->vlan[0], &act2->vlan[0]))
	+ return true;
	+
	+ if (action1 & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2 &&
	+ check_conflicting_actions_vlan(&act1->vlan[1], &act2->vlan[1]))
	+ return true;
	+
	+ return false;
	+}
	+
	+static int check_conflicting_ftes(struct fs_fte *fte,
	+ const struct mlx5_flow_context *flow_context,
	+ const struct mlx5_flow_act *flow_act)
	+{
	+ if (check_conflicting_actions(flow_act, &fte->action)) {
	+ mlx5_core_warn(get_dev(&fte->node),
	+ "Found two FTEs with conflicting actions\n");
	+ return -EEXIST;
	+ }
	+
	+ if ((flow_context->flags & FLOW_CONTEXT_HAS_TAG) &&
	+ fte->flow_context.flow_tag != flow_context->flow_tag) {
	+ mlx5_core_warn(get_dev(&fte->node),
	+ "FTE flow tag %u already exists with different flow tag %u\n",
	+ fte->flow_context.flow_tag,
	+ flow_context->flow_tag);
	+ return -EEXIST;
	+ }
	+
	+ return 0;
	+}
	+
	+static struct mlx5_flow_handle add_rule_fg(struct mlx5_flow_group fg,
	+ const struct mlx5_flow_spec *spec,
	+ struct mlx5_flow_act *flow_act,
	+ struct mlx5_flow_destination *dest,
	+ int dest_num,
	+ struct fs_fte *fte)
	+{
	+ struct mlx5_flow_handle *handle;
	+ int old_action;
	+ int i;
	+ int ret;
	+
	+ ret = check_conflicting_ftes(fte, &spec->flow_context, flow_act);
	+ if (ret)
	+ return ERR_PTR(ret);
	+
	+ old_action = fte->action.action;
	+ fte->action.action \|= flow_act->action;
	+ handle = add_rule_fte(fte, fg, dest, dest_num,
	+ old_action != flow_act->action);
	+ if (IS_ERR(handle)) {
	+ fte->action.action = old_action;
	+ return handle;
	+ }
	+
	+ for (i = 0; i < handle->num_rules; i++) {
	+ if (refcount_read(&handle->rule[i]->node.refcount) == 1) {
	+ tree_add_node(&handle->rule[i]->node, &fte->node);
	+ }
	+ }
	+ return handle;
	+}
	+
	+static bool counter_is_valid(u32 action)
	+{
	+ return (action & (MLX5_FLOW_CONTEXT_ACTION_DROP \|
	+ MLX5_FLOW_CONTEXT_ACTION_ALLOW \|
	+ MLX5_FLOW_CONTEXT_ACTION_FWD_DEST));
	+}
	+
	+static bool dest_is_valid(struct mlx5_flow_destination *dest,
	+ struct mlx5_flow_act *flow_act,
	+ struct mlx5_flow_table *ft)
	+{
	+ bool ignore_level = flow_act->flags & FLOW_ACT_IGNORE_FLOW_LEVEL;
	+ u32 action = flow_act->action;
	+
	+ if (dest && (dest->type == MLX5_FLOW_DESTINATION_TYPE_COUNTER))
	+ return counter_is_valid(action);
	+
	+ if (!(action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST))
	+ return true;
	+
	+ if (ignore_level) {
	+ if (ft->type != FS_FT_FDB &&
	+ ft->type != FS_FT_NIC_RX &&
	+ ft->type != FS_FT_NIC_TX)
	+ return false;
	+
	+ if (dest->type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE &&
	+ ft->type != dest->ft->type)
	+ return false;
	+ }
	+
	+ if (!dest \|\| ((dest->type ==
	+ MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE) &&
	+ (dest->ft->level <= ft->level && !ignore_level)))
	+ return false;
	+ return true;
	+}
	+
	+struct match_list {
	+ struct list_head list;
	+ struct mlx5_flow_group *g;
	+};
	+
	+static void free_match_list(struct match_list *head, bool ft_locked)
	+{
	+ struct match_list iter, match_tmp;
	+
	+ list_for_each_entry_safe(iter, match_tmp, &head->list,
	+ list) {
	+ tree_put_node(&iter->g->node, ft_locked);
	+ list_del(&iter->list);
	+ kfree(iter);
	+ }
	+}
	+
	+#define xa_for_each_rcu(xa, index, entry) \
	+ for ((entry) = NULL, (index) = 0; \
	+ ((entry) = xa_next(xa, &index, (entry) != NULL)) != NULL; )
	+
	+static int build_match_list(struct match_list *match_head,
	+ struct mlx5_flow_table *ft,
	+ const struct mlx5_flow_spec *spec,
	+ struct mlx5_flow_group *fg,
	+ bool ft_locked)
	+{
	+ struct mlx5_flow_group *g;
	+ unsigned long id;
	+
	+ rcu_read_lock();
	+ INIT_LIST_HEAD(&match_head->list);
	+ xa_for_each_rcu(&ft->fgs_xa, id, g) {
	+ struct match_list *curr_match;
	+
	+ if (memcmp(&g->mask, spec, sizeof_field(struct mlx5_flow_group,
	+ mask)))
	+ continue;
	+
	+ if (fg && fg != g)
	+ continue;
	+
	+ if (unlikely(!tree_get_node(&g->node)))
	+ continue;
	+
	+ curr_match = kmalloc(sizeof(*curr_match), GFP_ATOMIC);
	+ if (!curr_match) {
	+ rcu_read_unlock();
	+ free_match_list(match_head, ft_locked);
	+ return -ENOMEM;
	+ }
	+ curr_match->g = g;
	+ list_add_tail(&curr_match->list, &match_head->list);
	+ }
	+ rcu_read_unlock();
	+ return 0;
	+}
	+
	+static u64 matched_fgs_get_version(struct list_head *match_head)
	+{
	+ struct match_list *iter;
	+ u64 version = 0;
	+
	+ list_for_each_entry(iter, match_head, list)
	+ version += (u64)atomic_read(&iter->g->node.version);
	+ return version;
	+}
	+
	+static struct fs_fte *
	+lookup_fte_locked(struct mlx5_flow_group *g,
	+ const u32 *match_value,
	+ bool take_write)
	+{
	+ struct fs_fte *fte_tmp;
	+ unsigned long index;
	+
	+ if (take_write)
	+ nested_down_write_ref_node(&g->node, FS_LOCK_PARENT);
	+ else
	+ nested_down_read_ref_node(&g->node, FS_LOCK_PARENT);
	+ xa_for_each_rcu(&g->ftes_xa, index, fte_tmp) {
	+ if (!memcmp(match_value, fte_tmp->val, sizeof_field(struct fs_fte, val)))
	+ break;
	+ }
	+ if (!fte_tmp \|\| !tree_get_node(&fte_tmp->node)) {
	+ fte_tmp = NULL;
	+ goto out;
	+ }
	+ if (!fte_tmp->node.active) {
	+ tree_put_node(&fte_tmp->node, false);
	+ fte_tmp = NULL;
	+ goto out;
	+ }
	+
	+ nested_down_write_ref_node(&fte_tmp->node, FS_LOCK_CHILD);
	+out:
	+ if (take_write)
	+ up_write_ref_node(&g->node, false);
	+ else
	+ up_read_ref_node(&g->node);
	+ return fte_tmp;
	+}
	+
	+static struct mlx5_flow_handle *
	+try_add_to_existing_fg(struct mlx5_flow_table *ft,
	+ struct list_head *match_head,
	+ const struct mlx5_flow_spec *spec,
	+ struct mlx5_flow_act *flow_act,
	+ struct mlx5_flow_destination *dest,
	+ int dest_num,
	+ int ft_version)
	+{
	+ struct mlx5_flow_steering *steering = get_steering(&ft->node);
	+ struct mlx5_flow_group *g;
	+ struct mlx5_flow_handle *rule;
	+ struct match_list *iter;
	+ bool take_write = false;
	+ struct fs_fte *fte;
	+ u64 version = 0;
	+ int err;
	+
	+ fte = alloc_fte(ft, spec, flow_act);
	+ if (IS_ERR(fte))
	+ return ERR_PTR(-ENOMEM);
	+
	+search_again_locked:
	+ if (flow_act->flags & FLOW_ACT_NO_APPEND)
	+ goto skip_search;
	+ version = matched_fgs_get_version(match_head);
	+ /* Try to find an fte with identical match value and attempt update its
	+ * action.
	+ */
	+ list_for_each_entry(iter, match_head, list) {
	+ struct fs_fte *fte_tmp;
	+
	+ g = iter->g;
	+ fte_tmp = lookup_fte_locked(g, spec->match_value, take_write);
	+ if (!fte_tmp)
	+ continue;
	+ rule = add_rule_fg(g, spec, flow_act, dest, dest_num, fte_tmp);
	+ /* No error check needed here, because insert_fte() is not called */
	+ up_write_ref_node(&fte_tmp->node, false);
	+ tree_put_node(&fte_tmp->node, false);
	+ kmem_cache_free(steering->ftes_cache, fte);
	+ return rule;
	+ }
	+
	+skip_search:
	+ /* No group with matching fte found, or we skipped the search.
	+ * Try to add a new fte to any matching fg.
	+ */
	+
	+ /* Check the ft version, for case that new flow group
	+ * was added while the fgs weren't locked
	+ */
	+ if (atomic_read(&ft->node.version) != ft_version) {
	+ rule = ERR_PTR(-EAGAIN);
	+ goto out;
	+ }
	+
	+ /* Check the fgs version. If version have changed it could be that an
	+ * FTE with the same match value was added while the fgs weren't
	+ * locked.
	+ */
	+ if (!(flow_act->flags & FLOW_ACT_NO_APPEND) &&
	+ version != matched_fgs_get_version(match_head)) {
	+ take_write = true;
	+ goto search_again_locked;
	+ }
	+
	+ list_for_each_entry(iter, match_head, list) {
	+ g = iter->g;
	+
	+ nested_down_write_ref_node(&g->node, FS_LOCK_PARENT);
	+
	+ if (!g->node.active) {
	+ up_write_ref_node(&g->node, false);
	+ continue;
	+ }
	+
	+ err = insert_fte(g, fte);
	+ if (err) {
	+ up_write_ref_node(&g->node, false);
	+ if (err == -ENOSPC)
	+ continue;
	+ kmem_cache_free(steering->ftes_cache, fte);
	+ return ERR_PTR(err);
	+ }
	+
	+ nested_down_write_ref_node(&fte->node, FS_LOCK_CHILD);
	+ up_write_ref_node(&g->node, false);
	+ rule = add_rule_fg(g, spec, flow_act, dest, dest_num, fte);
	+ up_write_ref_node(&fte->node, false);
	+ if (IS_ERR(rule))
	+ tree_put_node(&fte->node, false);
	+ return rule;
	+ }
	+ rule = ERR_PTR(-ENOENT);
	+out:
	+ kmem_cache_free(steering->ftes_cache, fte);
	+ return rule;
	+}
	+
	+static struct mlx5_flow_handle *
	+_mlx5_add_flow_rules(struct mlx5_flow_table *ft,
	+ const struct mlx5_flow_spec *spec,
	+ struct mlx5_flow_act *flow_act,
	+ struct mlx5_flow_destination *dest,
	+ int dest_num)
	+
	+{
	+ struct mlx5_flow_steering *steering = get_steering(&ft->node);
	+ struct mlx5_flow_handle *rule;
	+ struct match_list match_head;
	+ struct mlx5_flow_group *g;
	+ bool take_write = false;
	+ struct fs_fte *fte;
	+ int version;
	+ int err;
	+ int i;
	+
	+ if (!check_valid_spec(spec))
	+ return ERR_PTR(-EINVAL);
	+
	+ if (flow_act->fg && ft->autogroup.active)
	+ return ERR_PTR(-EINVAL);
	+
	+ if (dest && dest_num <= 0)
	+ return ERR_PTR(-EINVAL);
	+
	+ for (i = 0; i < dest_num; i++) {
	+ if (!dest_is_valid(&dest[i], flow_act, ft))
	+ return ERR_PTR(-EINVAL);
	+ }
	+ nested_down_read_ref_node(&ft->node, FS_LOCK_GRANDPARENT);
	+search_again_locked:
	+ version = atomic_read(&ft->node.version);
	+
	+ /* Collect all fgs which has a matching match_criteria */
	+ err = build_match_list(&match_head, ft, spec, flow_act->fg, take_write);
	+ if (err) {
	+ if (take_write)
	+ up_write_ref_node(&ft->node, false);
	+ else
	+ up_read_ref_node(&ft->node);
	+ return ERR_PTR(err);
	+ }
	+
	+ if (!take_write)
	+ up_read_ref_node(&ft->node);
	+
	+ rule = try_add_to_existing_fg(ft, &match_head.list, spec, flow_act, dest,
	+ dest_num, version);
	+ free_match_list(&match_head, take_write);
	+ if (!IS_ERR(rule) \|\|
	+ (PTR_ERR(rule) != -ENOENT && PTR_ERR(rule) != -EAGAIN)) {
	+ if (take_write)
	+ up_write_ref_node(&ft->node, false);
	+ return rule;
	+ }
	+
	+ if (!take_write) {
	+ nested_down_write_ref_node(&ft->node, FS_LOCK_GRANDPARENT);
	+ take_write = true;
	+ }
	+
	+ if (PTR_ERR(rule) == -EAGAIN \|\|
	+ version != atomic_read(&ft->node.version))
	+ goto search_again_locked;
	+
	+ g = alloc_auto_flow_group(ft, spec);
	+ if (IS_ERR(g)) {
	+ rule = ERR_CAST(g);
	+ up_write_ref_node(&ft->node, false);
	+ return rule;
	+ }
	+
	+ fte = alloc_fte(ft, spec, flow_act);
	+ if (IS_ERR(fte)) {
	+ up_write_ref_node(&ft->node, false);
	+ err = PTR_ERR(fte);
	+ goto err_alloc_fte;
	+ }
	+
	+ nested_down_write_ref_node(&g->node, FS_LOCK_PARENT);
	+ up_write_ref_node(&ft->node, false);
	+
	+ err = create_auto_flow_group(ft, g);
	+ if (err)
	+ goto err_release_fg;
	+
	+ err = insert_fte(g, fte);
	+ if (err)
	+ goto err_release_fg;
	+
	+ nested_down_write_ref_node(&fte->node, FS_LOCK_CHILD);
	+ up_write_ref_node(&g->node, false);
	+ rule = add_rule_fg(g, spec, flow_act, dest, dest_num, fte);
	+ up_write_ref_node(&fte->node, false);
	+ if (IS_ERR(rule))
	+ tree_put_node(&fte->node, false);
	+ tree_put_node(&g->node, false);
	+ return rule;
	+
	+err_release_fg:
	+ up_write_ref_node(&g->node, false);
	+ kmem_cache_free(steering->ftes_cache, fte);
	+err_alloc_fte:
	+ tree_put_node(&g->node, false);
	+ return ERR_PTR(err);
	+}
	+
	+static bool fwd_next_prio_supported(struct mlx5_flow_table *ft)
	+{
	+ return ((ft->type == FS_FT_NIC_RX) &&
	+ (MLX5_CAP_FLOWTABLE(get_dev(&ft->node), nic_rx_multi_path_tirs)));
	+}
	+
	+struct mlx5_flow_handle *
	+mlx5_add_flow_rules(struct mlx5_flow_table *ft,
	+ const struct mlx5_flow_spec *spec,
	+ struct mlx5_flow_act *flow_act,
	+ struct mlx5_flow_destination *dest,
	+ int num_dest)
	+{
	+ struct mlx5_flow_root_namespace *root = find_root(&ft->node);
	+ static const struct mlx5_flow_spec zero_spec = {};
	+ struct mlx5_flow_destination *gen_dest = NULL;
	+ struct mlx5_flow_table *next_ft = NULL;
	+ struct mlx5_flow_handle *handle = NULL;
	+ u32 sw_action = flow_act->action;
	+ int i;
	+
	+ if (!spec)
	+ spec = &zero_spec;
	+
	+ if (!is_fwd_next_action(sw_action))
	+ return _mlx5_add_flow_rules(ft, spec, flow_act, dest, num_dest);
	+
	+ if (!fwd_next_prio_supported(ft))
	+ return ERR_PTR(-EOPNOTSUPP);
	+
	+ mutex_lock(&root->chain_lock);
	+ next_ft = find_next_fwd_ft(ft, flow_act);
	+ if (!next_ft) {
	+ handle = ERR_PTR(-EOPNOTSUPP);
	+ goto unlock;
	+ }
	+
	+ gen_dest = kcalloc(num_dest + 1, sizeof(*dest),
	+ GFP_KERNEL);
	+ if (!gen_dest) {
	+ handle = ERR_PTR(-ENOMEM);
	+ goto unlock;
	+ }
	+ for (i = 0; i < num_dest; i++)
	+ gen_dest[i] = dest[i];
	+ gen_dest[i].type =
	+ MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
	+ gen_dest[i].ft = next_ft;
	+ dest = gen_dest;
	+ num_dest++;
	+ flow_act->action &= ~(MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO \|
	+ MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS);
	+ flow_act->action \|= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
	+ handle = _mlx5_add_flow_rules(ft, spec, flow_act, dest, num_dest);
	+ if (IS_ERR(handle))
	+ goto unlock;
	+
	+ if (list_empty(&handle->rule[num_dest - 1]->next_ft)) {
	+ mutex_lock(&next_ft->lock);
	+ list_add(&handle->rule[num_dest - 1]->next_ft,
	+ &next_ft->fwd_rules);
	+ mutex_unlock(&next_ft->lock);
	+ handle->rule[num_dest - 1]->sw_action = sw_action;
	+ handle->rule[num_dest - 1]->ft = ft;
	+ }
	+unlock:
	+ mutex_unlock(&root->chain_lock);
	+ kfree(gen_dest);
	+ return handle;
	+}
	+EXPORT_SYMBOL(mlx5_add_flow_rules);
	+
	+void mlx5_del_flow_rules(struct mlx5_flow_handle **pp)
	+{
	+ struct mlx5_flow_handle *handle;
	+ struct fs_fte *fte;
	+ int i;
	+
	+ handle = *pp;
	+ *pp = NULL;
	+ if (IS_ERR_OR_NULL(handle))
	+ return;
	+
	+ /* In order to consolidate the HW changes we lock the FTE for other
	+ * changes, and increase its refcount, in order not to perform the
	+ * "del" functions of the FTE. Will handle them here.
	+ * The removal of the rules is done under locked FTE.
	+ * After removing all the handle's rules, if there are remaining
	+ * rules, it means we just need to modify the FTE in FW, and
	+ * unlock/decrease the refcount we increased before.
	+ * Otherwise, it means the FTE should be deleted. First delete the
	+ * FTE in FW. Then, unlock the FTE, and proceed the tree_put_node of
	+ * the FTE, which will handle the last decrease of the refcount, as
	+ * well as required handling of its parent.
	+ */
	+ fs_get_obj(fte, handle->rule[0]->node.parent);
	+ down_write_ref_node(&fte->node, false);
	+ for (i = handle->num_rules - 1; i >= 0; i--)
	+ tree_remove_node(&handle->rule[i]->node, true);
	+ if (list_empty(&fte->node.children)) {
	+ fte->node.del_hw_func(&fte->node);
	+ /* Avoid double call to del_hw_fte */
	+ fte->node.del_hw_func = NULL;
	+ up_write_ref_node(&fte->node, false);
	+ tree_put_node(&fte->node, false);
	+ } else if (fte->dests_size) {
	+ if (fte->modify_mask)
	+ modify_fte(fte);
	+ up_write_ref_node(&fte->node, false);
	+ } else {
	+ up_write_ref_node(&fte->node, false);
	+ }
	+ kfree(handle);
	+}
	+EXPORT_SYMBOL(mlx5_del_flow_rules);
	+
	+/* Assuming prio->node.children(flow tables) is sorted by level */
	+static struct mlx5_flow_table find_next_ft(struct mlx5_flow_table ft)
	+{
	+ struct fs_prio *prio;
	+
	+ fs_get_obj(prio, ft->node.parent);
	+
	+ if (!list_is_last(&ft->node.list, &prio->node.children))
	+ return list_next_entry(ft, node.list);
	+ return find_next_chained_ft(prio);
	+}
	+
	+static int update_root_ft_destroy(struct mlx5_flow_table *ft)
	+{
	+ struct mlx5_flow_root_namespace *root = find_root(&ft->node);
	+ struct mlx5_ft_underlay_qp *uqp;
	+ struct mlx5_flow_table *new_root_ft = NULL;
	+ int err = 0;
	+ u32 qpn;
	+
	+ if (root->root_ft != ft)
	+ return 0;
	+
	+ new_root_ft = find_next_ft(ft);
	+ if (!new_root_ft) {
	+ root->root_ft = NULL;
	+ return 0;
	+ }
	+
	+ if (list_empty(&root->underlay_qpns)) {
	+ /* Don't set any QPN (zero) in case QPN list is empty */
	+ qpn = 0;
	+ err = root->cmds->update_root_ft(root, new_root_ft,
	+ qpn, false);
	+ } else {
	+ list_for_each_entry(uqp, &root->underlay_qpns, list) {
	+ qpn = uqp->qpn;
	+ err = root->cmds->update_root_ft(root,
	+ new_root_ft, qpn,
	+ false);
	+ if (err)
	+ break;
	+ }
	+ }
	+
	+ if (err)
	+ mlx5_core_warn(root->dev,
	+ "Update root flow table of id(%u) qpn(%d) failed\n",
	+ ft->id, qpn);
	+ else
	+ root->root_ft = new_root_ft;
	+
	+ return 0;
	+}
	+
	+/* Connect flow table from previous priority to
	+ * the next flow table.
	+ */
	+static int disconnect_flow_table(struct mlx5_flow_table *ft)
	+{
	+ struct mlx5_core_dev *dev = get_dev(&ft->node);
	+ struct mlx5_flow_table *next_ft;
	+ struct fs_prio *prio;
	+ int err = 0;
	+
	+ err = update_root_ft_destroy(ft);
	+ if (err)
	+ return err;
	+
	+ fs_get_obj(prio, ft->node.parent);
	+ if (!(list_first_entry(&prio->node.children,
	+ struct mlx5_flow_table,
	+ node.list) == ft))
	+ return 0;
	+
	+ next_ft = find_next_ft(ft);
	+ err = connect_fwd_rules(dev, next_ft, ft);
	+ if (err)
	+ return err;
	+
	+ err = connect_prev_fts(dev, next_ft, prio);
	+ if (err)
	+ mlx5_core_warn(dev, "Failed to disconnect flow table %d\n",
	+ ft->id);
	+ return err;
	+}
	+
	+int mlx5_destroy_flow_table(struct mlx5_flow_table *ft)
	+{
	+ struct mlx5_flow_root_namespace *root = find_root(&ft->node);
	+ int err = 0;
	+
	+ mutex_lock(&root->chain_lock);
	+ if (!(ft->flags & MLX5_FLOW_TABLE_UNMANAGED))
	+ err = disconnect_flow_table(ft);
	+ if (err) {
	+ mutex_unlock(&root->chain_lock);
	+ return err;
	+ }
	+ if (tree_remove_node(&ft->node, false))
	+ mlx5_core_warn(get_dev(&ft->node), "Flow table %d wasn't destroyed, refcount > 1\n",
	+ ft->id);
	+ mutex_unlock(&root->chain_lock);
	+
	+ return err;
	+}
	+EXPORT_SYMBOL(mlx5_destroy_flow_table);
	+
	+void mlx5_destroy_flow_group(struct mlx5_flow_group *fg)
	+{
	+ if (tree_remove_node(&fg->node, false))
	+ mlx5_core_warn(get_dev(&fg->node), "Flow group %d wasn't destroyed, refcount > 1\n",
	+ fg->id);
	+}
	+EXPORT_SYMBOL(mlx5_destroy_flow_group);
	+
	+struct mlx5_flow_namespace mlx5_get_fdb_sub_ns(struct mlx5_core_dev dev,
	+ int n)
	+{
	+ struct mlx5_flow_steering *steering = dev->priv.steering;
	+
	+ if (!steering \|\| !steering->fdb_sub_ns)
	+ return NULL;
	+
	+ return steering->fdb_sub_ns[n];
	+}
	+EXPORT_SYMBOL(mlx5_get_fdb_sub_ns);
	+
	+static bool is_nic_rx_ns(enum mlx5_flow_namespace_type type)
	+{
	+ switch (type) {
	+ case MLX5_FLOW_NAMESPACE_BYPASS:
	+ case MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC:
	+ case MLX5_FLOW_NAMESPACE_LAG:
	+ case MLX5_FLOW_NAMESPACE_OFFLOADS:
	+ case MLX5_FLOW_NAMESPACE_ETHTOOL:
	+ case MLX5_FLOW_NAMESPACE_KERNEL:
	+ case MLX5_FLOW_NAMESPACE_LEFTOVERS:
	+ case MLX5_FLOW_NAMESPACE_ANCHOR:
	+ return true;
	+ default:
	+ return false;
	+ }
	+}
	+
	+struct mlx5_flow_namespace mlx5_get_flow_namespace(struct mlx5_core_dev dev,
	+ enum mlx5_flow_namespace_type type)
	+{
	+ struct mlx5_flow_steering *steering = dev->priv.steering;
	+ struct mlx5_flow_root_namespace *root_ns;
	+ int prio = 0;
	+ struct fs_prio *fs_prio;
	+ struct mlx5_flow_namespace *ns;
	+
	+ if (!steering)
	+ return NULL;
	+
	+ switch (type) {
	+ case MLX5_FLOW_NAMESPACE_FDB:
	+ if (steering->fdb_root_ns)
	+ return &steering->fdb_root_ns->ns;
	+ return NULL;
	+ case MLX5_FLOW_NAMESPACE_PORT_SEL:
	+ if (steering->port_sel_root_ns)
	+ return &steering->port_sel_root_ns->ns;
	+ return NULL;
	+ case MLX5_FLOW_NAMESPACE_SNIFFER_RX:
	+ if (steering->sniffer_rx_root_ns)
	+ return &steering->sniffer_rx_root_ns->ns;
	+ return NULL;
	+ case MLX5_FLOW_NAMESPACE_SNIFFER_TX:
	+ if (steering->sniffer_tx_root_ns)
	+ return &steering->sniffer_tx_root_ns->ns;
	+ return NULL;
	+ case MLX5_FLOW_NAMESPACE_FDB_BYPASS:
	+ root_ns = steering->fdb_root_ns;
	+ prio = FDB_BYPASS_PATH;
	+ break;
	+ case MLX5_FLOW_NAMESPACE_EGRESS:
	+ case MLX5_FLOW_NAMESPACE_EGRESS_IPSEC:
	+ case MLX5_FLOW_NAMESPACE_EGRESS_MACSEC:
	+ root_ns = steering->egress_root_ns;
	+ prio = type - MLX5_FLOW_NAMESPACE_EGRESS;
	+ break;
	+ case MLX5_FLOW_NAMESPACE_RDMA_RX:
	+ root_ns = steering->rdma_rx_root_ns;
	+ prio = RDMA_RX_BYPASS_PRIO;
	+ break;
	+ case MLX5_FLOW_NAMESPACE_RDMA_RX_KERNEL:
	+ root_ns = steering->rdma_rx_root_ns;
	+ prio = RDMA_RX_KERNEL_PRIO;
	+ break;
	+ case MLX5_FLOW_NAMESPACE_RDMA_TX:
	+ root_ns = steering->rdma_tx_root_ns;
	+ break;
	+ case MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS:
	+ root_ns = steering->rdma_rx_root_ns;
	+ prio = RDMA_RX_COUNTERS_PRIO;
	+ break;
	+ case MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS:
	+ root_ns = steering->rdma_tx_root_ns;
	+ prio = RDMA_TX_COUNTERS_PRIO;
	+ break;
	+ case MLX5_FLOW_NAMESPACE_RDMA_RX_IPSEC:
	+ root_ns = steering->rdma_rx_root_ns;
	+ prio = RDMA_RX_IPSEC_PRIO;
	+ break;
	+ case MLX5_FLOW_NAMESPACE_RDMA_TX_IPSEC:
	+ root_ns = steering->rdma_tx_root_ns;
	+ prio = RDMA_TX_IPSEC_PRIO;
	+ break;
	+ default: /* Must be NIC RX */
	+ WARN_ON(!is_nic_rx_ns(type));
	+ root_ns = steering->root_ns;
	+ prio = type;
	+ break;
	+ }
	+
	+ if (!root_ns)
	+ return NULL;
	+
	+ fs_prio = find_prio(&root_ns->ns, prio);
	+ if (!fs_prio)
	+ return NULL;
	+
	+ ns = list_first_entry(&fs_prio->node.children,
	+ typeof(*ns),
	+ node.list);
	+
	+ return ns;
	+}
	+EXPORT_SYMBOL(mlx5_get_flow_namespace);
	+
	+struct mlx5_flow_namespace mlx5_get_flow_vport_acl_namespace(struct mlx5_core_dev dev,
	+ enum mlx5_flow_namespace_type type,
	+ int vport)
	+{
	+ struct mlx5_flow_steering *steering = dev->priv.steering;
	+
	+ if (!steering)
	+ return NULL;
	+
	+ switch (type) {
	+ case MLX5_FLOW_NAMESPACE_ESW_EGRESS:
	+ if (vport >= steering->esw_egress_acl_vports)
	+ return NULL;
	+ if (steering->esw_egress_root_ns &&
	+ steering->esw_egress_root_ns[vport])
	+ return &steering->esw_egress_root_ns[vport]->ns;
	+ else
	+ return NULL;
	+ case MLX5_FLOW_NAMESPACE_ESW_INGRESS:
	+ if (vport >= steering->esw_ingress_acl_vports)
	+ return NULL;
	+ if (steering->esw_ingress_root_ns &&
	+ steering->esw_ingress_root_ns[vport])
	+ return &steering->esw_ingress_root_ns[vport]->ns;
	+ else
	+ return NULL;
	+ default:
	+ return NULL;
	+ }
	+}
	+
	+static struct fs_prio _fs_create_prio(struct mlx5_flow_namespace ns,
	+ unsigned int prio,
	+ int num_levels,
	+ enum fs_node_type type)
	+{
	+ struct fs_prio *fs_prio;
	+
	+ fs_prio = kzalloc(sizeof(*fs_prio), GFP_KERNEL);
	+ if (!fs_prio)
	+ return ERR_PTR(-ENOMEM);
	+
	+ fs_prio->node.type = type;
	+ tree_init_node(&fs_prio->node, NULL, del_sw_prio);
	+ tree_add_node(&fs_prio->node, &ns->node);
	+ fs_prio->num_levels = num_levels;
	+ fs_prio->prio = prio;
	+ list_add_tail(&fs_prio->node.list, &ns->node.children);
	+
	+ return fs_prio;
	+}
	+
	+static struct fs_prio fs_create_prio_chained(struct mlx5_flow_namespace ns,
	+ unsigned int prio,
	+ int num_levels)
	+{
	+ return _fs_create_prio(ns, prio, num_levels, FS_TYPE_PRIO_CHAINS);
	+}
	+
	+static struct fs_prio fs_create_prio(struct mlx5_flow_namespace ns,
	+ unsigned int prio, int num_levels)
	+{
	+ return _fs_create_prio(ns, prio, num_levels, FS_TYPE_PRIO);
	+}
	+
	+static struct mlx5_flow_namespace *fs_init_namespace(struct mlx5_flow_namespace
	+ *ns)
	+{
	+ ns->node.type = FS_TYPE_NAMESPACE;
	+
	+ return ns;
	+}
	+
	+static struct mlx5_flow_namespace fs_create_namespace(struct fs_prio prio,
	+ int def_miss_act)
	+{
	+ struct mlx5_flow_namespace *ns;
	+
	+ ns = kzalloc(sizeof(*ns), GFP_KERNEL);
	+ if (!ns)
	+ return ERR_PTR(-ENOMEM);
	+
	+ fs_init_namespace(ns);
	+ ns->def_miss_action = def_miss_act;
	+ tree_init_node(&ns->node, NULL, del_sw_ns);
	+ tree_add_node(&ns->node, &prio->node);
	+ list_add_tail(&ns->node.list, &prio->node.children);
	+
	+ return ns;
	+}
	+
	+static int create_leaf_prios(struct mlx5_flow_namespace *ns, int prio,
	+ struct init_tree_node *prio_metadata)
	+{
	+ struct fs_prio *fs_prio;
	+ int i;
	+
	+ for (i = 0; i < prio_metadata->num_leaf_prios; i++) {
	+ fs_prio = fs_create_prio(ns, prio++, prio_metadata->num_levels);
	+ if (IS_ERR(fs_prio))
	+ return PTR_ERR(fs_prio);
	+ }
	+ return 0;
	+}
	+
	+#define FLOW_TABLE_BIT_SZ 1
	+#define GET_FLOW_TABLE_CAP(dev, offset) \
	+ ((be32_to_cpu(((__be32 )(dev->hca_caps_cur[MLX5_CAP_FLOW_TABLE]) + \
	+ offset / 32)) >> \
	+ (32 - FLOW_TABLE_BIT_SZ - (offset & 0x1f))) & FLOW_TABLE_BIT_SZ)
	+static bool has_required_caps(struct mlx5_core_dev dev, struct node_caps caps)
	+{
	+ int i;
	+
	+ for (i = 0; i < caps->arr_sz; i++) {
	+ if (!GET_FLOW_TABLE_CAP(dev, caps->caps[i]))
	+ return false;
	+ }
	+ return true;
	+}
	+
	+static int init_root_tree_recursive(struct mlx5_flow_steering *steering,
	+ struct init_tree_node *init_node,
	+ struct fs_node *fs_parent_node,
	+ struct init_tree_node *init_parent_node,
	+ int prio)
	+{
	+ int max_ft_level = MLX5_CAP_FLOWTABLE(steering->dev,
	+ flow_table_properties_nic_receive.
	+ max_ft_level);
	+ struct mlx5_flow_namespace *fs_ns;
	+ struct fs_prio *fs_prio;
	+ struct fs_node *base;
	+ int i;
	+ int err;
	+
	+ if (init_node->type == FS_TYPE_PRIO) {
	+ if ((init_node->min_ft_level > max_ft_level) \|\|
	+ !has_required_caps(steering->dev, &init_node->caps))
	+ return 0;
	+
	+ fs_get_obj(fs_ns, fs_parent_node);
	+ if (init_node->num_leaf_prios)
	+ return create_leaf_prios(fs_ns, prio, init_node);
	+ fs_prio = fs_create_prio(fs_ns, prio, init_node->num_levels);
	+ if (IS_ERR(fs_prio))
	+ return PTR_ERR(fs_prio);
	+ base = &fs_prio->node;
	+ } else if (init_node->type == FS_TYPE_NAMESPACE) {
	+ fs_get_obj(fs_prio, fs_parent_node);
	+ fs_ns = fs_create_namespace(fs_prio, init_node->def_miss_action);
	+ if (IS_ERR(fs_ns))
	+ return PTR_ERR(fs_ns);
	+ base = &fs_ns->node;
	+ } else {
	+ return -EINVAL;
	+ }
	+ prio = 0;
	+ for (i = 0; i < init_node->ar_size; i++) {
	+ err = init_root_tree_recursive(steering, &init_node->children[i],
	+ base, init_node, prio);
	+ if (err)
	+ return err;
	+ if (init_node->children[i].type == FS_TYPE_PRIO &&
	+ init_node->children[i].num_leaf_prios) {
	+ prio += init_node->children[i].num_leaf_prios;
	+ }
	+ }
	+
	+ return 0;
	+}
	+
	+static int init_root_tree(struct mlx5_flow_steering *steering,
	+ struct init_tree_node *init_node,
	+ struct fs_node *fs_parent_node)
	+{
	+ int err;
	+ int i;
	+
	+ for (i = 0; i < init_node->ar_size; i++) {
	+ err = init_root_tree_recursive(steering, &init_node->children[i],
	+ fs_parent_node,
	+ init_node, i);
	+ if (err)
	+ return err;
	+ }
	+ return 0;
	+}
	+
	+static void del_sw_root_ns(struct fs_node *node)
	+{
	+ struct mlx5_flow_root_namespace *root_ns;
	+ struct mlx5_flow_namespace *ns;
	+
	+ fs_get_obj(ns, node);
	+ root_ns = container_of(ns, struct mlx5_flow_root_namespace, ns);
	+ mutex_destroy(&root_ns->chain_lock);
	+ kfree(node);
	+}
	+
	+static struct mlx5_flow_root_namespace
	+create_root_ns(struct mlx5_flow_steering steering,
	+ enum fs_flow_table_type table_type)
	+{
	+ const struct mlx5_flow_cmds *cmds = mlx5_fs_cmd_get_default(table_type);
	+ struct mlx5_flow_root_namespace *root_ns;
	+ struct mlx5_flow_namespace *ns;
	+
	+ /* Create the root namespace */
	+ root_ns = kzalloc(sizeof(*root_ns), GFP_KERNEL);
	+ if (!root_ns)
	+ return NULL;
	+
	+ root_ns->dev = steering->dev;
	+ root_ns->table_type = table_type;
	+ root_ns->cmds = cmds;
	+
	+ INIT_LIST_HEAD(&root_ns->underlay_qpns);
	+
	+ ns = &root_ns->ns;
	+ fs_init_namespace(ns);
	+ mutex_init(&root_ns->chain_lock);
	+ tree_init_node(&ns->node, NULL, del_sw_root_ns);
	+ tree_add_node(&ns->node, NULL);
	+
	+ return root_ns;
	+}
	+
	+static void set_prio_attrs_in_prio(struct fs_prio *prio, int acc_level);
	+
	+static int set_prio_attrs_in_ns(struct mlx5_flow_namespace *ns, int acc_level)
	+{
	+ struct fs_prio *prio;
	+
	+ fs_for_each_prio(prio, ns) {
	+ /* This updates prio start_level and num_levels */
	+ set_prio_attrs_in_prio(prio, acc_level);
	+ acc_level += prio->num_levels;
	+ }
	+ return acc_level;
	+}
	+
	+static void set_prio_attrs_in_prio(struct fs_prio *prio, int acc_level)
	+{
	+ struct mlx5_flow_namespace *ns;
	+ int acc_level_ns = acc_level;
	+
	+ prio->start_level = acc_level;
	+ fs_for_each_ns(ns, prio) {
	+ /* This updates start_level and num_levels of ns's priority descendants */
	+ acc_level_ns = set_prio_attrs_in_ns(ns, acc_level);
	+
	+ /* If this a prio with chains, and we can jump from one chain
	+ * (namespace) to another, so we accumulate the levels
	+ */
	+ if (prio->node.type == FS_TYPE_PRIO_CHAINS)
	+ acc_level = acc_level_ns;
	+ }
	+
	+ if (!prio->num_levels)
	+ prio->num_levels = acc_level_ns - prio->start_level;
	+ WARN_ON(prio->num_levels < acc_level_ns - prio->start_level);
	+}
	+
	+static void set_prio_attrs(struct mlx5_flow_root_namespace *root_ns)
	+{
	+ struct mlx5_flow_namespace *ns = &root_ns->ns;
	+ struct fs_prio *prio;
	+ int start_level = 0;
	+
	+ fs_for_each_prio(prio, ns) {
	+ set_prio_attrs_in_prio(prio, start_level);
	+ start_level += prio->num_levels;
	+ }
	+}
	+
	+#define ANCHOR_PRIO 0
	+#define ANCHOR_SIZE 1
	+#define ANCHOR_LEVEL 0
	+static int create_anchor_flow_table(struct mlx5_flow_steering *steering)
	+{
	+ struct mlx5_flow_namespace *ns = NULL;
	+ struct mlx5_flow_table_attr ft_attr = {};
	+ struct mlx5_flow_table *ft;
	+
	+ ns = mlx5_get_flow_namespace(steering->dev, MLX5_FLOW_NAMESPACE_ANCHOR);
	+ if (WARN_ON(!ns))
	+ return -EINVAL;
	+
	+ ft_attr.max_fte = ANCHOR_SIZE;
	+ ft_attr.level = ANCHOR_LEVEL;
	+ ft_attr.prio = ANCHOR_PRIO;
	+
	+ ft = mlx5_create_flow_table(ns, &ft_attr);
	+ if (IS_ERR(ft)) {
	+ mlx5_core_err(steering->dev, "Failed to create last anchor flow table");
	+ return PTR_ERR(ft);
	+ }
	+ return 0;
	+}
	+
	+static int init_root_ns(struct mlx5_flow_steering *steering)
	+{
	+ int err;
	+
	+ steering->root_ns = create_root_ns(steering, FS_FT_NIC_RX);
	+ if (!steering->root_ns)
	+ return -ENOMEM;
	+
	+ err = init_root_tree(steering, &root_fs, &steering->root_ns->ns.node);
	+ if (err)
	+ goto out_err;
	+
	+ set_prio_attrs(steering->root_ns);
	+ err = create_anchor_flow_table(steering);
	+ if (err)
	+ goto out_err;
	+
	+ return 0;
	+
	+out_err:
	+ cleanup_root_ns(steering->root_ns);
	+ steering->root_ns = NULL;
	+ return err;
	+}
	+
	+static void clean_tree(struct fs_node *node)
	+{
	+ if (node) {
	+ struct fs_node *iter;
	+ struct fs_node *temp;
	+
	+ tree_get_node(node);
	+ list_for_each_entry_safe(iter, temp, &node->children, list)
	+ clean_tree(iter);
	+ tree_put_node(node, false);
	+ tree_remove_node(node, false);
	+ }
	+}
	+
	+static void cleanup_root_ns(struct mlx5_flow_root_namespace *root_ns)
	+{
	+ if (!root_ns)
	+ return;
	+
	+ clean_tree(&root_ns->ns.node);
	+}
	+
	+static int init_sniffer_tx_root_ns(struct mlx5_flow_steering *steering)
	+{
	+ struct fs_prio *prio;
	+
	+ steering->sniffer_tx_root_ns = create_root_ns(steering, FS_FT_SNIFFER_TX);
	+ if (!steering->sniffer_tx_root_ns)
	+ return -ENOMEM;
	+
	+ /* Create single prio */
	+ prio = fs_create_prio(&steering->sniffer_tx_root_ns->ns, 0, 1);
	+ return PTR_ERR_OR_ZERO(prio);
	+}
	+
	+static int init_sniffer_rx_root_ns(struct mlx5_flow_steering *steering)
	+{
	+ struct fs_prio *prio;
	+
	+ steering->sniffer_rx_root_ns = create_root_ns(steering, FS_FT_SNIFFER_RX);
	+ if (!steering->sniffer_rx_root_ns)
	+ return -ENOMEM;
	+
	+ /* Create single prio */
	+ prio = fs_create_prio(&steering->sniffer_rx_root_ns->ns, 0, 1);
	+ return PTR_ERR_OR_ZERO(prio);
	+}
	+
	+#define PORT_SEL_NUM_LEVELS 3
	+static int init_port_sel_root_ns(struct mlx5_flow_steering *steering)
	+{
	+ struct fs_prio *prio;
	+
	+ steering->port_sel_root_ns = create_root_ns(steering, FS_FT_PORT_SEL);
	+ if (!steering->port_sel_root_ns)
	+ return -ENOMEM;
	+
	+ /* Create single prio */
	+ prio = fs_create_prio(&steering->port_sel_root_ns->ns, 0,
	+ PORT_SEL_NUM_LEVELS);
	+ return PTR_ERR_OR_ZERO(prio);
	+}
	+
	+static int init_rdma_rx_root_ns(struct mlx5_flow_steering *steering)
	+{
	+ int err;
	+
	+ steering->rdma_rx_root_ns = create_root_ns(steering, FS_FT_RDMA_RX);
	+ if (!steering->rdma_rx_root_ns)
	+ return -ENOMEM;
	+
	+ err = init_root_tree(steering, &rdma_rx_root_fs,
	+ &steering->rdma_rx_root_ns->ns.node);
	+ if (err)
	+ goto out_err;
	+
	+ set_prio_attrs(steering->rdma_rx_root_ns);
	+
	+ return 0;
	+
	+out_err:
	+ cleanup_root_ns(steering->rdma_rx_root_ns);
	+ steering->rdma_rx_root_ns = NULL;
	+ return err;
	+}
	+
	+static int init_rdma_tx_root_ns(struct mlx5_flow_steering *steering)
	+{
	+ int err;
	+
	+ steering->rdma_tx_root_ns = create_root_ns(steering, FS_FT_RDMA_TX);
	+ if (!steering->rdma_tx_root_ns)
	+ return -ENOMEM;
	+
	+ err = init_root_tree(steering, &rdma_tx_root_fs,
	+ &steering->rdma_tx_root_ns->ns.node);
	+ if (err)
	+ goto out_err;
	+
	+ set_prio_attrs(steering->rdma_tx_root_ns);
	+
	+ return 0;
	+
	+out_err:
	+ cleanup_root_ns(steering->rdma_tx_root_ns);
	+ steering->rdma_tx_root_ns = NULL;
	+ return err;
	+}
	+
	+/* FT and tc chains are stored in the same array so we can re-use the
	+ * mlx5_get_fdb_sub_ns() and tc api for FT chains.
	+ * When creating a new ns for each chain store it in the first available slot.
	+ * Assume tc chains are created and stored first and only then the FT chain.
	+ */
	+static void store_fdb_sub_ns_prio_chain(struct mlx5_flow_steering *steering,
	+ struct mlx5_flow_namespace *ns)
	+{
	+ int chain = 0;
	+
	+ while (steering->fdb_sub_ns[chain])
	+ ++chain;
	+
	+ steering->fdb_sub_ns[chain] = ns;
	+}
	+
	+static int create_fdb_sub_ns_prio_chain(struct mlx5_flow_steering *steering,
	+ struct fs_prio *maj_prio)
	+{
	+ struct mlx5_flow_namespace *ns;
	+ struct fs_prio *min_prio;
	+ int prio;
	+
	+ ns = fs_create_namespace(maj_prio, MLX5_FLOW_TABLE_MISS_ACTION_DEF);
	+ if (IS_ERR(ns))
	+ return PTR_ERR(ns);
	+
	+ for (prio = 0; prio < FDB_TC_MAX_PRIO; prio++) {
	+ min_prio = fs_create_prio(ns, prio, FDB_TC_LEVELS_PER_PRIO);
	+ if (IS_ERR(min_prio))
	+ return PTR_ERR(min_prio);
	+ }
	+
	+ store_fdb_sub_ns_prio_chain(steering, ns);
	+
	+ return 0;
	+}
	+
	+static int create_fdb_chains(struct mlx5_flow_steering *steering,
	+ int fs_prio,
	+ int chains)
	+{
	+ struct fs_prio *maj_prio;
	+ int levels;
	+ int chain;
	+ int err;
	+
	+ levels = FDB_TC_LEVELS_PER_PRIO * FDB_TC_MAX_PRIO * chains;
	+ maj_prio = fs_create_prio_chained(&steering->fdb_root_ns->ns,
	+ fs_prio,
	+ levels);
	+ if (IS_ERR(maj_prio))
	+ return PTR_ERR(maj_prio);
	+
	+ for (chain = 0; chain < chains; chain++) {
	+ err = create_fdb_sub_ns_prio_chain(steering, maj_prio);
	+ if (err)
	+ return err;
	+ }
	+
	+ return 0;
	+}
	+
	+static int create_fdb_fast_path(struct mlx5_flow_steering *steering)
	+{
	+ int err;
	+
	+ steering->fdb_sub_ns = kcalloc(FDB_NUM_CHAINS,
	+ sizeof(*steering->fdb_sub_ns),
	+ GFP_KERNEL);
	+ if (!steering->fdb_sub_ns)
	+ return -ENOMEM;
	+
	+ err = create_fdb_chains(steering, FDB_TC_OFFLOAD, FDB_TC_MAX_CHAIN + 1);
	+ if (err)
	+ return err;
	+
	+ err = create_fdb_chains(steering, FDB_FT_OFFLOAD, 1);
	+ if (err)
	+ return err;
	+
	+ return 0;
	+}
	+
	+static int create_fdb_bypass(struct mlx5_flow_steering *steering)
	+{
	+ struct mlx5_flow_namespace *ns;
	+ struct fs_prio *prio;
	+ int i;
	+
	+ prio = fs_create_prio(&steering->fdb_root_ns->ns, FDB_BYPASS_PATH, 0);
	+ if (IS_ERR(prio))
	+ return PTR_ERR(prio);
	+
	+ ns = fs_create_namespace(prio, MLX5_FLOW_TABLE_MISS_ACTION_DEF);
	+ if (IS_ERR(ns))
	+ return PTR_ERR(ns);
	+
	+ for (i = 0; i < MLX5_BY_PASS_NUM_REGULAR_PRIOS; i++) {
	+ prio = fs_create_prio(ns, i, 1);
	+ if (IS_ERR(prio))
	+ return PTR_ERR(prio);
	+ }
	+ return 0;
	+}
	+
	+static void cleanup_fdb_root_ns(struct mlx5_flow_steering *steering)
	+{
	+ cleanup_root_ns(steering->fdb_root_ns);
	+ steering->fdb_root_ns = NULL;
	+ kfree(steering->fdb_sub_ns);
	+ steering->fdb_sub_ns = NULL;
	+}
	+
	+static int init_fdb_root_ns(struct mlx5_flow_steering *steering)
	+{
	+ struct fs_prio *maj_prio;
	+ int err;
	+
	+ steering->fdb_root_ns = create_root_ns(steering, FS_FT_FDB);
	+ if (!steering->fdb_root_ns)
	+ return -ENOMEM;
	+
	+ err = create_fdb_bypass(steering);
	+ if (err)
	+ goto out_err;
	+
	+ err = create_fdb_fast_path(steering);
	+ if (err)
	+ goto out_err;
	+
	+ maj_prio = fs_create_prio(&steering->fdb_root_ns->ns, FDB_TC_MISS, 1);
	+ if (IS_ERR(maj_prio)) {
	+ err = PTR_ERR(maj_prio);
	+ goto out_err;
	+ }
	+
	+ maj_prio = fs_create_prio(&steering->fdb_root_ns->ns, FDB_BR_OFFLOAD, 4);
	+ if (IS_ERR(maj_prio)) {
	+ err = PTR_ERR(maj_prio);
	+ goto out_err;
	+ }
	+
	+ maj_prio = fs_create_prio(&steering->fdb_root_ns->ns, FDB_SLOW_PATH, 1);
	+ if (IS_ERR(maj_prio)) {
	+ err = PTR_ERR(maj_prio);
	+ goto out_err;
	+ }
	+
	+ /* We put this priority last, knowing that nothing will get here
	+ * unless explicitly forwarded to. This is possible because the
	+ * slow path tables have catch all rules and nothing gets passed
	+ * those tables.
	+ */
	+ maj_prio = fs_create_prio(&steering->fdb_root_ns->ns, FDB_PER_VPORT, 1);
	+ if (IS_ERR(maj_prio)) {
	+ err = PTR_ERR(maj_prio);
	+ goto out_err;
	+ }
	+
	+ set_prio_attrs(steering->fdb_root_ns);
	+ return 0;
	+
	+out_err:
	+ cleanup_fdb_root_ns(steering);
	+ return err;
	+}
	+
	+static int init_egress_acl_root_ns(struct mlx5_flow_steering *steering, int vport)
	+{
	+ struct fs_prio *prio;
	+
	+ steering->esw_egress_root_ns[vport] = create_root_ns(steering, FS_FT_ESW_EGRESS_ACL);
	+ if (!steering->esw_egress_root_ns[vport])
	+ return -ENOMEM;
	+
	+ /* create 1 prio*/
	+ prio = fs_create_prio(&steering->esw_egress_root_ns[vport]->ns, 0, 1);
	+ return PTR_ERR_OR_ZERO(prio);
	+}
	+
	+static int init_ingress_acl_root_ns(struct mlx5_flow_steering *steering, int vport)
	+{
	+ struct fs_prio *prio;
	+
	+ steering->esw_ingress_root_ns[vport] = create_root_ns(steering, FS_FT_ESW_INGRESS_ACL);
	+ if (!steering->esw_ingress_root_ns[vport])
	+ return -ENOMEM;
	+
	+ /* create 1 prio*/
	+ prio = fs_create_prio(&steering->esw_ingress_root_ns[vport]->ns, 0, 1);
	+ return PTR_ERR_OR_ZERO(prio);
	+}
	+
	+int mlx5_fs_egress_acls_init(struct mlx5_core_dev *dev, int total_vports)
	+{
	+ struct mlx5_flow_steering *steering = dev->priv.steering;
	+ int err;
	+ int i;
	+
	+ steering->esw_egress_root_ns =
	+ kcalloc(total_vports,
	+ sizeof(*steering->esw_egress_root_ns),
	+ GFP_KERNEL);
	+ if (!steering->esw_egress_root_ns)
	+ return -ENOMEM;
	+
	+ for (i = 0; i < total_vports; i++) {
	+ err = init_egress_acl_root_ns(steering, i);
	+ if (err)
	+ goto cleanup_root_ns;
	+ }
	+ steering->esw_egress_acl_vports = total_vports;
	+ return 0;
	+
	+cleanup_root_ns:
	+ for (i--; i >= 0; i--)
	+ cleanup_root_ns(steering->esw_egress_root_ns[i]);
	+ kfree(steering->esw_egress_root_ns);
	+ steering->esw_egress_root_ns = NULL;
	+ return err;
	+}
	+
	+void mlx5_fs_egress_acls_cleanup(struct mlx5_core_dev *dev)
	+{
	+ struct mlx5_flow_steering *steering = dev->priv.steering;
	+ int i;
	+
	+ if (!steering->esw_egress_root_ns)
	+ return;
	+
	+ for (i = 0; i < steering->esw_egress_acl_vports; i++)
	+ cleanup_root_ns(steering->esw_egress_root_ns[i]);
	+
	+ kfree(steering->esw_egress_root_ns);
	+ steering->esw_egress_root_ns = NULL;
	+}
	+
	+int mlx5_fs_ingress_acls_init(struct mlx5_core_dev *dev, int total_vports)
	+{
	+ struct mlx5_flow_steering *steering = dev->priv.steering;
	+ int err;
	+ int i;
	+
	+ steering->esw_ingress_root_ns =
	+ kcalloc(total_vports,
	+ sizeof(*steering->esw_ingress_root_ns),
	+ GFP_KERNEL);
	+ if (!steering->esw_ingress_root_ns)
	+ return -ENOMEM;
	+
	+ for (i = 0; i < total_vports; i++) {
	+ err = init_ingress_acl_root_ns(steering, i);
	+ if (err)
	+ goto cleanup_root_ns;
	+ }
	+ steering->esw_ingress_acl_vports = total_vports;
	+ return 0;
	+
	+cleanup_root_ns:
	+ for (i--; i >= 0; i--)
	+ cleanup_root_ns(steering->esw_ingress_root_ns[i]);
	+ kfree(steering->esw_ingress_root_ns);
	+ steering->esw_ingress_root_ns = NULL;
	+ return err;
	+}
	+
	+void mlx5_fs_ingress_acls_cleanup(struct mlx5_core_dev *dev)
	+{
	+ struct mlx5_flow_steering *steering = dev->priv.steering;
	+ int i;
	+
	+ if (!steering->esw_ingress_root_ns)
	+ return;
	+
	+ for (i = 0; i < steering->esw_ingress_acl_vports; i++)
	+ cleanup_root_ns(steering->esw_ingress_root_ns[i]);
	+
	+ kfree(steering->esw_ingress_root_ns);
	+ steering->esw_ingress_root_ns = NULL;
	+}
	+
	+u32 mlx5_fs_get_capabilities(struct mlx5_core_dev *dev, enum mlx5_flow_namespace_type type)
	+{
	+ struct mlx5_flow_root_namespace *root;
	+ struct mlx5_flow_namespace *ns;
	+
	+ ns = mlx5_get_flow_namespace(dev, type);
	+ if (!ns)
	+ return 0;
	+
	+ root = find_root(&ns->node);
	+ if (!root)
	+ return 0;
	+
	+ return root->cmds->get_capabilities(root, root->table_type);
	+}
	+
	+static int init_egress_root_ns(struct mlx5_flow_steering *steering)
	+{
	+ int err;
	+
	+ steering->egress_root_ns = create_root_ns(steering,
	+ FS_FT_NIC_TX);
	+ if (!steering->egress_root_ns)
	+ return -ENOMEM;
	+
	+ err = init_root_tree(steering, &egress_root_fs,
	+ &steering->egress_root_ns->ns.node);
	+ if (err)
	+ goto cleanup;
	+ set_prio_attrs(steering->egress_root_ns);
	+ return 0;
	+cleanup:
	+ cleanup_root_ns(steering->egress_root_ns);
	+ steering->egress_root_ns = NULL;
	+ return err;
	+}
	+
	+void mlx5_fs_core_cleanup(struct mlx5_core_dev *dev)
	+{
	+ struct mlx5_flow_steering *steering = dev->priv.steering;
	+
	+ cleanup_root_ns(steering->root_ns);
	+ cleanup_fdb_root_ns(steering);
	+ cleanup_root_ns(steering->port_sel_root_ns);
	+ cleanup_root_ns(steering->sniffer_rx_root_ns);
	+ cleanup_root_ns(steering->sniffer_tx_root_ns);
	+ cleanup_root_ns(steering->rdma_rx_root_ns);
	+ cleanup_root_ns(steering->rdma_tx_root_ns);
	+ cleanup_root_ns(steering->egress_root_ns);
	+}
	+
	+int mlx5_fs_core_init(struct mlx5_core_dev *dev)
	+{
	+ struct mlx5_flow_steering *steering = dev->priv.steering;
	+ int err;
	+
	+ if ((((MLX5_CAP_GEN(dev, port_type) == MLX5_CAP_PORT_TYPE_ETH) &&
	+ (MLX5_CAP_GEN(dev, nic_flow_table))) \|\|
	+ ((MLX5_CAP_GEN(dev, port_type) == MLX5_CAP_PORT_TYPE_IB) &&
	+ MLX5_CAP_GEN(dev, ipoib_enhanced_offloads))) &&
	+ MLX5_CAP_FLOWTABLE_NIC_RX(dev, ft_support)) {
	+ err = init_root_ns(steering);
	+ if (err)
	+ goto err;
	+ }
	+
	+ if (MLX5_ESWITCH_MANAGER(dev)) {
	+ if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev, ft_support)) {
	+ err = init_fdb_root_ns(steering);
	+ if (err)
	+ goto err;
	+ }
	+ }
	+
	+ if (MLX5_CAP_FLOWTABLE_SNIFFER_RX(dev, ft_support)) {
	+ err = init_sniffer_rx_root_ns(steering);
	+ if (err)
	+ goto err;
	+ }
	+
	+ if (MLX5_CAP_FLOWTABLE_SNIFFER_TX(dev, ft_support)) {
	+ err = init_sniffer_tx_root_ns(steering);
	+ if (err)
	+ goto err;
	+ }
	+
	+ if (MLX5_CAP_FLOWTABLE_PORT_SELECTION(dev, ft_support)) {
	+ err = init_port_sel_root_ns(steering);
	+ if (err)
	+ goto err;
	+ }
	+
	+ if (MLX5_CAP_FLOWTABLE_RDMA_RX(dev, ft_support) &&
	+ MLX5_CAP_FLOWTABLE_RDMA_RX(dev, table_miss_action_domain)) {
	+ err = init_rdma_rx_root_ns(steering);
	+ if (err)
	+ goto err;
	+ }
	+
	+ if (MLX5_CAP_FLOWTABLE_RDMA_TX(dev, ft_support)) {
	+ err = init_rdma_tx_root_ns(steering);
	+ if (err)
	+ goto err;
	+ }
	+
	+ if (MLX5_CAP_FLOWTABLE_NIC_TX(dev, ft_support)) {
	+ err = init_egress_root_ns(steering);
	+ if (err)
	+ goto err;
	+ }
	+
	+ return 0;
	+
	+err:
	+ mlx5_fs_core_cleanup(dev);
	+ return err;
	+}
	+
	+void mlx5_fs_core_free(struct mlx5_core_dev *dev)
	+{
	+ struct mlx5_flow_steering *steering = dev->priv.steering;
	+
	+ kmem_cache_destroy(steering->ftes_cache);
	+ kmem_cache_destroy(steering->fgs_cache);
	+ kfree(steering);
	+ mlx5_ft_pool_destroy(dev);
	+ mlx5_cleanup_fc_stats(dev);
	+}
	+
	+int mlx5_fs_core_alloc(struct mlx5_core_dev *dev)
	+{
	+ struct mlx5_flow_steering *steering;
	+ int err = 0;
	+
	+ err = mlx5_init_fc_stats(dev);
	+ if (err)
	+ return err;
	+
	+ err = mlx5_ft_pool_init(dev);
	+ if (err)
	+ goto err;
	+
	+ steering = kzalloc(sizeof(*steering), GFP_KERNEL);
	+ if (!steering) {
	+ err = -ENOMEM;
	+ goto err;
	+ }
	+
	+ steering->dev = dev;
	+ dev->priv.steering = steering;
	+
	+ steering->mode = MLX5_FLOW_STEERING_MODE_DMFS;
	+
	+ steering->fgs_cache = kmem_cache_create("mlx5_fs_fgs",
	+ sizeof(struct mlx5_flow_group), 0,
	+ 0, NULL);
	+ steering->ftes_cache = kmem_cache_create("mlx5_fs_ftes", sizeof(struct fs_fte), 0,
	+ 0, NULL);
	+ if (!steering->ftes_cache \|\| !steering->fgs_cache) {
	+ err = -ENOMEM;
	+ goto err;
	+ }
	+
	+ return 0;
	+
	+err:
	+ mlx5_fs_core_free(dev);
	+ return err;
	+}
	+
	+int mlx5_fs_add_rx_underlay_qpn(struct mlx5_core_dev *dev, u32 underlay_qpn)
	+{
	+ struct mlx5_flow_root_namespace *root = dev->priv.steering->root_ns;
	+ struct mlx5_ft_underlay_qp *new_uqp;
	+ int err = 0;
	+
	+ new_uqp = kzalloc(sizeof(*new_uqp), GFP_KERNEL);
	+ if (!new_uqp)
	+ return -ENOMEM;
	+
	+ mutex_lock(&root->chain_lock);
	+
	+ if (!root->root_ft) {
	+ err = -EINVAL;
	+ goto update_ft_fail;
	+ }
	+
	+ err = root->cmds->update_root_ft(root, root->root_ft, underlay_qpn,
	+ false);
	+ if (err) {
	+ mlx5_core_warn(dev, "Failed adding underlay QPN (%u) to root FT err(%d)\n",
	+ underlay_qpn, err);
	+ goto update_ft_fail;
	+ }
	+
	+ new_uqp->qpn = underlay_qpn;
	+ list_add_tail(&new_uqp->list, &root->underlay_qpns);
	+
	+ mutex_unlock(&root->chain_lock);
	+
	+ return 0;
	+
	+update_ft_fail:
	+ mutex_unlock(&root->chain_lock);
	+ kfree(new_uqp);
	+ return err;
	+}
	+EXPORT_SYMBOL(mlx5_fs_add_rx_underlay_qpn);
	+
	+int mlx5_fs_remove_rx_underlay_qpn(struct mlx5_core_dev *dev, u32 underlay_qpn)
	+{
	+ struct mlx5_flow_root_namespace *root = dev->priv.steering->root_ns;
	+ struct mlx5_ft_underlay_qp *uqp;
	+ bool found = false;
	+ int err = 0;
	+
	+ mutex_lock(&root->chain_lock);
	+ list_for_each_entry(uqp, &root->underlay_qpns, list) {
	+ if (uqp->qpn == underlay_qpn) {
	+ found = true;
	+ break;
	+ }
	+ }
	+
	+ if (!found) {
	+ mlx5_core_warn(dev, "Failed finding underlay qp (%u) in qpn list\n",
	+ underlay_qpn);
	+ err = -EINVAL;
	+ goto out;
	+ }
	+
	+ err = root->cmds->update_root_ft(root, root->root_ft, underlay_qpn,
	+ true);
	+ if (err)
	+ mlx5_core_warn(dev, "Failed removing underlay QPN (%u) from root FT err(%d)\n",
	+ underlay_qpn, err);
	+
	+ list_del(&uqp->list);
	+ mutex_unlock(&root->chain_lock);
	+ kfree(uqp);
	+
	+ return 0;
	+
	+out:
	+ mutex_unlock(&root->chain_lock);
	+ return err;
	+}
	+EXPORT_SYMBOL(mlx5_fs_remove_rx_underlay_qpn);
	+
	+static struct mlx5_flow_root_namespace
	+get_root_namespace(struct mlx5_core_dev dev, enum mlx5_flow_namespace_type ns_type)
	+{
	+ struct mlx5_flow_namespace *ns;
	+
	+ if (ns_type == MLX5_FLOW_NAMESPACE_ESW_EGRESS \|\|
	+ ns_type == MLX5_FLOW_NAMESPACE_ESW_INGRESS)
	+ ns = mlx5_get_flow_vport_acl_namespace(dev, ns_type, 0);
	+ else
	+ ns = mlx5_get_flow_namespace(dev, ns_type);
	+ if (!ns)
	+ return NULL;
	+
	+ return find_root(&ns->node);
	+}
	+
	+struct mlx5_modify_hdr mlx5_modify_header_alloc(struct mlx5_core_dev dev,
	+ enum mlx5_flow_namespace_type ns_type,
	+ u8 num_actions,
	+ void *modify_actions)
	+{
	+ struct mlx5_flow_root_namespace *root;
	+ struct mlx5_modify_hdr *modify_hdr;
	+ int err;
	+
	+ root = get_root_namespace(dev, ns_type);
	+ if (!root)
	+ return ERR_PTR(-EOPNOTSUPP);
	+
	+ modify_hdr = kzalloc(sizeof(*modify_hdr), GFP_KERNEL);
	+ if (!modify_hdr)
	+ return ERR_PTR(-ENOMEM);
	+
	+ modify_hdr->ns_type = ns_type;
	+ err = root->cmds->modify_header_alloc(root, ns_type, num_actions,
	+ modify_actions, modify_hdr);
	+ if (err) {
	+ kfree(modify_hdr);
	+ return ERR_PTR(err);
	+ }
	+
	+ return modify_hdr;
	+}
	+EXPORT_SYMBOL(mlx5_modify_header_alloc);
	+
	+void mlx5_modify_header_dealloc(struct mlx5_core_dev *dev,
	+ struct mlx5_modify_hdr *modify_hdr)
	+{
	+ struct mlx5_flow_root_namespace *root;
	+
	+ root = get_root_namespace(dev, modify_hdr->ns_type);
	+ if (WARN_ON(!root))
	+ return;
	+ root->cmds->modify_header_dealloc(root, modify_hdr);
	+ kfree(modify_hdr);
	+}
	+EXPORT_SYMBOL(mlx5_modify_header_dealloc);
	+
	+struct mlx5_pkt_reformat mlx5_packet_reformat_alloc(struct mlx5_core_dev dev,
	+ struct mlx5_pkt_reformat_params *params,
	+ enum mlx5_flow_namespace_type ns_type)
	+{
	+ struct mlx5_pkt_reformat *pkt_reformat;
	+ struct mlx5_flow_root_namespace *root;
	+ int err;
	+
	+ root = get_root_namespace(dev, ns_type);
	+ if (!root)
	+ return ERR_PTR(-EOPNOTSUPP);
	+
	+ pkt_reformat = kzalloc(sizeof(*pkt_reformat), GFP_KERNEL);
	+ if (!pkt_reformat)
	+ return ERR_PTR(-ENOMEM);
	+
	+ pkt_reformat->ns_type = ns_type;
	+ pkt_reformat->reformat_type = params->type;
	+ err = root->cmds->packet_reformat_alloc(root, params, ns_type,
	+ pkt_reformat);
	+ if (err) {
	+ kfree(pkt_reformat);
	+ return ERR_PTR(err);
	+ }
	+
	+ return pkt_reformat;
	+}
	+EXPORT_SYMBOL(mlx5_packet_reformat_alloc);
	+
	+void mlx5_packet_reformat_dealloc(struct mlx5_core_dev *dev,
	+ struct mlx5_pkt_reformat *pkt_reformat)
	+{
	+ struct mlx5_flow_root_namespace *root;
	+
	+ root = get_root_namespace(dev, pkt_reformat->ns_type);
	+ if (WARN_ON(!root))
	+ return;
	+ root->cmds->packet_reformat_dealloc(root, pkt_reformat);
	+ kfree(pkt_reformat);
	+}
	+EXPORT_SYMBOL(mlx5_packet_reformat_dealloc);
	+
	+int mlx5_flow_namespace_set_peer(struct mlx5_flow_root_namespace *ns,
	+ struct mlx5_flow_root_namespace *peer_ns)
	+{
	+ if (peer_ns && ns->mode != peer_ns->mode) {
	+ mlx5_core_err(ns->dev,
	+ "Can't peer namespace of different steering mode\n");
	+ return -EINVAL;
	+ }
	+
	+ return ns->cmds->set_peer(ns, peer_ns);
	+}
	+
	+/* This function should be called only at init stage of the namespace.
	+ * It is not safe to call this function while steering operations
	+ * are executed in the namespace.
	+ */
	+int mlx5_flow_namespace_set_mode(struct mlx5_flow_namespace *ns,
	+ enum mlx5_flow_steering_mode mode)
	+{
	+ struct mlx5_flow_root_namespace *root;
	+ const struct mlx5_flow_cmds *cmds;
	+ int err;
	+
	+ root = find_root(&ns->node);
	+ if (&root->ns != ns)
	+ /* Can't set cmds to non root namespace */
	+ return -EINVAL;
	+
	+ if (root->table_type != FS_FT_FDB)
	+ return -EOPNOTSUPP;
	+
	+ if (root->mode == mode)
	+ return 0;
	+
	+ cmds = mlx5_fs_cmd_get_fw_cmds();
	+ if (!cmds)
	+ return -EOPNOTSUPP;
	+
	+ err = cmds->create_ns(root);
	+ if (err) {
	+ mlx5_core_err(root->dev, "Failed to create flow namespace (%d)\n",
	+ err);
	+ return err;
	+ }
	+
	+ root->cmds->destroy_ns(root);
	+ root->cmds = cmds;
	+ root->mode = mode;
	+
	+ return 0;
	+}
	diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fs_counters.c b/sys/dev/mlx5/mlx5_core/mlx5_fs_counters.c
	--- a/sys/dev/mlx5/mlx5_core/mlx5_fs_counters.c
	+++ b/sys/dev/mlx5/mlx5_core/mlx5_fs_counters.c
	@@ -31,7 +31,7 @@
	#include <linux/rbtree.h>
	#include <dev/mlx5/mlx5_core/mlx5_core.h>
	#include <dev/mlx5/mlx5_core/fs_core.h>
	-#include <dev/mlx5/mlx5_core/mlx5_fc_cmd.h>
	+#include <dev/mlx5/mlx5_core/fs_cmd.h>

	#define MLX5_FC_STATS_PERIOD msecs_to_jiffies(1000)
	#define MLX5_FC_BULK_QUERY_ALLOC_PERIOD msecs_to_jiffies(180 * 1000)
	diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fs_ft_pool.c b/sys/dev/mlx5/mlx5_core/mlx5_fs_ft_pool.c
	new file mode 100644
	--- /dev/null
	+++ b/sys/dev/mlx5/mlx5_core/mlx5_fs_ft_pool.c
	@@ -0,0 +1,85 @@
	+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
	+/* Copyright (c) 2021 Mellanox Technologies. */
	+
	+#include "fs_ft_pool.h"
	+
	+/* Firmware currently has 4 pool of 4 sizes that it supports (FT_POOLS),
	+ * and a virtual memory region of 16M (MLX5_FT_SIZE), this region is duplicated
	+ * for each flow table pool. We can allocate up to 16M of each pool,
	+ * and we keep track of how much we used via mlx5_ft_pool_get_avail_sz.
	+ * Firmware doesn't report any of this for now.
	+ * ESW_POOL is expected to be sorted from large to small and match firmware
	+ * pools.
	+ */
	+#define FT_SIZE (16 * 1024 * 1024)
	+static const unsigned int FT_POOLS[] = { 4 * 1024 * 1024,
	+ 1 * 1024 * 1024,
	+ 64 * 1024,
	+ 128,
	+ 1 /* size for termination tables */ };
	+struct mlx5_ft_pool {
	+ int ft_left[ARRAY_SIZE(FT_POOLS)];
	+};
	+
	+int mlx5_ft_pool_init(struct mlx5_core_dev *dev)
	+{
	+ struct mlx5_ft_pool *ft_pool;
	+ int i;
	+
	+ ft_pool = kzalloc(sizeof(*ft_pool), GFP_KERNEL);
	+ if (!ft_pool)
	+ return -ENOMEM;
	+
	+ for (i = ARRAY_SIZE(FT_POOLS) - 1; i >= 0; i--)
	+ ft_pool->ft_left[i] = FT_SIZE / FT_POOLS[i];
	+
	+ dev->priv.ft_pool = ft_pool;
	+ return 0;
	+}
	+
	+void mlx5_ft_pool_destroy(struct mlx5_core_dev *dev)
	+{
	+ kfree(dev->priv.ft_pool);
	+}
	+
	+int
	+mlx5_ft_pool_get_avail_sz(struct mlx5_core_dev *dev, enum fs_flow_table_type table_type,
	+ int desired_size)
	+{
	+ u32 max_ft_size = 1 << MLX5_CAP_FLOWTABLE_TYPE(dev, log_max_ft_size, table_type);
	+ int i, found_i = -1;
	+
	+ for (i = ARRAY_SIZE(FT_POOLS) - 1; i >= 0; i--) {
	+ if (dev->priv.ft_pool->ft_left[i] && FT_POOLS[i] >= desired_size &&
	+ FT_POOLS[i] <= max_ft_size) {
	+ found_i = i;
	+ if (desired_size != POOL_NEXT_SIZE)
	+ break;
	+ }
	+ }
	+
	+ if (found_i != -1) {
	+ --dev->priv.ft_pool->ft_left[found_i];
	+ return FT_POOLS[found_i];
	+ }
	+
	+ return 0;
	+}
	+
	+void
	+mlx5_ft_pool_put_sz(struct mlx5_core_dev *dev, int sz)
	+{
	+ int i;
	+
	+ if (!sz)
	+ return;
	+
	+ for (i = ARRAY_SIZE(FT_POOLS) - 1; i >= 0; i--) {
	+ if (sz == FT_POOLS[i]) {
	+ ++dev->priv.ft_pool->ft_left[i];
	+ return;
	+ }
	+ }
	+
	+ WARN_ONCE(1, "Couldn't find size %d in flow table size pool", sz);
	+}
	diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fs_tcp.c b/sys/dev/mlx5/mlx5_core/mlx5_fs_tcp.c
	--- a/sys/dev/mlx5/mlx5_core/mlx5_fs_tcp.c
	+++ b/sys/dev/mlx5/mlx5_core/mlx5_fs_tcp.c
	@@ -81,12 +81,12 @@
	#endif

	void
	-mlx5e_accel_fs_del_inpcb(struct mlx5_flow_rule *rule)
	+mlx5e_accel_fs_del_inpcb(struct mlx5_flow_handle *rule)
	{
	- mlx5_del_flow_rule(&rule);
	+ mlx5_del_flow_rules(&rule);
	}

	-struct mlx5_flow_rule *
	+struct mlx5_flow_handle *
	mlx5e_accel_fs_add_inpcb(struct mlx5e_priv *priv,
	struct inpcb *inp, uint32_t tirn, uint32_t flow_tag,
	uint16_t vlan_id)
	@@ -96,18 +96,17 @@
	#if defined(INET) \|\| defined(INET6)
	struct mlx5e_accel_fs_tcp *fs_tcp = &priv->fts.accel_tcp;
	#endif
	- struct mlx5_flow_rule *flow;
	+ struct mlx5_flow_handle *flow;
	struct mlx5_flow_spec *spec;
	- struct mlx5_flow_act flow_act = {
	- .actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG,
	- .flow_tag = flow_tag,
	- };
	+ struct mlx5_flow_act flow_act = {};

	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
	if (!spec)
	return (ERR_PTR(-ENOMEM));

	spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
	+ spec->flow_context.flags = FLOW_CONTEXT_HAS_TAG;
	+ spec->flow_context.flow_tag = flow_tag;

	INP_RLOCK(inp);
	/* Set VLAN ID to match, if any. */
	@@ -160,13 +159,9 @@

	dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
	dest.tir_num = tirn;
	+ flow_act.action = MLX5_FLOW_RULE_FWD_ACTION_DEST;

	- flow = mlx5_add_flow_rule(ft->t, spec->match_criteria_enable,
	- spec->match_criteria,
	- spec->match_value,
	- MLX5_FLOW_RULE_FWD_ACTION_DEST,
	- &flow_act,
	- &dest);
	+ flow = mlx5_add_flow_rules(ft->t, spec, &flow_act, &dest, 1);
	out:
	kvfree(spec);
	return (flow);
	@@ -175,18 +170,18 @@
	static int
	accel_fs_tcp_add_default_rule(struct mlx5e_priv *priv, int type)
	{
	- static u32 match_criteria[MLX5_ST_SZ_DW(fte_match_param)];
	- static u32 match_value[MLX5_ST_SZ_DW(fte_match_param)];
	+ static struct mlx5_flow_spec spec = {};
	struct mlx5_flow_destination dest = {};
	struct mlx5e_accel_fs_tcp *fs_tcp;
	- struct mlx5_flow_rule *rule;
	+ struct mlx5_flow_handle *rule;
	struct mlx5_flow_act flow_act = {
	- .actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG,
	- .flow_tag = MLX5_FS_DEFAULT_FLOW_TAG,
	+ .action = MLX5_FLOW_RULE_FWD_ACTION_DEST,
	};

	fs_tcp = &priv->fts.accel_tcp;

	+ spec.flow_context.flags = FLOW_CONTEXT_HAS_TAG;
	+ spec.flow_context.flow_tag = MLX5_FS_DEFAULT_FLOW_TAG;
	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;

	/*
	@@ -197,10 +192,11 @@
	* of flow tables.
	*/
	dest.ft = (type == MLX5E_ACCEL_FS_TCP_NUM_TYPES - 1) ?
	- priv->fts.vlan.t : fs_tcp->tables[type + 1].t;
	+ ((priv->fts.ipsec_ft) ? priv->fts.ipsec_ft : priv->fts.vlan.t) :
	+ fs_tcp->tables[type + 1].t;

	- rule = mlx5_add_flow_rule(fs_tcp->tables[type].t, 0, match_criteria, match_value,
	- MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
	+ rule = mlx5_add_flow_rules(fs_tcp->tables[type].t, &spec, &flow_act,
	+ &dest, 1);
	if (IS_ERR(rule))
	return (PTR_ERR(rule));

	@@ -317,11 +313,13 @@
	accel_fs_tcp_create_table(struct mlx5e_priv *priv, int type)
	{
	struct mlx5e_flow_table *ft = &priv->fts.accel_tcp.tables[type];
	+ struct mlx5_flow_table_attr ft_attr = {};
	int err;

	ft->num_groups = 0;
	- ft->t = mlx5_create_flow_table(priv->fts.accel_tcp.ns, 0, "tcp",
	- MLX5E_ACCEL_FS_TCP_TABLE_SIZE);
	+ ft_attr.max_fte = MLX5E_ACCEL_FS_TCP_TABLE_SIZE;
	+ ft_attr.level = type;
	+ ft->t = mlx5_create_flow_table(priv->fts.accel_tcp.ns, &ft_attr);
	if (IS_ERR(ft->t)) {
	err = PTR_ERR(ft->t);
	ft->t = NULL;
	@@ -365,7 +363,7 @@
	return;

	for (i = 0; i < MLX5E_ACCEL_FS_TCP_NUM_TYPES; i++) {
	- mlx5_del_flow_rule(&priv->fts.accel_tcp.default_rules[i]);
	+ mlx5_del_flow_rules(&priv->fts.accel_tcp.default_rules[i]);
	accel_fs_tcp_destroy_table(priv, i);
	}
	}
	@@ -402,7 +400,7 @@

	err_destroy_rules:
	while (i--)
	- mlx5_del_flow_rule(&priv->fts.accel_tcp.default_rules[i]);
	+ mlx5_del_flow_rules(&priv->fts.accel_tcp.default_rules[i]);
	i = MLX5E_ACCEL_FS_TCP_NUM_TYPES;

	err_destroy_tables:
	diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fs_tree.c b/sys/dev/mlx5/mlx5_core/mlx5_fs_tree.c
	deleted file mode 100644
	--- a/sys/dev/mlx5/mlx5_core/mlx5_fs_tree.c
	+++ /dev/null
	@@ -1,2871 +0,0 @@
	-/*-
	- * Copyright (c) 2013-2021, Mellanox Technologies, Ltd. All rights reserved.
	- *
	- * Redistribution and use in source and binary forms, with or without
	- * modification, are permitted provided that the following conditions
	- * are met:
	- * 1. Redistributions of source code must retain the above copyright
	- * notice, this list of conditions and the following disclaimer.
	- * 2. Redistributions in binary form must reproduce the above copyright
	- * notice, this list of conditions and the following disclaimer in the
	- * documentation and/or other materials provided with the distribution.
	- *
	- * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
	- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	- * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
	- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	- * SUCH DAMAGE.
	- *
	- * $FreeBSD$
	- */
	-
	-#include "opt_rss.h"
	-
	-#include <linux/module.h>
	-#include <dev/mlx5/driver.h>
	-#include <dev/mlx5/mlx5_core/mlx5_core.h>
	-#include <dev/mlx5/mlx5_core/fs_core.h>
	-#include <linux/string.h>
	-#include <linux/compiler.h>
	-
	-#define INIT_TREE_NODE_ARRAY_SIZE(...) (sizeof((struct init_tree_node[]){__VA_ARGS__}) /\
	- sizeof(struct init_tree_node))
	-
	-#define ADD_PRIO(name_val, flags_val, min_level_val, max_ft_val, caps_val, \
	- ...) {.type = FS_TYPE_PRIO,\
	- .name = name_val,\
	- .min_ft_level = min_level_val,\
	- .flags = flags_val,\
	- .max_ft = max_ft_val,\
	- .caps = caps_val,\
	- .children = (struct init_tree_node[]) {__VA_ARGS__},\
	- .ar_size = INIT_TREE_NODE_ARRAY_SIZE(__VA_ARGS__) \
	-}
	-
	-#define ADD_FT_PRIO(name_val, flags_val, max_ft_val, ...)\
	- ADD_PRIO(name_val, flags_val, 0, max_ft_val, {},\
	- __VA_ARGS__)\
	-
	-#define ADD_NS(name_val, ...) {.type = FS_TYPE_NAMESPACE,\
	- .name = name_val,\
	- .children = (struct init_tree_node[]) {__VA_ARGS__},\
	- .ar_size = INIT_TREE_NODE_ARRAY_SIZE(__VA_ARGS__) \
	-}
	-
	-#define INIT_CAPS_ARRAY_SIZE(...) (sizeof((long[]){__VA_ARGS__}) /\
	- sizeof(long))
	-
	-#define FS_CAP(cap) (__mlx5_bit_off(flow_table_nic_cap, cap))
	-
	-#define FS_REQUIRED_CAPS(...) {.arr_sz = INIT_CAPS_ARRAY_SIZE(__VA_ARGS__), \
	- .caps = (long[]) {__VA_ARGS__}}
	-
	-/* Flowtable sizes: */
	-#define BYPASS_MAX_FT 5
	-#define BYPASS_PRIO_MAX_FT 1
	-#define OFFLOADS_MAX_FT 2
	-#define KERNEL_MAX_FT 5
	-#define LEFTOVER_MAX_FT 1
	-
	-/* Flowtable levels: */
	-#define OFFLOADS_MIN_LEVEL 3
	-#define KERNEL_MIN_LEVEL (OFFLOADS_MIN_LEVEL + 1)
	-#define LEFTOVER_MIN_LEVEL (KERNEL_MIN_LEVEL + 1)
	-#define BYPASS_MIN_LEVEL (MLX5_NUM_BYPASS_FTS + LEFTOVER_MIN_LEVEL)
	-
	-struct node_caps {
	- size_t arr_sz;
	- long *caps;
	-};
	-
	-struct init_tree_node {
	- enum fs_type type;
	- const char *name;
	- struct init_tree_node *children;
	- int ar_size;
	- struct node_caps caps;
	- u8 flags;
	- int min_ft_level;
	- int prio;
	- int max_ft;
	-} root_fs = {
	- .type = FS_TYPE_NAMESPACE,
	- .name = "root",
	- .ar_size = 4,
	- .children = (struct init_tree_node[]) {
	- ADD_PRIO("by_pass_prio", 0, BYPASS_MIN_LEVEL, 0,
	- FS_REQUIRED_CAPS(FS_CAP(flow_table_properties_nic_receive.flow_modify_en),
	- FS_CAP(flow_table_properties_nic_receive.modify_root)),
	- ADD_NS("by_pass_ns",
	- ADD_FT_PRIO("prio0", 0,
	- BYPASS_PRIO_MAX_FT),
	- ADD_FT_PRIO("prio1", 0,
	- BYPASS_PRIO_MAX_FT),
	- ADD_FT_PRIO("prio2", 0,
	- BYPASS_PRIO_MAX_FT),
	- ADD_FT_PRIO("prio3", 0,
	- BYPASS_PRIO_MAX_FT),
	- ADD_FT_PRIO("prio4", 0,
	- BYPASS_PRIO_MAX_FT),
	- ADD_FT_PRIO("prio5", 0,
	- BYPASS_PRIO_MAX_FT),
	- ADD_FT_PRIO("prio6", 0,
	- BYPASS_PRIO_MAX_FT),
	- ADD_FT_PRIO("prio7", 0,
	- BYPASS_PRIO_MAX_FT),
	- ADD_FT_PRIO("prio-mcast", 0,
	- BYPASS_PRIO_MAX_FT))),
	- ADD_PRIO("offloads_prio", 0, OFFLOADS_MIN_LEVEL, 0, {},
	- ADD_NS("offloads_ns",
	- ADD_FT_PRIO("prio_offloads-0", 0,
	- OFFLOADS_MAX_FT))),
	- ADD_PRIO("kernel_prio", 0, KERNEL_MIN_LEVEL, 0, {},
	- ADD_NS("kernel_ns",
	- ADD_FT_PRIO("prio_kernel-0", 0,
	- KERNEL_MAX_FT))),
	- ADD_PRIO("leftovers_prio", MLX5_CORE_FS_PRIO_SHARED,
	- LEFTOVER_MIN_LEVEL, 0,
	- FS_REQUIRED_CAPS(FS_CAP(flow_table_properties_nic_receive.flow_modify_en),
	- FS_CAP(flow_table_properties_nic_receive.modify_root)),
	- ADD_NS("leftover_ns",
	- ADD_FT_PRIO("leftovers_prio-0",
	- MLX5_CORE_FS_PRIO_SHARED,
	- LEFTOVER_MAX_FT)))
	- }
	-};
	-
	-/* Tree creation functions */
	-
	-static struct mlx5_flow_root_namespace find_root(struct fs_base node)
	-{
	- struct fs_base *parent;
	-
	- /* Make sure we only read it once while we go up the tree */
	- while ((parent = node->parent))
	- node = parent;
	-
	- if (node->type != FS_TYPE_NAMESPACE) {
	- return NULL;
	- }
	-
	- return container_of(container_of(node,
	- struct mlx5_flow_namespace,
	- base),
	- struct mlx5_flow_root_namespace,
	- ns);
	-}
	-
	-static inline struct mlx5_core_dev fs_get_dev(struct fs_base node)
	-{
	- struct mlx5_flow_root_namespace *root = find_root(node);
	-
	- if (root)
	- return root->dev;
	- return NULL;
	-}
	-
	-static void fs_init_node(struct fs_base *node,
	- unsigned int refcount)
	-{
	- kref_init(&node->refcount);
	- atomic_set(&node->users_refcount, refcount);
	- init_completion(&node->complete);
	- INIT_LIST_HEAD(&node->list);
	- mutex_init(&node->lock);
	-}
	-
	-static void _fs_add_node(struct fs_base *node,
	- const char *name,
	- struct fs_base *parent)
	-{
	- if (parent)
	- atomic_inc(&parent->users_refcount);
	- node->name = kstrdup_const(name, GFP_KERNEL);
	- node->parent = parent;
	-}
	-
	-static void fs_add_node(struct fs_base *node,
	- struct fs_base parent, const char name,
	- unsigned int refcount)
	-{
	- fs_init_node(node, refcount);
	- _fs_add_node(node, name, parent);
	-}
	-
	-static void _fs_put(struct fs_base node, void (kref_cb)(struct kref *kref),
	- bool parent_locked);
	-
	-static void fs_del_dst(struct mlx5_flow_rule *dst);
	-static void _fs_del_ft(struct mlx5_flow_table *ft);
	-static void fs_del_fg(struct mlx5_flow_group *fg);
	-static void fs_del_fte(struct fs_fte *fte);
	-
	-static void cmd_remove_node(struct fs_base *base)
	-{
	- switch (base->type) {
	- case FS_TYPE_FLOW_DEST:
	- fs_del_dst(container_of(base, struct mlx5_flow_rule, base));
	- break;
	- case FS_TYPE_FLOW_TABLE:
	- _fs_del_ft(container_of(base, struct mlx5_flow_table, base));
	- break;
	- case FS_TYPE_FLOW_GROUP:
	- fs_del_fg(container_of(base, struct mlx5_flow_group, base));
	- break;
	- case FS_TYPE_FLOW_ENTRY:
	- fs_del_fte(container_of(base, struct fs_fte, base));
	- break;
	- default:
	- break;
	- }
	-}
	-
	-static void __fs_remove_node(struct kref *kref)
	-{
	- struct fs_base *node = container_of(kref, struct fs_base, refcount);
	-
	- if (node->parent)
	- mutex_lock(&node->parent->lock);
	- mutex_lock(&node->lock);
	- cmd_remove_node(node);
	- mutex_unlock(&node->lock);
	- complete(&node->complete);
	- if (node->parent) {
	- mutex_unlock(&node->parent->lock);
	- _fs_put(node->parent, _fs_remove_node, false);
	- }
	-}
	-
	-void _fs_remove_node(struct kref *kref)
	-{
	- struct fs_base *node = container_of(kref, struct fs_base, refcount);
	-
	- __fs_remove_node(kref);
	- kfree_const(node->name);
	- kfree(node);
	-}
	-
	-static void fs_get(struct fs_base *node)
	-{
	- atomic_inc(&node->users_refcount);
	-}
	-
	-static void _fs_put(struct fs_base node, void (kref_cb)(struct kref *kref),
	- bool parent_locked)
	-{
	- struct fs_base *parent_node = node->parent;
	-
	- if (parent_node && !parent_locked)
	- mutex_lock(&parent_node->lock);
	- if (atomic_dec_and_test(&node->users_refcount)) {
	- if (parent_node) {
	- /remove from parent's list/
	- list_del_init(&node->list);
	- mutex_unlock(&parent_node->lock);
	- }
	- kref_put(&node->refcount, kref_cb);
	- if (parent_node && parent_locked)
	- mutex_lock(&parent_node->lock);
	- } else if (parent_node && !parent_locked) {
	- mutex_unlock(&parent_node->lock);
	- }
	-}
	-
	-static void fs_put(struct fs_base *node)
	-{
	- _fs_put(node, __fs_remove_node, false);
	-}
	-
	-static void fs_put_parent_locked(struct fs_base *node)
	-{
	- _fs_put(node, __fs_remove_node, true);
	-}
	-
	-static void fs_remove_node(struct fs_base *node)
	-{
	- fs_put(node);
	- wait_for_completion(&node->complete);
	- kfree_const(node->name);
	- kfree(node);
	-}
	-
	-static void fs_remove_node_parent_locked(struct fs_base *node)
	-{
	- fs_put_parent_locked(node);
	- wait_for_completion(&node->complete);
	- kfree_const(node->name);
	- kfree(node);
	-}
	-
	-static struct fs_fte *fs_alloc_fte(u32 sw_action,
	- struct mlx5_flow_act *flow_act,
	- u32 *match_value,
	- unsigned int index)
	-{
	- struct fs_fte *fte;
	-
	-
	- fte = kzalloc(sizeof(*fte), GFP_KERNEL);
	- if (!fte)
	- return ERR_PTR(-ENOMEM);
	-
	- memcpy(fte->val, match_value, sizeof(fte->val));
	- fte->base.type = FS_TYPE_FLOW_ENTRY;
	- fte->dests_size = 0;
	- fte->index = index;
	- INIT_LIST_HEAD(&fte->dests);
	- fte->flow_act = *flow_act;
	- fte->sw_action = sw_action;
	-
	- return fte;
	-}
	-
	-static struct fs_fte alloc_star_ft_entry(struct mlx5_flow_table ft,
	- struct mlx5_flow_group *fg,
	- u32 *match_value,
	- unsigned int index)
	-{
	- int err;
	- struct fs_fte *fte;
	- struct mlx5_flow_rule *dst;
	- struct mlx5_flow_act flow_act = {
	- .actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG,
	- .flow_tag = MLX5_FS_DEFAULT_FLOW_TAG,
	- };
	-
	- if (fg->num_ftes == fg->max_ftes)
	- return ERR_PTR(-ENOSPC);
	-
	- fte = fs_alloc_fte(MLX5_FLOW_RULE_FWD_ACTION_DEST,
	- &flow_act, match_value, index);
	- if (IS_ERR(fte))
	- return fte;
	-
	- /create dst/
	- dst = kzalloc(sizeof(*dst), GFP_KERNEL);
	- if (!dst) {
	- err = -ENOMEM;
	- goto free_fte;
	- }
	-
	- fte->base.parent = &fg->base;
	- fte->dests_size = 1;
	- dst->dest_attr.type = MLX5_FLOW_CONTEXT_DEST_TYPE_FLOW_TABLE;
	- dst->base.parent = &fte->base;
	- list_add(&dst->base.list, &fte->dests);
	- /* assumed that the callee creates the star rules sorted by index */
	- list_add_tail(&fte->base.list, &fg->ftes);
	- fg->num_ftes++;
	-
	- return fte;
	-
	-free_fte:
	- kfree(fte);
	- return ERR_PTR(err);
	-}
	-
	-/* assume that fte can't be changed */
	-static void free_star_fte_entry(struct fs_fte *fte)
	-{
	- struct mlx5_flow_group *fg;
	- struct mlx5_flow_rule dst, temp;
	-
	- fs_get_parent(fg, fte);
	-
	- list_for_each_entry_safe(dst, temp, &fte->dests, base.list) {
	- fte->dests_size--;
	- list_del(&dst->base.list);
	- kfree(dst);
	- }
	-
	- list_del(&fte->base.list);
	- fg->num_ftes--;
	- kfree(fte);
	-}
	-
	-static struct mlx5_flow_group fs_alloc_fg(u32 create_fg_in)
	-{
	- struct mlx5_flow_group *fg;
	- void *match_criteria = MLX5_ADDR_OF(create_flow_group_in,
	- create_fg_in, match_criteria);
	- u8 match_criteria_enable = MLX5_GET(create_flow_group_in,
	- create_fg_in,
	- match_criteria_enable);
	- fg = kzalloc(sizeof(*fg), GFP_KERNEL);
	- if (!fg)
	- return ERR_PTR(-ENOMEM);
	-
	- INIT_LIST_HEAD(&fg->ftes);
	- fg->mask.match_criteria_enable = match_criteria_enable;
	- memcpy(&fg->mask.match_criteria, match_criteria,
	- sizeof(fg->mask.match_criteria));
	- fg->base.type = FS_TYPE_FLOW_GROUP;
	- fg->start_index = MLX5_GET(create_flow_group_in, create_fg_in,
	- start_flow_index);
	- fg->max_ftes = MLX5_GET(create_flow_group_in, create_fg_in,
	- end_flow_index) - fg->start_index + 1;
	- return fg;
	-}
	-
	-static struct mlx5_flow_table find_next_ft(struct fs_prio prio);
	-static struct mlx5_flow_table find_prev_ft(struct mlx5_flow_table curr,
	- struct fs_prio *prio);
	-
	-/* assumed src_ft and dst_ft can't be freed */
	-static int fs_set_star_rule(struct mlx5_core_dev *dev,
	- struct mlx5_flow_table *src_ft,
	- struct mlx5_flow_table *dst_ft)
	-{
	- struct mlx5_flow_rule *src_dst;
	- struct fs_fte *src_fte;
	- int err = 0;
	- u32 *match_value;
	- int match_len = MLX5_ST_SZ_BYTES(fte_match_param);
	-
	- src_dst = list_first_entry(&src_ft->star_rule.fte->dests,
	- struct mlx5_flow_rule, base.list);
	- match_value = mlx5_vzalloc(match_len);
	- if (!match_value) {
	- mlx5_core_warn(dev, "failed to allocate inbox\n");
	- return -ENOMEM;
	- }
	- /Create match context/
	-
	- fs_get_parent(src_fte, src_dst);
	-
	- src_dst->dest_attr.ft = dst_ft;
	- if (dst_ft) {
	- err = mlx5_cmd_fs_set_fte(dev,
	- src_ft->vport,
	- &src_fte->status,
	- match_value, src_ft->type,
	- src_ft->id, src_fte->index,
	- src_ft->star_rule.fg->id,
	- &src_fte->flow_act,
	- src_fte->sw_action,
	- src_fte->dests_size,
	- &src_fte->dests);
	- if (err)
	- goto free;
	-
	- fs_get(&dst_ft->base);
	- } else {
	- mlx5_cmd_fs_delete_fte(dev,
	- src_ft->vport,
	- &src_fte->status,
	- src_ft->type, src_ft->id,
	- src_fte->index);
	- }
	-
	-free:
	- kvfree(match_value);
	- return err;
	-}
	-
	-static int connect_prev_fts(struct fs_prio *locked_prio,
	- struct fs_prio *prev_prio,
	- struct mlx5_flow_table *next_ft)
	-{
	- struct mlx5_flow_table *iter;
	- int err = 0;
	- struct mlx5_core_dev *dev = fs_get_dev(&prev_prio->base);
	-
	- if (!dev)
	- return -ENODEV;
	-
	- mutex_lock(&prev_prio->base.lock);
	- fs_for_each_ft(iter, prev_prio) {
	- struct mlx5_flow_rule *src_dst =
	- list_first_entry(&iter->star_rule.fte->dests,
	- struct mlx5_flow_rule, base.list);
	- struct mlx5_flow_table *prev_ft = src_dst->dest_attr.ft;
	-
	- if (prev_ft == next_ft)
	- continue;
	-
	- err = fs_set_star_rule(dev, iter, next_ft);
	- if (err) {
	- mlx5_core_warn(dev,
	- "mlx5: flow steering can't connect prev and next\n");
	- goto unlock;
	- } else {
	- /* Assume ft's prio is locked */
	- if (prev_ft) {
	- struct fs_prio *prio;
	-
	- fs_get_parent(prio, prev_ft);
	- if (prio == locked_prio)
	- fs_put_parent_locked(&prev_ft->base);
	- else
	- fs_put(&prev_ft->base);
	- }
	- }
	- }
	-
	-unlock:
	- mutex_unlock(&prev_prio->base.lock);
	- return 0;
	-}
	-
	-static int create_star_rule(struct mlx5_flow_table ft, struct fs_prio prio)
	-{
	- struct mlx5_flow_group *fg;
	- int err;
	- u32 *fg_in;
	- u32 *match_value;
	- struct mlx5_flow_table *next_ft;
	- struct mlx5_flow_table *prev_ft;
	- struct mlx5_flow_root_namespace *root = find_root(&prio->base);
	- int fg_inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
	- int match_len = MLX5_ST_SZ_BYTES(fte_match_param);
	-
	- fg_in = mlx5_vzalloc(fg_inlen);
	- if (!fg_in) {
	- mlx5_core_warn(root->dev, "failed to allocate inbox\n");
	- return -ENOMEM;
	- }
	-
	- match_value = mlx5_vzalloc(match_len);
	- if (!match_value) {
	- mlx5_core_warn(root->dev, "failed to allocate inbox\n");
	- kvfree(fg_in);
	- return -ENOMEM;
	- }
	-
	- MLX5_SET(create_flow_group_in, fg_in, start_flow_index, ft->max_fte);
	- MLX5_SET(create_flow_group_in, fg_in, end_flow_index, ft->max_fte);
	- fg = fs_alloc_fg(fg_in);
	- if (IS_ERR(fg)) {
	- err = PTR_ERR(fg);
	- goto out;
	- }
	- ft->star_rule.fg = fg;
	- err = mlx5_cmd_fs_create_fg(fs_get_dev(&prio->base),
	- fg_in, ft->vport, ft->type,
	- ft->id,
	- &fg->id);
	- if (err)
	- goto free_fg;
	-
	- ft->star_rule.fte = alloc_star_ft_entry(ft, fg,
	- match_value,
	- ft->max_fte);
	- if (IS_ERR(ft->star_rule.fte))
	- goto free_star_rule;
	-
	- mutex_lock(&root->fs_chain_lock);
	- next_ft = find_next_ft(prio);
	- err = fs_set_star_rule(root->dev, ft, next_ft);
	- if (err) {
	- mutex_unlock(&root->fs_chain_lock);
	- goto free_star_rule;
	- }
	- if (next_ft) {
	- struct fs_prio *parent;
	-
	- fs_get_parent(parent, next_ft);
	- fs_put(&next_ft->base);
	- }
	- prev_ft = find_prev_ft(ft, prio);
	- if (prev_ft) {
	- struct fs_prio *prev_parent;
	-
	- fs_get_parent(prev_parent, prev_ft);
	-
	- err = connect_prev_fts(NULL, prev_parent, ft);
	- if (err) {
	- mutex_unlock(&root->fs_chain_lock);
	- goto destroy_chained_star_rule;
	- }
	- fs_put(&prev_ft->base);
	- }
	- mutex_unlock(&root->fs_chain_lock);
	- kvfree(fg_in);
	- kvfree(match_value);
	-
	- return 0;
	-
	-destroy_chained_star_rule:
	- fs_set_star_rule(fs_get_dev(&prio->base), ft, NULL);
	- if (next_ft)
	- fs_put(&next_ft->base);
	-free_star_rule:
	- free_star_fte_entry(ft->star_rule.fte);
	- mlx5_cmd_fs_destroy_fg(fs_get_dev(&ft->base), ft->vport,
	- ft->type, ft->id,
	- fg->id);
	-free_fg:
	- kfree(fg);
	-out:
	- kvfree(fg_in);
	- kvfree(match_value);
	- return err;
	-}
	-
	-static void destroy_star_rule(struct mlx5_flow_table ft, struct fs_prio prio)
	-{
	- int err;
	- struct mlx5_flow_root_namespace *root;
	- struct mlx5_core_dev *dev = fs_get_dev(&prio->base);
	- struct mlx5_flow_table prev_ft, next_ft;
	- struct fs_prio *prev_prio;
	-
	- WARN_ON(!dev);
	-
	- root = find_root(&prio->base);
	- if (!root)
	- mlx5_core_err(dev,
	- "flow steering failed to find root of priority %s",
	- prio->base.name);
	-
	- /* In order to ensure atomic deletion, first update
	- * prev ft to point on the next ft.
	- */
	- mutex_lock(&root->fs_chain_lock);
	- prev_ft = find_prev_ft(ft, prio);
	- next_ft = find_next_ft(prio);
	- if (prev_ft) {
	- fs_get_parent(prev_prio, prev_ft);
	- /Prev is connected to ft, only if ft is the first(last) in the prio/
	- err = connect_prev_fts(prio, prev_prio, next_ft);
	- if (err)
	- mlx5_core_warn(root->dev,
	- "flow steering can't connect prev and next of flow table\n");
	- fs_put(&prev_ft->base);
	- }
	-
	- err = fs_set_star_rule(root->dev, ft, NULL);
	- /One put is for fs_get in find next ft/
	- if (next_ft) {
	- fs_put(&next_ft->base);
	- if (!err)
	- fs_put(&next_ft->base);
	- }
	-
	- mutex_unlock(&root->fs_chain_lock);
	- err = mlx5_cmd_fs_destroy_fg(dev, ft->vport, ft->type, ft->id,
	- ft->star_rule.fg->id);
	- if (err)
	- mlx5_core_warn(dev,
	- "flow steering can't destroy star entry group(index:%d) of ft:%s\n", ft->star_rule.fg->start_index,
	- ft->base.name);
	- free_star_fte_entry(ft->star_rule.fte);
	-
	- kfree(ft->star_rule.fg);
	- ft->star_rule.fg = NULL;
	-}
	-
	-static struct fs_prio find_prio(struct mlx5_flow_namespace ns,
	- unsigned int prio)
	-{
	- struct fs_prio *iter_prio;
	-
	- fs_for_each_prio(iter_prio, ns) {
	- if (iter_prio->prio == prio)
	- return iter_prio;
	- }
	-
	- return NULL;
	-}
	-
	-static unsigned int _alloc_new_level(struct fs_prio *prio,
	- struct mlx5_flow_namespace *match);
	-
	-static unsigned int __alloc_new_level(struct mlx5_flow_namespace *ns,
	- struct fs_prio *prio)
	-{
	- unsigned int level = 0;
	- struct fs_prio *p;
	-
	- if (!ns)
	- return 0;
	-
	- mutex_lock(&ns->base.lock);
	- fs_for_each_prio(p, ns) {
	- if (p != prio)
	- level += p->max_ft;
	- else
	- break;
	- }
	- mutex_unlock(&ns->base.lock);
	-
	- fs_get_parent(prio, ns);
	- if (prio)
	- WARN_ON(prio->base.type != FS_TYPE_PRIO);
	-
	- return level + _alloc_new_level(prio, ns);
	-}
	-
	-/* Called under lock of priority, hence locking all upper objects */
	-static unsigned int _alloc_new_level(struct fs_prio *prio,
	- struct mlx5_flow_namespace *match)
	-{
	- struct mlx5_flow_namespace *ns;
	- struct fs_base *it;
	- unsigned int level = 0;
	-
	- if (!prio)
	- return 0;
	-
	- mutex_lock(&prio->base.lock);
	- fs_for_each_ns_or_ft_reverse(it, prio) {
	- if (it->type == FS_TYPE_NAMESPACE) {
	- struct fs_prio *p;
	-
	- fs_get_obj(ns, it);
	-
	- if (match != ns) {
	- mutex_lock(&ns->base.lock);
	- fs_for_each_prio(p, ns)
	- level += p->max_ft;
	- mutex_unlock(&ns->base.lock);
	- } else {
	- break;
	- }
	- } else {
	- struct mlx5_flow_table *ft;
	-
	- fs_get_obj(ft, it);
	- mutex_unlock(&prio->base.lock);
	- return level + ft->level + 1;
	- }
	- }
	-
	- fs_get_parent(ns, prio);
	- mutex_unlock(&prio->base.lock);
	- return __alloc_new_level(ns, prio) + level;
	-}
	-
	-static unsigned int alloc_new_level(struct fs_prio *prio)
	-{
	- return _alloc_new_level(prio, NULL);
	-}
	-
	-static int update_root_ft_create(struct mlx5_flow_root_namespace *root,
	- struct mlx5_flow_table *ft)
	-{
	- int err = 0;
	- int min_level = INT_MAX;
	-
	- if (root->root_ft)
	- min_level = root->root_ft->level;
	-
	- if (ft->level < min_level)
	- err = mlx5_cmd_update_root_ft(root->dev, ft->type,
	- ft->id);
	- else
	- return err;
	-
	- if (err)
	- mlx5_core_warn(root->dev, "Update root flow table of id=%u failed\n",
	- ft->id);
	- else
	- root->root_ft = ft;
	-
	- return err;
	-}
	-
	-static struct mlx5_flow_table _create_ft_common(struct mlx5_flow_namespace ns,
	- u16 vport,
	- struct fs_prio *fs_prio,
	- int max_fte,
	- const char *name)
	-{
	- struct mlx5_flow_table *ft;
	- int err;
	- int log_table_sz;
	- int ft_size;
	- char gen_name[20];
	- struct mlx5_flow_root_namespace *root = find_root(&ns->base);
	- struct mlx5_core_dev *dev = fs_get_dev(&ns->base);
	-
	- if (!root) {
	- mlx5_core_err(dev,
	- "flow steering failed to find root of namespace %s",
	- ns->base.name);
	- return ERR_PTR(-ENODEV);
	- }
	-
	- if (fs_prio->num_ft == fs_prio->max_ft)
	- return ERR_PTR(-ENOSPC);
	-
	- ft = kzalloc(sizeof(*ft), GFP_KERNEL);
	- if (!ft)
	- return ERR_PTR(-ENOMEM);
	-
	- fs_init_node(&ft->base, 1);
	- INIT_LIST_HEAD(&ft->fgs);
	-
	- /* Temporarily WA until we expose the level set in the API */
	- if (root->table_type == FS_FT_ESW_EGRESS_ACL \|\|
	- root->table_type == FS_FT_ESW_INGRESS_ACL)
	- ft->level = 0;
	- else
	- ft->level = alloc_new_level(fs_prio);
	-
	- ft->base.type = FS_TYPE_FLOW_TABLE;
	- ft->vport = vport;
	- ft->type = root->table_type;
	- /Two entries are reserved for star rules/
	- ft_size = roundup_pow_of_two(max_fte + 2);
	- /User isn't aware to those rules/
	- ft->max_fte = ft_size - 2;
	- log_table_sz = ilog2(ft_size);
	-
	- if (name == NULL \|\| name[0] == '\0') {
	- snprintf(gen_name, sizeof(gen_name), "flow_table_%u", ft->id);
	- name = gen_name;
	- }
	-
	- err = mlx5_cmd_fs_create_ft(root->dev, ft->vport, ft->type,
	- ft->level, log_table_sz, name, &ft->id);
	- if (err)
	- goto free_ft;
	-
	- err = create_star_rule(ft, fs_prio);
	- if (err)
	- goto del_ft;
	-
	- if ((root->table_type == FS_FT_NIC_RX) && MLX5_CAP_FLOWTABLE(root->dev,
	- flow_table_properties_nic_receive.modify_root)) {
	- err = update_root_ft_create(root, ft);
	- if (err)
	- goto destroy_star_rule;
	- }
	-
	- _fs_add_node(&ft->base, name, &fs_prio->base);
	-
	- list_add_tail(&ft->base.list, &fs_prio->objs);
	- fs_prio->num_ft++;
	-
	- return ft;
	-
	-destroy_star_rule:
	- destroy_star_rule(ft, fs_prio);
	-del_ft:
	- mlx5_cmd_fs_destroy_ft(root->dev, ft->vport, ft->type, ft->id);
	-free_ft:
	- kfree(ft);
	- return ERR_PTR(err);
	-}
	-
	-static struct mlx5_flow_table create_ft_common(struct mlx5_flow_namespace ns,
	- u16 vport,
	- unsigned int prio,
	- int max_fte,
	- const char *name)
	-{
	- struct fs_prio *fs_prio = NULL;
	- fs_prio = find_prio(ns, prio);
	- if (!fs_prio)
	- return ERR_PTR(-EINVAL);
	-
	- return _create_ft_common(ns, vport, fs_prio, max_fte, name);
	-}
	-
	-
	-static struct mlx5_flow_table find_first_ft_in_ns(struct mlx5_flow_namespace ns,
	- struct list_head *start);
	-
	-static struct mlx5_flow_table find_first_ft_in_prio(struct fs_prio prio,
	- struct list_head *start);
	-
	-static struct mlx5_flow_table mlx5_create_autogrouped_shared_flow_table(struct fs_prio fs_prio)
	-{
	- struct mlx5_flow_table *ft;
	-
	- ft = find_first_ft_in_prio(fs_prio, &fs_prio->objs);
	- if (ft) {
	- ft->shared_refcount++;
	- return ft;
	- }
	-
	- return NULL;
	-}
	-
	-struct mlx5_flow_table mlx5_create_auto_grouped_flow_table(struct mlx5_flow_namespace ns,
	- int prio,
	- const char *name,
	- int num_flow_table_entries,
	- int max_num_groups,
	- int num_reserved_entries)
	-{
	- struct mlx5_flow_table *ft = NULL;
	- struct fs_prio *fs_prio;
	- bool is_shared_prio;
	-
	- if (max_num_groups > (num_flow_table_entries - num_reserved_entries))
	- return ERR_PTR(-EINVAL);
	- if (num_reserved_entries > num_flow_table_entries)
	- return ERR_PTR(-EINVAL);
	-
	- fs_prio = find_prio(ns, prio);
	- if (!fs_prio)
	- return ERR_PTR(-EINVAL);
	-
	- is_shared_prio = fs_prio->flags & MLX5_CORE_FS_PRIO_SHARED;
	- if (is_shared_prio) {
	- mutex_lock(&fs_prio->shared_lock);
	- ft = mlx5_create_autogrouped_shared_flow_table(fs_prio);
	- }
	-
	- if (ft)
	- goto return_ft;
	-
	- ft = create_ft_common(ns, 0, prio, num_flow_table_entries,
	- name);
	- if (IS_ERR(ft))
	- goto return_ft;
	-
	- ft->autogroup.active = true;
	- ft->autogroup.max_types = max_num_groups;
	- ft->autogroup.max_fte = num_flow_table_entries - num_reserved_entries;
	- /* We save place for flow groups in addition to max types */
	- ft->autogroup.group_size = ft->autogroup.max_fte / (max_num_groups + 1);
	-
	- if (is_shared_prio)
	- ft->shared_refcount = 1;
	-
	-return_ft:
	- if (is_shared_prio)
	- mutex_unlock(&fs_prio->shared_lock);
	- return ft;
	-}
	-EXPORT_SYMBOL(mlx5_create_auto_grouped_flow_table);
	-
	-struct mlx5_flow_table mlx5_create_vport_flow_table(struct mlx5_flow_namespace ns,
	- u16 vport,
	- int prio,
	- const char *name,
	- int num_flow_table_entries)
	-{
	- return create_ft_common(ns, vport, prio, num_flow_table_entries, name);
	-}
	-EXPORT_SYMBOL(mlx5_create_vport_flow_table);
	-
	-struct mlx5_flow_table mlx5_create_flow_table(struct mlx5_flow_namespace ns,
	- int prio,
	- const char *name,
	- int num_flow_table_entries)
	-{
	- return create_ft_common(ns, 0, prio, num_flow_table_entries, name);
	-}
	-EXPORT_SYMBOL(mlx5_create_flow_table);
	-
	-static void _fs_del_ft(struct mlx5_flow_table *ft)
	-{
	- int err;
	- struct mlx5_core_dev *dev = fs_get_dev(&ft->base);
	- struct fs_prio *prio;
	-
	- err = mlx5_cmd_fs_destroy_ft(dev, ft->vport, ft->type, ft->id);
	- if (err)
	- mlx5_core_warn(dev, "flow steering can't destroy ft %s\n",
	- ft->base.name);
	-
	- fs_get_parent(prio, ft);
	- prio->num_ft--;
	-}
	-
	-static int update_root_ft_destroy(struct mlx5_flow_root_namespace *root,
	- struct mlx5_flow_table *ft)
	-{
	- int err = 0;
	- struct fs_prio *prio;
	- struct mlx5_flow_table *next_ft = NULL;
	- struct mlx5_flow_table *put_ft = NULL;
	-
	- if (root->root_ft != ft)
	- return 0;
	-
	- fs_get_parent(prio, ft);
	- /*Assuming objs containis only flow tables and
	- * flow tables are sorted by level.
	- */
	- if (!list_is_last(&ft->base.list, &prio->objs)) {
	- next_ft = list_next_entry(ft, base.list);
	- } else {
	- next_ft = find_next_ft(prio);
	- put_ft = next_ft;
	- }
	-
	- if (next_ft) {
	- err = mlx5_cmd_update_root_ft(root->dev, next_ft->type,
	- next_ft->id);
	- if (err)
	- mlx5_core_warn(root->dev, "Update root flow table of id=%u failed\n",
	- ft->id);
	- }
	- if (!err)
	- root->root_ft = next_ft;
	-
	- if (put_ft)
	- fs_put(&put_ft->base);
	-
	- return err;
	-}
	-
	-/Objects in the same prio are destroyed in the reverse order they were createrd/
	-int mlx5_destroy_flow_table(struct mlx5_flow_table *ft)
	-{
	- int err = 0;
	- struct fs_prio *prio;
	- struct mlx5_flow_root_namespace *root;
	- bool is_shared_prio;
	- struct mlx5_core_dev *dev;
	-
	- fs_get_parent(prio, ft);
	- root = find_root(&prio->base);
	- dev = fs_get_dev(&prio->base);
	-
	- if (!root) {
	- mlx5_core_err(dev,
	- "flow steering failed to find root of priority %s",
	- prio->base.name);
	- return -ENODEV;
	- }
	-
	- is_shared_prio = prio->flags & MLX5_CORE_FS_PRIO_SHARED;
	- if (is_shared_prio) {
	- mutex_lock(&prio->shared_lock);
	- if (ft->shared_refcount > 1) {
	- --ft->shared_refcount;
	- fs_put(&ft->base);
	- mutex_unlock(&prio->shared_lock);
	- return 0;
	- }
	- }
	-
	- mutex_lock(&prio->base.lock);
	- mutex_lock(&ft->base.lock);
	-
	- err = update_root_ft_destroy(root, ft);
	- if (err)
	- goto unlock_ft;
	-
	- /* delete two last entries */
	- destroy_star_rule(ft, prio);
	-
	- mutex_unlock(&ft->base.lock);
	- fs_remove_node_parent_locked(&ft->base);
	- mutex_unlock(&prio->base.lock);
	- if (is_shared_prio)
	- mutex_unlock(&prio->shared_lock);
	-
	- return err;
	-
	-unlock_ft:
	- mutex_unlock(&ft->base.lock);
	- mutex_unlock(&prio->base.lock);
	- if (is_shared_prio)
	- mutex_unlock(&prio->shared_lock);
	-
	- return err;
	-}
	-EXPORT_SYMBOL(mlx5_destroy_flow_table);
	-
	-static struct mlx5_flow_group fs_create_fg(struct mlx5_core_dev dev,
	- struct mlx5_flow_table *ft,
	- struct list_head *prev,
	- u32 *fg_in,
	- int refcount)
	-{
	- struct mlx5_flow_group *fg;
	- unsigned int group_size;
	- int err;
	- char name[20];
	-
	- fg = fs_alloc_fg(fg_in);
	- if (IS_ERR(fg))
	- return fg;
	-
	- group_size = MLX5_GET(create_flow_group_in, fg_in, end_flow_index) -
	- MLX5_GET(create_flow_group_in, fg_in, start_flow_index) + 1;
	- err = mlx5_cmd_fs_create_fg(dev, fg_in,
	- ft->vport, ft->type, ft->id,
	- &fg->id);
	- if (err)
	- goto free_fg;
	-
	- mutex_lock(&ft->base.lock);
	-
	- if (ft->autogroup.active && group_size == ft->autogroup.group_size)
	- ft->autogroup.num_types++;
	-
	- snprintf(name, sizeof(name), "group_%u", fg->id);
	- /Add node to tree/
	- fs_add_node(&fg->base, &ft->base, name, refcount);
	- /Add node to group list/
	- list_add(&fg->base.list, prev);
	- mutex_unlock(&ft->base.lock);
	-
	- return fg;
	-
	-free_fg:
	- kfree(fg);
	- return ERR_PTR(err);
	-}
	-
	-struct mlx5_flow_group mlx5_create_flow_group(struct mlx5_flow_table ft,
	- u32 *in)
	-{
	- struct mlx5_flow_group *fg;
	- struct mlx5_core_dev *dev = fs_get_dev(&ft->base);
	- unsigned int start_index;
	-
	- start_index = MLX5_GET(create_flow_group_in, in, start_flow_index);
	- if (!dev)
	- return ERR_PTR(-ENODEV);
	-
	- if (ft->autogroup.active && start_index < ft->autogroup.max_fte)
	- return ERR_PTR(-EPERM);
	-
	- fg = fs_create_fg(dev, ft, ft->fgs.prev, in, 1);
	-
	- return fg;
	-}
	-EXPORT_SYMBOL(mlx5_create_flow_group);
	-
	-/Group is destoyed when all the rules in the group were removed/
	-static void fs_del_fg(struct mlx5_flow_group *fg)
	-{
	- struct mlx5_flow_table *parent_ft;
	- struct mlx5_core_dev *dev;
	-
	- fs_get_parent(parent_ft, fg);
	- dev = fs_get_dev(&parent_ft->base);
	- WARN_ON(!dev);
	-
	- if (parent_ft->autogroup.active &&
	- fg->max_ftes == parent_ft->autogroup.group_size &&
	- fg->start_index < parent_ft->autogroup.max_fte)
	- parent_ft->autogroup.num_types--;
	-
	- if (mlx5_cmd_fs_destroy_fg(dev, parent_ft->vport,
	- parent_ft->type,
	- parent_ft->id, fg->id))
	- mlx5_core_warn(dev, "flow steering can't destroy fg\n");
	-}
	-
	-void mlx5_destroy_flow_group(struct mlx5_flow_group *fg)
	-{
	- fs_remove_node(&fg->base);
	-}
	-EXPORT_SYMBOL(mlx5_destroy_flow_group);
	-
	-static bool _fs_match_exact_val(void mask, void val1, void *val2, size_t size)
	-{
	- unsigned int i;
	-
	- /* TODO: optimize by comparing 64bits when possible */
	- for (i = 0; i < size; i++, mask++, val1++, val2++)
	- if ((((u8 )val1) & ((u8 )mask)) !=
	- (((u8 )val2) & ((u8 )mask)))
	- return false;
	-
	- return true;
	-}
	-
	-bool fs_match_exact_val(struct mlx5_core_fs_mask *mask,
	- void val1, void val2)
	-{
	- if (mask->match_criteria_enable &
	- 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS) {
	- void *fte_match1 = MLX5_ADDR_OF(fte_match_param,
	- val1, outer_headers);
	- void *fte_match2 = MLX5_ADDR_OF(fte_match_param,
	- val2, outer_headers);
	- void *fte_mask = MLX5_ADDR_OF(fte_match_param,
	- mask->match_criteria, outer_headers);
	-
	- if (!_fs_match_exact_val(fte_mask, fte_match1, fte_match2,
	- MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4)))
	- return false;
	- }
	-
	- if (mask->match_criteria_enable &
	- 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS) {
	- void *fte_match1 = MLX5_ADDR_OF(fte_match_param,
	- val1, misc_parameters);
	- void *fte_match2 = MLX5_ADDR_OF(fte_match_param,
	- val2, misc_parameters);
	- void *fte_mask = MLX5_ADDR_OF(fte_match_param,
	- mask->match_criteria, misc_parameters);
	-
	- if (!_fs_match_exact_val(fte_mask, fte_match1, fte_match2,
	- MLX5_ST_SZ_BYTES(fte_match_set_misc)))
	- return false;
	- }
	- if (mask->match_criteria_enable &
	- 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_INNER_HEADERS) {
	- void *fte_match1 = MLX5_ADDR_OF(fte_match_param,
	- val1, inner_headers);
	- void *fte_match2 = MLX5_ADDR_OF(fte_match_param,
	- val2, inner_headers);
	- void *fte_mask = MLX5_ADDR_OF(fte_match_param,
	- mask->match_criteria, inner_headers);
	-
	- if (!_fs_match_exact_val(fte_mask, fte_match1, fte_match2,
	- MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4)))
	- return false;
	- }
	- return true;
	-}
	-
	-bool fs_match_exact_mask(u8 match_criteria_enable1,
	- u8 match_criteria_enable2,
	- void mask1, void mask2)
	-{
	- return match_criteria_enable1 == match_criteria_enable2 &&
	- !memcmp(mask1, mask2, MLX5_ST_SZ_BYTES(fte_match_param));
	-}
	-
	-static struct mlx5_flow_table find_first_ft_in_ns_reverse(struct mlx5_flow_namespace ns,
	- struct list_head *start);
	-
	-static struct mlx5_flow_table _find_first_ft_in_prio_reverse(struct fs_prio prio,
	- struct list_head *start)
	-{
	- struct fs_base *it = container_of(start, struct fs_base, list);
	-
	- if (!prio)
	- return NULL;
	-
	- fs_for_each_ns_or_ft_continue_reverse(it, prio) {
	- struct mlx5_flow_namespace *ns;
	- struct mlx5_flow_table *ft;
	-
	- if (it->type == FS_TYPE_FLOW_TABLE) {
	- fs_get_obj(ft, it);
	- fs_get(&ft->base);
	- return ft;
	- }
	-
	- fs_get_obj(ns, it);
	- WARN_ON(ns->base.type != FS_TYPE_NAMESPACE);
	-
	- ft = find_first_ft_in_ns_reverse(ns, &ns->prios);
	- if (ft)
	- return ft;
	- }
	-
	- return NULL;
	-}
	-
	-static struct mlx5_flow_table find_first_ft_in_prio_reverse(struct fs_prio prio,
	- struct list_head *start)
	-{
	- struct mlx5_flow_table *ft;
	-
	- if (!prio)
	- return NULL;
	-
	- mutex_lock(&prio->base.lock);
	- ft = _find_first_ft_in_prio_reverse(prio, start);
	- mutex_unlock(&prio->base.lock);
	-
	- return ft;
	-}
	-
	-static struct mlx5_flow_table find_first_ft_in_ns_reverse(struct mlx5_flow_namespace ns,
	- struct list_head *start)
	-{
	- struct fs_prio *prio;
	-
	- if (!ns)
	- return NULL;
	-
	- fs_get_obj(prio, container_of(start, struct fs_base, list));
	- mutex_lock(&ns->base.lock);
	- fs_for_each_prio_continue_reverse(prio, ns) {
	- struct mlx5_flow_table *ft;
	-
	- ft = find_first_ft_in_prio_reverse(prio, &prio->objs);
	- if (ft) {
	- mutex_unlock(&ns->base.lock);
	- return ft;
	- }
	- }
	- mutex_unlock(&ns->base.lock);
	-
	- return NULL;
	-}
	-
	-/* Returned a held ft, assumed curr is protected, assumed curr's parent is
	- * locked
	- */
	-static struct mlx5_flow_table find_prev_ft(struct mlx5_flow_table curr,
	- struct fs_prio *prio)
	-{
	- struct mlx5_flow_table *ft = NULL;
	- struct fs_base *curr_base;
	-
	- if (!curr)
	- return NULL;
	-
	- /* prio has either namespace or flow-tables, but not both */
	- if (!list_empty(&prio->objs) &&
	- list_first_entry(&prio->objs, struct mlx5_flow_table, base.list) !=
	- curr)
	- return NULL;
	-
	- while (!ft && prio) {
	- struct mlx5_flow_namespace *ns;
	-
	- fs_get_parent(ns, prio);
	- ft = find_first_ft_in_ns_reverse(ns, &prio->base.list);
	- curr_base = &ns->base;
	- fs_get_parent(prio, ns);
	-
	- if (prio && !ft)
	- ft = find_first_ft_in_prio_reverse(prio,
	- &curr_base->list);
	- }
	- return ft;
	-}
	-
	-static struct mlx5_flow_table _find_first_ft_in_prio(struct fs_prio prio,
	- struct list_head *start)
	-{
	- struct fs_base *it = container_of(start, struct fs_base, list);
	-
	- if (!prio)
	- return NULL;
	-
	- fs_for_each_ns_or_ft_continue(it, prio) {
	- struct mlx5_flow_namespace *ns;
	- struct mlx5_flow_table *ft;
	-
	- if (it->type == FS_TYPE_FLOW_TABLE) {
	- fs_get_obj(ft, it);
	- fs_get(&ft->base);
	- return ft;
	- }
	-
	- fs_get_obj(ns, it);
	- WARN_ON(ns->base.type != FS_TYPE_NAMESPACE);
	-
	- ft = find_first_ft_in_ns(ns, &ns->prios);
	- if (ft)
	- return ft;
	- }
	-
	- return NULL;
	-}
	-
	-static struct mlx5_flow_table find_first_ft_in_prio(struct fs_prio prio,
	- struct list_head *start)
	-{
	- struct mlx5_flow_table *ft;
	-
	- if (!prio)
	- return NULL;
	-
	- mutex_lock(&prio->base.lock);
	- ft = _find_first_ft_in_prio(prio, start);
	- mutex_unlock(&prio->base.lock);
	-
	- return ft;
	-}
	-
	-static struct mlx5_flow_table find_first_ft_in_ns(struct mlx5_flow_namespace ns,
	- struct list_head *start)
	-{
	- struct fs_prio *prio;
	-
	- if (!ns)
	- return NULL;
	-
	- fs_get_obj(prio, container_of(start, struct fs_base, list));
	- mutex_lock(&ns->base.lock);
	- fs_for_each_prio_continue(prio, ns) {
	- struct mlx5_flow_table *ft;
	-
	- ft = find_first_ft_in_prio(prio, &prio->objs);
	- if (ft) {
	- mutex_unlock(&ns->base.lock);
	- return ft;
	- }
	- }
	- mutex_unlock(&ns->base.lock);
	-
	- return NULL;
	-}
	-
	-/* returned a held ft, assumed curr is protected, assumed curr's parent is
	- * locked
	- */
	-static struct mlx5_flow_table find_next_ft(struct fs_prio prio)
	-{
	- struct mlx5_flow_table *ft = NULL;
	- struct fs_base *curr_base;
	-
	- while (!ft && prio) {
	- struct mlx5_flow_namespace *ns;
	-
	- fs_get_parent(ns, prio);
	- ft = find_first_ft_in_ns(ns, &prio->base.list);
	- curr_base = &ns->base;
	- fs_get_parent(prio, ns);
	-
	- if (!ft && prio)
	- ft = _find_first_ft_in_prio(prio, &curr_base->list);
	- }
	- return ft;
	-}
	-
	-
	-/* called under ft mutex lock */
	-static struct mlx5_flow_group create_autogroup(struct mlx5_flow_table ft,
	- u8 match_criteria_enable,
	- u32 *match_criteria)
	-{
	- unsigned int group_size;
	- unsigned int candidate_index = 0;
	- struct mlx5_flow_group *g;
	- struct mlx5_flow_group *ret;
	- struct list_head *prev = &ft->fgs;
	- struct mlx5_core_dev *dev;
	- u32 *in;
	- int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
	- void *match_criteria_addr;
	- u32 max_fte = ft->autogroup.max_fte;
	-
	- if (!ft->autogroup.active)
	- return ERR_PTR(-ENOENT);
	-
	- dev = fs_get_dev(&ft->base);
	- if (!dev)
	- return ERR_PTR(-ENODEV);
	-
	- in = mlx5_vzalloc(inlen);
	- if (!in) {
	- mlx5_core_warn(dev, "failed to allocate inbox\n");
	- return ERR_PTR(-ENOMEM);
	- }
	-
	-
	- if (ft->autogroup.num_types < ft->autogroup.max_types)
	- group_size = ft->autogroup.group_size;
	- else
	- group_size = 1;
	-
	- if (group_size == 0) {
	- mlx5_core_warn(dev,
	- "flow steering can't create group size of 0\n");
	- ret = ERR_PTR(-EINVAL);
	- goto out;
	- }
	-
	- /* sorted by start_index */
	- fs_for_each_fg(g, ft) {
	- if (candidate_index + group_size > g->start_index)
	- candidate_index = g->start_index + g->max_ftes;
	- else
	- break;
	- prev = &g->base.list;
	- }
	-
	- if (candidate_index + group_size > max_fte) {
	- ret = ERR_PTR(-ENOSPC);
	- goto out;
	- }
	-
	- MLX5_SET(create_flow_group_in, in, match_criteria_enable,
	- match_criteria_enable);
	- MLX5_SET(create_flow_group_in, in, start_flow_index, candidate_index);
	- MLX5_SET(create_flow_group_in, in, end_flow_index, candidate_index +
	- group_size - 1);
	- match_criteria_addr = MLX5_ADDR_OF(create_flow_group_in,
	- in, match_criteria);
	- memcpy(match_criteria_addr, match_criteria,
	- MLX5_ST_SZ_BYTES(fte_match_param));
	-
	- ret = fs_create_fg(dev, ft, prev, in, 0);
	-out:
	- kvfree(in);
	- return ret;
	-}
	-
	-static struct mlx5_flow_namespace get_ns_with_notifiers(struct fs_base node)
	-{
	- struct mlx5_flow_namespace *ns = NULL;
	-
	- while (node && (node->type != FS_TYPE_NAMESPACE \|\|
	- list_empty(&container_of(node, struct
	- mlx5_flow_namespace,
	- base)->list_notifiers)))
	- node = node->parent;
	-
	- if (node)
	- fs_get_obj(ns, node);
	-
	- return ns;
	-}
	-
	-
	-/Assumption- fte is locked/
	-static void call_to_add_rule_notifiers(struct mlx5_flow_rule *dst,
	- struct fs_fte *fte)
	-{
	- struct mlx5_flow_namespace *ns;
	- struct mlx5_flow_handler *iter_handler;
	- struct fs_client_priv_data *iter_client;
	- void *data;
	- bool is_new_rule = list_first_entry(&fte->dests,
	- struct mlx5_flow_rule,
	- base.list) == dst;
	- int err;
	-
	- ns = get_ns_with_notifiers(&fte->base);
	- if (!ns)
	- return;
	-
	- down_read(&ns->notifiers_rw_sem);
	- list_for_each_entry(iter_handler, &ns->list_notifiers,
	- list) {
	- if (iter_handler->add_dst_cb) {
	- data = NULL;
	- mutex_lock(&dst->clients_lock);
	- list_for_each_entry(
	- iter_client, &dst->clients_data, list) {
	- if (iter_client->fs_handler == iter_handler) {
	- data = iter_client->client_dst_data;
	- break;
	- }
	- }
	- mutex_unlock(&dst->clients_lock);
	- err = iter_handler->add_dst_cb(dst,
	- is_new_rule,
	- data,
	- iter_handler->client_context);
	- if (err)
	- break;
	- }
	- }
	- up_read(&ns->notifiers_rw_sem);
	-}
	-
	-static void call_to_del_rule_notifiers(struct mlx5_flow_rule *dst,
	- struct fs_fte *fte)
	-{
	- struct mlx5_flow_namespace *ns;
	- struct mlx5_flow_handler *iter_handler;
	- struct fs_client_priv_data *iter_client;
	- void *data;
	- bool ctx_changed = (fte->dests_size == 0);
	-
	- ns = get_ns_with_notifiers(&fte->base);
	- if (!ns)
	- return;
	- down_read(&ns->notifiers_rw_sem);
	- list_for_each_entry(iter_handler, &ns->list_notifiers,
	- list) {
	- data = NULL;
	- mutex_lock(&dst->clients_lock);
	- list_for_each_entry(iter_client, &dst->clients_data, list) {
	- if (iter_client->fs_handler == iter_handler) {
	- data = iter_client->client_dst_data;
	- break;
	- }
	- }
	- mutex_unlock(&dst->clients_lock);
	- if (iter_handler->del_dst_cb) {
	- iter_handler->del_dst_cb(dst, ctx_changed, data,
	- iter_handler->client_context);
	- }
	- }
	- up_read(&ns->notifiers_rw_sem);
	-}
	-
	-/* fte should not be deleted while calling this function */
	-static struct mlx5_flow_rule _fs_add_dst_fte(struct fs_fte fte,
	- struct mlx5_flow_group *fg,
	- struct mlx5_flow_destination *dest)
	-{
	- struct mlx5_flow_table *ft;
	- struct mlx5_flow_rule *dst;
	- int err;
	-
	- dst = kzalloc(sizeof(*dst), GFP_KERNEL);
	- if (!dst)
	- return ERR_PTR(-ENOMEM);
	-
	- memcpy(&dst->dest_attr, dest, sizeof(*dest));
	- dst->base.type = FS_TYPE_FLOW_DEST;
	- INIT_LIST_HEAD(&dst->clients_data);
	- mutex_init(&dst->clients_lock);
	- fs_get_parent(ft, fg);
	- /Add dest to dests list- added as first element after the head/
	- list_add_tail(&dst->base.list, &fte->dests);
	- fte->dests_size++;
	- err = mlx5_cmd_fs_set_fte(fs_get_dev(&ft->base),
	- ft->vport,
	- &fte->status,
	- fte->val, ft->type,
	- ft->id, fte->index, fg->id, &fte->flow_act,
	- fte->sw_action, fte->dests_size, &fte->dests);
	- if (err)
	- goto free_dst;
	-
	- list_del(&dst->base.list);
	-
	- return dst;
	-
	-free_dst:
	- list_del(&dst->base.list);
	- kfree(dst);
	- fte->dests_size--;
	- return ERR_PTR(err);
	-}
	-
	-static char get_dest_name(struct mlx5_flow_destination dest)
	-{
	- char name = kzalloc(sizeof(char) 20, GFP_KERNEL);
	-
	- switch (dest->type) {
	- case MLX5_FLOW_CONTEXT_DEST_TYPE_FLOW_TABLE:
	- snprintf(name, 20, "dest_%s_%u", "flow_table",
	- dest->ft->id);
	- return name;
	- case MLX5_FLOW_CONTEXT_DEST_TYPE_VPORT:
	- snprintf(name, 20, "dest_%s_%u", "vport",
	- dest->vport_num);
	- return name;
	- case MLX5_FLOW_CONTEXT_DEST_TYPE_TIR:
	- snprintf(name, 20, "dest_%s_%u", "tir", dest->tir_num);
	- return name;
	- default:
	- kfree(name);
	- return NULL;
	- }
	-}
	-
	-/* assumed fg is locked */
	-static unsigned int fs_get_free_fg_index(struct mlx5_flow_group *fg,
	- struct list_head **prev)
	-{
	- struct fs_fte *fte;
	- unsigned int start = fg->start_index;
	-
	- if (prev)
	- *prev = &fg->ftes;
	-
	- /* assumed list is sorted by index */
	- fs_for_each_fte(fte, fg) {
	- if (fte->index != start)
	- return start;
	- start++;
	- if (prev)
	- *prev = &fte->base.list;
	- }
	-
	- return start;
	-}
	-
	-
	-static struct fs_fte fs_create_fte(struct mlx5_flow_group fg,
	- u32 *match_value,
	- u32 sw_action,
	- struct mlx5_flow_act *flow_act,
	- struct list_head **prev)
	-{
	- struct fs_fte *fte;
	- int index = 0;
	-
	- index = fs_get_free_fg_index(fg, prev);
	- fte = fs_alloc_fte(sw_action, flow_act, match_value, index);
	- if (IS_ERR(fte))
	- return fte;
	-
	- return fte;
	-}
	-
	-static void add_rule_to_tree(struct mlx5_flow_rule *rule,
	- struct fs_fte *fte)
	-{
	- char *dest_name;
	-
	- dest_name = get_dest_name(&rule->dest_attr);
	- fs_add_node(&rule->base, &fte->base, dest_name, 1);
	- /* re-add to list, since fs_add_node reset our list */
	- list_add_tail(&rule->base.list, &fte->dests);
	- kfree(dest_name);
	- call_to_add_rule_notifiers(rule, fte);
	-}
	-
	-static void fs_del_dst(struct mlx5_flow_rule *dst)
	-{
	- struct mlx5_flow_table *ft;
	- struct mlx5_flow_group *fg;
	- struct fs_fte *fte;
	- u32 *match_value;
	- struct mlx5_core_dev *dev = fs_get_dev(&dst->base);
	- int match_len = MLX5_ST_SZ_BYTES(fte_match_param);
	- int err;
	-
	- WARN_ON(!dev);
	-
	- match_value = mlx5_vzalloc(match_len);
	- if (!match_value) {
	- mlx5_core_warn(dev, "failed to allocate inbox\n");
	- return;
	- }
	-
	- fs_get_parent(fte, dst);
	- fs_get_parent(fg, fte);
	- mutex_lock(&fg->base.lock);
	- memcpy(match_value, fte->val, sizeof(fte->val));
	- /* ft can't be changed as fg is locked */
	- fs_get_parent(ft, fg);
	- list_del(&dst->base.list);
	- fte->dests_size--;
	- if (fte->dests_size) {
	- err = mlx5_cmd_fs_set_fte(dev, ft->vport,
	- &fte->status, match_value, ft->type,
	- ft->id, fte->index, fg->id,
	- &fte->flow_act, fte->sw_action,
	- fte->dests_size, &fte->dests);
	- if (err) {
	- mlx5_core_warn(dev, "%s can't delete dst %s\n",
	- __func__, dst->base.name);
	- goto err;
	- }
	- }
	- call_to_del_rule_notifiers(dst, fte);
	-err:
	- mutex_unlock(&fg->base.lock);
	- kvfree(match_value);
	-}
	-
	-static void fs_del_fte(struct fs_fte *fte)
	-{
	- struct mlx5_flow_table *ft;
	- struct mlx5_flow_group *fg;
	- int err;
	- struct mlx5_core_dev *dev;
	-
	- fs_get_parent(fg, fte);
	- fs_get_parent(ft, fg);
	-
	- dev = fs_get_dev(&ft->base);
	- WARN_ON(!dev);
	-
	- err = mlx5_cmd_fs_delete_fte(dev, ft->vport, &fte->status,
	- ft->type, ft->id, fte->index);
	- if (err)
	- mlx5_core_warn(dev, "flow steering can't delete fte %s\n",
	- fte->base.name);
	-
	- fg->num_ftes--;
	-}
	-
	-static bool check_conflicting_actions(const struct mlx5_flow_act *act1,
	- const struct mlx5_flow_act *act2)
	-{
	- u32 action1 = act1->actions;
	- u32 action2 = act2->actions;
	- u32 xored_actions;
	-
	- xored_actions = action1 ^ action2;
	-
	- if (xored_actions & (MLX5_FLOW_ACT_ACTIONS_FLOW_TAG))
	- return true;
	-
	- if (action1 & MLX5_FLOW_ACT_ACTIONS_FLOW_TAG &&
	- act1->flow_tag != act2->flow_tag)
	- return true;
	-
	- /* Can even have complex actions in merged rules */
	- if (action1 & MLX5_FLOW_ACT_ACTIONS_MODIFY_HDR)
	- return true;
	-
	- if (action1 & MLX5_FLOW_ACT_ACTIONS_PACKET_REFORMAT)
	- return true;
	-
	- if (action1 & MLX5_FLOW_ACT_ACTIONS_COUNT)
	- return true;
	-
	- return false;
	-}
	-
	-/* assuming parent fg is locked */
	-/* Add dst algorithm */
	-static struct mlx5_flow_rule fs_add_dst_fg(struct mlx5_flow_group fg,
	- u32 *match_value,
	- u32 sw_action,
	- struct mlx5_flow_act *flow_act,
	- struct mlx5_flow_destination *dest)
	-{
	- struct fs_fte *fte;
	- struct mlx5_flow_rule *dst;
	- struct mlx5_flow_table *ft;
	- struct list_head *prev;
	- char fte_name[20];
	-
	- mutex_lock(&fg->base.lock);
	- if (flow_act->flags & MLX5_FLOW_ACT_NO_APPEND)
	- goto insert_fte;
	-
	- fs_for_each_fte(fte, fg) {
	- /* TODO: Check of size against PRM max size */
	- mutex_lock(&fte->base.lock);
	- if (fs_match_exact_val(&fg->mask, match_value, &fte->val) &&
	- sw_action == fte->sw_action &&
	- !check_conflicting_actions(flow_act, &fte->flow_act)) {
	- dst = _fs_add_dst_fte(fte, fg, dest);
	- mutex_unlock(&fte->base.lock);
	- if (IS_ERR(dst))
	- goto unlock_fg;
	- goto add_rule;
	- }
	- mutex_unlock(&fte->base.lock);
	- }
	-
	-insert_fte:
	- fs_get_parent(ft, fg);
	- if (fg->num_ftes == fg->max_ftes) {
	- dst = ERR_PTR(-ENOSPC);
	- goto unlock_fg;
	- }
	-
	- fte = fs_create_fte(fg, match_value, sw_action, flow_act, &prev);
	- if (IS_ERR(fte)) {
	- dst = (void *)fte;
	- goto unlock_fg;
	- }
	- dst = _fs_add_dst_fte(fte, fg, dest);
	- if (IS_ERR(dst)) {
	- kfree(fte);
	- goto unlock_fg;
	- }
	-
	- fg->num_ftes++;
	-
	- snprintf(fte_name, sizeof(fte_name), "fte%u", fte->index);
	- /* Add node to tree */
	- fs_add_node(&fte->base, &fg->base, fte_name, 0);
	- list_add(&fte->base.list, prev);
	-add_rule:
	- add_rule_to_tree(dst, fte);
	-unlock_fg:
	- mutex_unlock(&fg->base.lock);
	- return dst;
	-}
	-
	-static struct mlx5_flow_rule fs_add_dst_ft(struct mlx5_flow_table ft,
	- u8 match_criteria_enable,
	- u32 *match_criteria,
	- u32 *match_value,
	- u32 sw_action,
	- struct mlx5_flow_act *flow_act,
	- struct mlx5_flow_destination *dest)
	-{
	- /? where dst_entry is allocated/
	- struct mlx5_flow_group *g;
	- struct mlx5_flow_rule *dst;
	-
	- fs_get(&ft->base);
	- mutex_lock(&ft->base.lock);
	- fs_for_each_fg(g, ft)
	- if (fs_match_exact_mask(g->mask.match_criteria_enable,
	- match_criteria_enable,
	- g->mask.match_criteria,
	- match_criteria)) {
	- mutex_unlock(&ft->base.lock);
	-
	- dst = fs_add_dst_fg(g, match_value, sw_action, flow_act, dest);
	- if (PTR_ERR(dst) && PTR_ERR(dst) != -ENOSPC)
	- goto unlock;
	- }
	- mutex_unlock(&ft->base.lock);
	-
	- g = create_autogroup(ft, match_criteria_enable, match_criteria);
	- if (IS_ERR(g)) {
	- dst = (void *)g;
	- goto unlock;
	- }
	-
	- dst = fs_add_dst_fg(g, match_value,
	- sw_action, flow_act, dest);
	- if (IS_ERR(dst)) {
	- /* Remove assumes refcount > 0 and autogroup creates a group
	- * with a refcount = 0.
	- */
	- fs_get(&g->base);
	- fs_remove_node(&g->base);
	- goto unlock;
	- }
	-
	-unlock:
	- fs_put(&ft->base);
	- return dst;
	-}
	-
	-struct mlx5_flow_rule *
	-mlx5_add_flow_rule(struct mlx5_flow_table *ft,
	- u8 match_criteria_enable,
	- u32 *match_criteria,
	- u32 *match_value,
	- u32 sw_action,
	- struct mlx5_flow_act *flow_act,
	- struct mlx5_flow_destination *dest)
	-{
	- struct mlx5_flow_rule *dst;
	- struct mlx5_flow_namespace *ns;
	-
	- ns = get_ns_with_notifiers(&ft->base);
	- if (ns)
	- down_read(&ns->dests_rw_sem);
	- dst = fs_add_dst_ft(ft, match_criteria_enable, match_criteria,
	- match_value, sw_action, flow_act, dest);
	- if (ns)
	- up_read(&ns->dests_rw_sem);
	-
	- return dst;
	-
	-
	-}
	-EXPORT_SYMBOL(mlx5_add_flow_rule);
	-
	-void mlx5_del_flow_rule(struct mlx5_flow_rule **pp)
	-{
	- struct mlx5_flow_namespace *ns;
	- struct mlx5_flow_rule *dst;
	-
	- dst = *pp;
	- *pp = NULL;
	-
	- if (IS_ERR_OR_NULL(dst))
	- return;
	- ns = get_ns_with_notifiers(&dst->base);
	- if (ns)
	- down_read(&ns->dests_rw_sem);
	- fs_remove_node(&dst->base);
	- if (ns)
	- up_read(&ns->dests_rw_sem);
	-}
	-EXPORT_SYMBOL(mlx5_del_flow_rule);
	-
	-#define MLX5_CORE_FS_ROOT_NS_NAME "root"
	-#define MLX5_CORE_FS_ESW_EGRESS_ACL "esw_egress_root"
	-#define MLX5_CORE_FS_ESW_INGRESS_ACL "esw_ingress_root"
	-#define MLX5_CORE_FS_FDB_ROOT_NS_NAME "fdb_root"
	-#define MLX5_CORE_FS_SNIFFER_RX_ROOT_NS_NAME "sniffer_rx_root"
	-#define MLX5_CORE_FS_SNIFFER_TX_ROOT_NS_NAME "sniffer_tx_root"
	-#define MLX5_CORE_FS_PRIO_MAX_FT 4
	-#define MLX5_CORE_FS_PRIO_MAX_NS 1
	-
	-static struct fs_prio fs_create_prio(struct mlx5_flow_namespace ns,
	- unsigned prio, int max_ft,
	- const char *name, u8 flags)
	-{
	- struct fs_prio *fs_prio;
	-
	- fs_prio = kzalloc(sizeof(*fs_prio), GFP_KERNEL);
	- if (!fs_prio)
	- return ERR_PTR(-ENOMEM);
	-
	- fs_prio->base.type = FS_TYPE_PRIO;
	- fs_add_node(&fs_prio->base, &ns->base, name, 1);
	- fs_prio->max_ft = max_ft;
	- fs_prio->max_ns = MLX5_CORE_FS_PRIO_MAX_NS;
	- fs_prio->prio = prio;
	- fs_prio->flags = flags;
	- list_add_tail(&fs_prio->base.list, &ns->prios);
	- INIT_LIST_HEAD(&fs_prio->objs);
	- mutex_init(&fs_prio->shared_lock);
	-
	- return fs_prio;
	-}
	-
	-static void cleanup_root_ns(struct mlx5_core_dev *dev)
	-{
	- struct mlx5_flow_root_namespace *root_ns = dev->root_ns;
	- struct fs_prio *iter_prio;
	-
	- if (!root_ns)
	- return;
	-
	- /* stage 1 */
	- fs_for_each_prio(iter_prio, &root_ns->ns) {
	- struct mlx5_flow_namespace *iter_ns;
	-
	- fs_for_each_ns(iter_ns, iter_prio) {
	- while (!list_empty(&iter_ns->prios)) {
	- struct fs_base *iter_prio2 =
	- list_first_entry(&iter_ns->prios,
	- struct fs_base,
	- list);
	-
	- fs_remove_node(iter_prio2);
	- }
	- }
	- }
	-
	- /* stage 2 */
	- fs_for_each_prio(iter_prio, &root_ns->ns) {
	- while (!list_empty(&iter_prio->objs)) {
	- struct fs_base *iter_ns =
	- list_first_entry(&iter_prio->objs,
	- struct fs_base,
	- list);
	-
	- fs_remove_node(iter_ns);
	- }
	- }
	- /* stage 3 */
	- while (!list_empty(&root_ns->ns.prios)) {
	- struct fs_base *iter_prio =
	- list_first_entry(&root_ns->ns.prios,
	- struct fs_base,
	- list);
	-
	- fs_remove_node(iter_prio);
	- }
	-
	- fs_remove_node(&root_ns->ns.base);
	- dev->root_ns = NULL;
	-}
	-
	-static void cleanup_single_prio_root_ns(struct mlx5_core_dev *dev,
	- struct mlx5_flow_root_namespace *root_ns)
	-{
	- struct fs_base *prio;
	-
	- if (!root_ns)
	- return;
	-
	- if (!list_empty(&root_ns->ns.prios)) {
	- prio = list_first_entry(&root_ns->ns.prios,
	- struct fs_base,
	- list);
	- fs_remove_node(prio);
	- }
	- fs_remove_node(&root_ns->ns.base);
	- root_ns = NULL;
	-}
	-
	-void mlx5_cleanup_fs(struct mlx5_core_dev *dev)
	-{
	- mlx5_cleanup_fc_stats(dev);
	- cleanup_root_ns(dev);
	- cleanup_single_prio_root_ns(dev, dev->sniffer_rx_root_ns);
	- cleanup_single_prio_root_ns(dev, dev->sniffer_tx_root_ns);
	- cleanup_single_prio_root_ns(dev, dev->fdb_root_ns);
	- cleanup_single_prio_root_ns(dev, dev->esw_egress_root_ns);
	- cleanup_single_prio_root_ns(dev, dev->esw_ingress_root_ns);
	-}
	-
	-static struct mlx5_flow_namespace *fs_init_namespace(struct mlx5_flow_namespace
	- *ns)
	-{
	- ns->base.type = FS_TYPE_NAMESPACE;
	- init_rwsem(&ns->dests_rw_sem);
	- init_rwsem(&ns->notifiers_rw_sem);
	- INIT_LIST_HEAD(&ns->prios);
	- INIT_LIST_HEAD(&ns->list_notifiers);
	-
	- return ns;
	-}
	-
	-static struct mlx5_flow_root_namespace create_root_ns(struct mlx5_core_dev dev,
	- enum fs_ft_type
	- table_type,
	- char *name)
	-{
	- struct mlx5_flow_root_namespace *root_ns;
	- struct mlx5_flow_namespace *ns;
	-
	- /* create the root namespace */
	- root_ns = mlx5_vzalloc(sizeof(*root_ns));
	- if (!root_ns)
	- goto err;
	-
	- root_ns->dev = dev;
	- root_ns->table_type = table_type;
	- mutex_init(&root_ns->fs_chain_lock);
	-
	- ns = &root_ns->ns;
	- fs_init_namespace(ns);
	- fs_add_node(&ns->base, NULL, name, 1);
	-
	- return root_ns;
	-err:
	- return NULL;
	-}
	-
	-static int init_fdb_root_ns(struct mlx5_core_dev *dev)
	-{
	- struct fs_prio *prio;
	-
	- dev->fdb_root_ns = create_root_ns(dev, FS_FT_FDB,
	- MLX5_CORE_FS_FDB_ROOT_NS_NAME);
	- if (!dev->fdb_root_ns)
	- return -ENOMEM;
	-
	- /* create 1 prio*/
	- prio = fs_create_prio(&dev->fdb_root_ns->ns, 0, 1, "fdb_prio", 0);
	- if (IS_ERR(prio))
	- return PTR_ERR(prio);
	- else
	- return 0;
	-}
	-
	-#define MAX_VPORTS 128
	-
	-static int init_egress_acl_root_ns(struct mlx5_core_dev *dev)
	-{
	- struct fs_prio *prio;
	-
	- dev->esw_egress_root_ns = create_root_ns(dev, FS_FT_ESW_EGRESS_ACL,
	- MLX5_CORE_FS_ESW_EGRESS_ACL);
	- if (!dev->esw_egress_root_ns)
	- return -ENOMEM;
	-
	- /* create 1 prio*/
	- prio = fs_create_prio(&dev->esw_egress_root_ns->ns, 0, MAX_VPORTS,
	- "esw_egress_prio", 0);
	- if (IS_ERR(prio))
	- return PTR_ERR(prio);
	- else
	- return 0;
	-}
	-
	-static int init_ingress_acl_root_ns(struct mlx5_core_dev *dev)
	-{
	- struct fs_prio *prio;
	-
	- dev->esw_ingress_root_ns = create_root_ns(dev, FS_FT_ESW_INGRESS_ACL,
	- MLX5_CORE_FS_ESW_INGRESS_ACL);
	- if (!dev->esw_ingress_root_ns)
	- return -ENOMEM;
	-
	- /* create 1 prio*/
	- prio = fs_create_prio(&dev->esw_ingress_root_ns->ns, 0, MAX_VPORTS,
	- "esw_ingress_prio", 0);
	- if (IS_ERR(prio))
	- return PTR_ERR(prio);
	- else
	- return 0;
	-}
	-
	-static int init_sniffer_rx_root_ns(struct mlx5_core_dev *dev)
	-{
	- struct fs_prio *prio;
	-
	- dev->sniffer_rx_root_ns = create_root_ns(dev, FS_FT_SNIFFER_RX,
	- MLX5_CORE_FS_SNIFFER_RX_ROOT_NS_NAME);
	- if (!dev->sniffer_rx_root_ns)
	- return -ENOMEM;
	-
	- /* create 1 prio*/
	- prio = fs_create_prio(&dev->sniffer_rx_root_ns->ns, 0, 1,
	- "sniffer_prio", 0);
	- if (IS_ERR(prio))
	- return PTR_ERR(prio);
	- else
	- return 0;
	-}
	-
	-
	-static int init_sniffer_tx_root_ns(struct mlx5_core_dev *dev)
	-{
	- struct fs_prio *prio;
	-
	- dev->sniffer_tx_root_ns = create_root_ns(dev, FS_FT_SNIFFER_TX,
	- MLX5_CORE_FS_SNIFFER_TX_ROOT_NS_NAME);
	- if (!dev->sniffer_tx_root_ns)
	- return -ENOMEM;
	-
	- /* create 1 prio*/
	- prio = fs_create_prio(&dev->sniffer_tx_root_ns->ns, 0, 1,
	- "sniffer_prio", 0);
	- if (IS_ERR(prio))
	- return PTR_ERR(prio);
	- else
	- return 0;
	-}
	-
	-static struct mlx5_flow_namespace fs_create_namespace(struct fs_prio prio,
	- const char *name)
	-{
	- struct mlx5_flow_namespace *ns;
	-
	- ns = kzalloc(sizeof(*ns), GFP_KERNEL);
	- if (!ns)
	- return ERR_PTR(-ENOMEM);
	-
	- fs_init_namespace(ns);
	- fs_add_node(&ns->base, &prio->base, name, 1);
	- list_add_tail(&ns->base.list, &prio->objs);
	-
	- return ns;
	-}
	-
	-#define FLOW_TABLE_BIT_SZ 1
	-#define GET_FLOW_TABLE_CAP(dev, offset) \
	- ((be32_to_cpu(((__be32 )(dev->hca_caps_cur[MLX5_CAP_FLOW_TABLE]) + \
	- offset / 32)) >> \
	- (32 - FLOW_TABLE_BIT_SZ - (offset & 0x1f))) & FLOW_TABLE_BIT_SZ)
	-
	-static bool has_required_caps(struct mlx5_core_dev dev, struct node_caps caps)
	-{
	- int i;
	-
	- for (i = 0; i < caps->arr_sz; i++) {
	- if (!GET_FLOW_TABLE_CAP(dev, caps->caps[i]))
	- return false;
	- }
	- return true;
	-}
	-
	-static int _init_root_tree(struct mlx5_core_dev *dev, int max_ft_level,
	- struct init_tree_node node, struct fs_base base_parent,
	- struct init_tree_node *tree_parent)
	-{
	- struct mlx5_flow_namespace *fs_ns;
	- struct fs_prio *fs_prio;
	- int priority;
	- struct fs_base *base;
	- int i;
	- int err = 0;
	-
	- if (node->type == FS_TYPE_PRIO) {
	- if ((node->min_ft_level > max_ft_level) \|\|
	- !has_required_caps(dev, &node->caps))
	- goto out;
	-
	- fs_get_obj(fs_ns, base_parent);
	- priority = node - tree_parent->children;
	- fs_prio = fs_create_prio(fs_ns, priority,
	- node->max_ft,
	- node->name, node->flags);
	- if (IS_ERR(fs_prio)) {
	- err = PTR_ERR(fs_prio);
	- goto out;
	- }
	- base = &fs_prio->base;
	- } else if (node->type == FS_TYPE_NAMESPACE) {
	- fs_get_obj(fs_prio, base_parent);
	- fs_ns = fs_create_namespace(fs_prio, node->name);
	- if (IS_ERR(fs_ns)) {
	- err = PTR_ERR(fs_ns);
	- goto out;
	- }
	- base = &fs_ns->base;
	- } else {
	- return -EINVAL;
	- }
	- for (i = 0; i < node->ar_size; i++) {
	- err = _init_root_tree(dev, max_ft_level, &node->children[i], base,
	- node);
	- if (err)
	- break;
	- }
	-out:
	- return err;
	-}
	-
	-static int init_root_tree(struct mlx5_core_dev *dev, int max_ft_level,
	- struct init_tree_node node, struct fs_base parent)
	-{
	- int i;
	- struct mlx5_flow_namespace *fs_ns;
	- int err = 0;
	-
	- fs_get_obj(fs_ns, parent);
	- for (i = 0; i < node->ar_size; i++) {
	- err = _init_root_tree(dev, max_ft_level,
	- &node->children[i], &fs_ns->base, node);
	- if (err)
	- break;
	- }
	- return err;
	-}
	-
	-static int sum_max_ft_in_prio(struct fs_prio *prio);
	-static int sum_max_ft_in_ns(struct mlx5_flow_namespace *ns)
	-{
	- struct fs_prio *prio;
	- int sum = 0;
	-
	- fs_for_each_prio(prio, ns) {
	- sum += sum_max_ft_in_prio(prio);
	- }
	- return sum;
	-}
	-
	-static int sum_max_ft_in_prio(struct fs_prio *prio)
	-{
	- int sum = 0;
	- struct fs_base *it;
	- struct mlx5_flow_namespace *ns;
	-
	- if (prio->max_ft)
	- return prio->max_ft;
	-
	- fs_for_each_ns_or_ft(it, prio) {
	- if (it->type == FS_TYPE_FLOW_TABLE)
	- continue;
	-
	- fs_get_obj(ns, it);
	- sum += sum_max_ft_in_ns(ns);
	- }
	- prio->max_ft = sum;
	- return sum;
	-}
	-
	-static void set_max_ft(struct mlx5_flow_namespace *ns)
	-{
	- struct fs_prio *prio;
	-
	- if (!ns)
	- return;
	-
	- fs_for_each_prio(prio, ns)
	- sum_max_ft_in_prio(prio);
	-}
	-
	-static int init_root_ns(struct mlx5_core_dev *dev)
	-{
	- int max_ft_level = MLX5_CAP_FLOWTABLE(dev,
	- flow_table_properties_nic_receive.
	- max_ft_level);
	-
	- dev->root_ns = create_root_ns(dev, FS_FT_NIC_RX,
	- MLX5_CORE_FS_ROOT_NS_NAME);
	- if (IS_ERR_OR_NULL(dev->root_ns))
	- goto err;
	-
	-
	- if (init_root_tree(dev, max_ft_level, &root_fs, &dev->root_ns->ns.base))
	- goto err;
	-
	- set_max_ft(&dev->root_ns->ns);
	-
	- return 0;
	-err:
	- return -ENOMEM;
	-}
	-
	-u8 mlx5_get_match_criteria_enable(struct mlx5_flow_rule *rule)
	-{
	- struct fs_base *pbase;
	- struct mlx5_flow_group *fg;
	-
	- pbase = rule->base.parent;
	- WARN_ON(!pbase);
	- pbase = pbase->parent;
	- WARN_ON(!pbase);
	-
	- fs_get_obj(fg, pbase);
	- return fg->mask.match_criteria_enable;
	-}
	-
	-void mlx5_get_match_value(u32 *match_value,
	- struct mlx5_flow_rule *rule)
	-{
	- struct fs_base *pbase;
	- struct fs_fte *fte;
	-
	- pbase = rule->base.parent;
	- WARN_ON(!pbase);
	- fs_get_obj(fte, pbase);
	-
	- memcpy(match_value, fte->val, sizeof(fte->val));
	-}
	-
	-void mlx5_get_match_criteria(u32 *match_criteria,
	- struct mlx5_flow_rule *rule)
	-{
	- struct fs_base *pbase;
	- struct mlx5_flow_group *fg;
	-
	- pbase = rule->base.parent;
	- WARN_ON(!pbase);
	- pbase = pbase->parent;
	- WARN_ON(!pbase);
	-
	- fs_get_obj(fg, pbase);
	- memcpy(match_criteria, &fg->mask.match_criteria,
	- sizeof(fg->mask.match_criteria));
	-}
	-
	-int mlx5_init_fs(struct mlx5_core_dev *dev)
	-{
	- int err;
	-
	- if (MLX5_CAP_GEN(dev, nic_flow_table)) {
	- err = init_root_ns(dev);
	- if (err)
	- goto err;
	- }
	-
	- err = init_fdb_root_ns(dev);
	- if (err)
	- goto err;
	-
	- err = init_egress_acl_root_ns(dev);
	- if (err)
	- goto err;
	-
	- err = init_ingress_acl_root_ns(dev);
	- if (err)
	- goto err;
	-
	- err = init_sniffer_tx_root_ns(dev);
	- if (err)
	- goto err;
	-
	- err = init_sniffer_rx_root_ns(dev);
	- if (err)
	- goto err;
	-
	- err = mlx5_init_fc_stats(dev);
	- if (err)
	- goto err;
	-
	- return 0;
	-err:
	- mlx5_cleanup_fs(dev);
	- return err;
	-}
	-
	-struct mlx5_flow_namespace mlx5_get_flow_namespace(struct mlx5_core_dev dev,
	- enum mlx5_flow_namespace_type type)
	-{
	- struct mlx5_flow_root_namespace *root_ns = dev->root_ns;
	- int prio;
	- static struct fs_prio *fs_prio;
	- struct mlx5_flow_namespace *ns;
	-
	- switch (type) {
	- case MLX5_FLOW_NAMESPACE_BYPASS:
	- prio = 0;
	- break;
	- case MLX5_FLOW_NAMESPACE_OFFLOADS:
	- prio = 1;
	- break;
	- case MLX5_FLOW_NAMESPACE_KERNEL:
	- prio = 2;
	- break;
	- case MLX5_FLOW_NAMESPACE_LEFTOVERS:
	- prio = 3;
	- break;
	- case MLX5_FLOW_NAMESPACE_FDB:
	- if (dev->fdb_root_ns)
	- return &dev->fdb_root_ns->ns;
	- else
	- return NULL;
	- case MLX5_FLOW_NAMESPACE_ESW_EGRESS:
	- if (dev->esw_egress_root_ns)
	- return &dev->esw_egress_root_ns->ns;
	- else
	- return NULL;
	- case MLX5_FLOW_NAMESPACE_ESW_INGRESS:
	- if (dev->esw_ingress_root_ns)
	- return &dev->esw_ingress_root_ns->ns;
	- else
	- return NULL;
	- case MLX5_FLOW_NAMESPACE_SNIFFER_RX:
	- if (dev->sniffer_rx_root_ns)
	- return &dev->sniffer_rx_root_ns->ns;
	- else
	- return NULL;
	- case MLX5_FLOW_NAMESPACE_SNIFFER_TX:
	- if (dev->sniffer_tx_root_ns)
	- return &dev->sniffer_tx_root_ns->ns;
	- else
	- return NULL;
	- default:
	- return NULL;
	- }
	-
	- if (!root_ns)
	- return NULL;
	-
	- fs_prio = find_prio(&root_ns->ns, prio);
	- if (!fs_prio)
	- return NULL;
	-
	- ns = list_first_entry(&fs_prio->objs,
	- typeof(*ns),
	- base.list);
	-
	- return ns;
	-}
	-EXPORT_SYMBOL(mlx5_get_flow_namespace);
	-
	-
	-int mlx5_set_rule_private_data(struct mlx5_flow_rule *rule,
	- struct mlx5_flow_handler *fs_handler,
	- void *client_data)
	-{
	- struct fs_client_priv_data *priv_data;
	-
	- mutex_lock(&rule->clients_lock);
	- /Check that hanlder isn't exists in the list already/
	- list_for_each_entry(priv_data, &rule->clients_data, list) {
	- if (priv_data->fs_handler == fs_handler) {
	- priv_data->client_dst_data = client_data;
	- goto unlock;
	- }
	- }
	- priv_data = kzalloc(sizeof(*priv_data), GFP_KERNEL);
	- if (!priv_data) {
	- mutex_unlock(&rule->clients_lock);
	- return -ENOMEM;
	- }
	-
	- priv_data->client_dst_data = client_data;
	- priv_data->fs_handler = fs_handler;
	- list_add(&priv_data->list, &rule->clients_data);
	-
	-unlock:
	- mutex_unlock(&rule->clients_lock);
	-
	- return 0;
	-}
	-
	-static int remove_from_clients(struct mlx5_flow_rule *rule,
	- bool ctx_changed,
	- void *client_data,
	- void *context)
	-{
	- struct fs_client_priv_data *iter_client;
	- struct fs_client_priv_data *temp_client;
	- struct mlx5_flow_handler *handler = (struct
	- mlx5_flow_handler*)context;
	-
	- mutex_lock(&rule->clients_lock);
	- list_for_each_entry_safe(iter_client, temp_client,
	- &rule->clients_data, list) {
	- if (iter_client->fs_handler == handler) {
	- list_del(&iter_client->list);
	- kfree(iter_client);
	- break;
	- }
	- }
	- mutex_unlock(&rule->clients_lock);
	-
	- return 0;
	-}
	-
	-struct mlx5_flow_handler mlx5_register_rule_notifier(struct mlx5_core_dev dev,
	- enum mlx5_flow_namespace_type ns_type,
	- rule_event_fn add_cb,
	- rule_event_fn del_cb,
	- void *context)
	-{
	- struct mlx5_flow_namespace *ns;
	- struct mlx5_flow_handler *handler;
	-
	- ns = mlx5_get_flow_namespace(dev, ns_type);
	- if (!ns)
	- return ERR_PTR(-EINVAL);
	-
	- handler = kzalloc(sizeof(*handler), GFP_KERNEL);
	- if (!handler)
	- return ERR_PTR(-ENOMEM);
	-
	- handler->add_dst_cb = add_cb;
	- handler->del_dst_cb = del_cb;
	- handler->client_context = context;
	- handler->ns = ns;
	- down_write(&ns->notifiers_rw_sem);
	- list_add_tail(&handler->list, &ns->list_notifiers);
	- up_write(&ns->notifiers_rw_sem);
	-
	- return handler;
	-}
	-
	-static void iterate_rules_in_ns(struct mlx5_flow_namespace *ns,
	- rule_event_fn add_rule_cb,
	- void *context);
	-
	-void mlx5_unregister_rule_notifier(struct mlx5_flow_handler *handler)
	-{
	- struct mlx5_flow_namespace *ns = handler->ns;
	-
	- /Remove from dst's clients/
	- down_write(&ns->dests_rw_sem);
	- down_write(&ns->notifiers_rw_sem);
	- iterate_rules_in_ns(ns, remove_from_clients, handler);
	- list_del(&handler->list);
	- up_write(&ns->notifiers_rw_sem);
	- up_write(&ns->dests_rw_sem);
	- kfree(handler);
	-}
	-
	-static void iterate_rules_in_ft(struct mlx5_flow_table *ft,
	- rule_event_fn add_rule_cb,
	- void *context)
	-{
	- struct mlx5_flow_group *iter_fg;
	- struct fs_fte *iter_fte;
	- struct mlx5_flow_rule *iter_rule;
	- int err = 0;
	- bool is_new_rule;
	-
	- mutex_lock(&ft->base.lock);
	- fs_for_each_fg(iter_fg, ft) {
	- mutex_lock(&iter_fg->base.lock);
	- fs_for_each_fte(iter_fte, iter_fg) {
	- mutex_lock(&iter_fte->base.lock);
	- is_new_rule = true;
	- fs_for_each_dst(iter_rule, iter_fte) {
	- fs_get(&iter_rule->base);
	- err = add_rule_cb(iter_rule,
	- is_new_rule,
	- NULL,
	- context);
	- fs_put_parent_locked(&iter_rule->base);
	- if (err)
	- break;
	- is_new_rule = false;
	- }
	- mutex_unlock(&iter_fte->base.lock);
	- if (err)
	- break;
	- }
	- mutex_unlock(&iter_fg->base.lock);
	- if (err)
	- break;
	- }
	- mutex_unlock(&ft->base.lock);
	-}
	-
	-static void iterate_rules_in_prio(struct fs_prio *prio,
	- rule_event_fn add_rule_cb,
	- void *context)
	-{
	- struct fs_base *it;
	-
	- mutex_lock(&prio->base.lock);
	- fs_for_each_ns_or_ft(it, prio) {
	- if (it->type == FS_TYPE_FLOW_TABLE) {
	- struct mlx5_flow_table *ft;
	-
	- fs_get_obj(ft, it);
	- iterate_rules_in_ft(ft, add_rule_cb, context);
	- } else {
	- struct mlx5_flow_namespace *ns;
	-
	- fs_get_obj(ns, it);
	- iterate_rules_in_ns(ns, add_rule_cb, context);
	- }
	- }
	- mutex_unlock(&prio->base.lock);
	-}
	-
	-static void iterate_rules_in_ns(struct mlx5_flow_namespace *ns,
	- rule_event_fn add_rule_cb,
	- void *context)
	-{
	- struct fs_prio *iter_prio;
	-
	- mutex_lock(&ns->base.lock);
	- fs_for_each_prio(iter_prio, ns) {
	- iterate_rules_in_prio(iter_prio, add_rule_cb, context);
	- }
	- mutex_unlock(&ns->base.lock);
	-}
	-
	-void mlx5_flow_iterate_existing_rules(struct mlx5_flow_namespace *ns,
	- rule_event_fn add_rule_cb,
	- void *context)
	-{
	- down_write(&ns->dests_rw_sem);
	- down_read(&ns->notifiers_rw_sem);
	- iterate_rules_in_ns(ns, add_rule_cb, context);
	- up_read(&ns->notifiers_rw_sem);
	- up_write(&ns->dests_rw_sem);
	-}
	-
	-
	-void mlx5_del_flow_rules_list(struct mlx5_flow_rules_list *rules_list)
	-{
	- struct mlx5_flow_rule_node *iter_node;
	- struct mlx5_flow_rule_node *temp_node;
	-
	- list_for_each_entry_safe(iter_node, temp_node, &rules_list->head, list) {
	- list_del(&iter_node->list);
	- kfree(iter_node);
	- }
	-
	- kfree(rules_list);
	-}
	-
	-#define ROCEV1_ETHERTYPE 0x8915
	-static int set_rocev1_rules(struct list_head *rules_list)
	-{
	- struct mlx5_flow_rule_node *rocev1_rule;
	-
	- rocev1_rule = kzalloc(sizeof(*rocev1_rule), GFP_KERNEL);
	- if (!rocev1_rule)
	- return -ENOMEM;
	-
	- rocev1_rule->match_criteria_enable =
	- 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS;
	- MLX5_SET(fte_match_set_lyr_2_4, rocev1_rule->match_criteria, ethertype,
	- 0xffff);
	- MLX5_SET(fte_match_set_lyr_2_4, rocev1_rule->match_value, ethertype,
	- ROCEV1_ETHERTYPE);
	-
	- list_add_tail(&rocev1_rule->list, rules_list);
	-
	- return 0;
	-}
	-
	-#define ROCEV2_UDP_PORT 4791
	-static int set_rocev2_rules(struct list_head *rules_list)
	-{
	- struct mlx5_flow_rule_node *ipv4_rule;
	- struct mlx5_flow_rule_node *ipv6_rule;
	-
	- ipv4_rule = kzalloc(sizeof(*ipv4_rule), GFP_KERNEL);
	- if (!ipv4_rule)
	- return -ENOMEM;
	-
	- ipv6_rule = kzalloc(sizeof(*ipv6_rule), GFP_KERNEL);
	- if (!ipv6_rule) {
	- kfree(ipv4_rule);
	- return -ENOMEM;
	- }
	-
	- ipv4_rule->match_criteria_enable =
	- 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS;
	- MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_criteria, ethertype,
	- 0xffff);
	- MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_value, ethertype,
	- 0x0800);
	- MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_criteria, ip_protocol,
	- 0xff);
	- MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_value, ip_protocol,
	- IPPROTO_UDP);
	- MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_criteria, udp_dport,
	- 0xffff);
	- MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_value, udp_dport,
	- ROCEV2_UDP_PORT);
	-
	- ipv6_rule->match_criteria_enable =
	- 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS;
	- MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_criteria, ethertype,
	- 0xffff);
	- MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_value, ethertype,
	- 0x86dd);
	- MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_criteria, ip_protocol,
	- 0xff);
	- MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_value, ip_protocol,
	- IPPROTO_UDP);
	- MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_criteria, udp_dport,
	- 0xffff);
	- MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_value, udp_dport,
	- ROCEV2_UDP_PORT);
	-
	- list_add_tail(&ipv4_rule->list, rules_list);
	- list_add_tail(&ipv6_rule->list, rules_list);
	-
	- return 0;
	-}
	-
	-
	-struct mlx5_flow_rules_list *get_roce_flow_rules(u8 roce_mode)
	-{
	- int err = 0;
	- struct mlx5_flow_rules_list *rules_list =
	- kzalloc(sizeof(*rules_list), GFP_KERNEL);
	-
	- if (!rules_list)
	- return NULL;
	-
	- INIT_LIST_HEAD(&rules_list->head);
	-
	- if (roce_mode & MLX5_ROCE_VERSION_1_CAP) {
	- err = set_rocev1_rules(&rules_list->head);
	- if (err)
	- goto free_list;
	- }
	- if (roce_mode & MLX5_ROCE_VERSION_2_CAP)
	- err = set_rocev2_rules(&rules_list->head);
	- if (err)
	- goto free_list;
	-
	- return rules_list;
	-
	-free_list:
	- mlx5_del_flow_rules_list(rules_list);
	- return NULL;
	-}
	-
	-struct mlx5_modify_hdr mlx5_modify_header_alloc(struct mlx5_core_dev dev,
	- enum mlx5_flow_namespace_type ns_type,
	- u8 num_actions,
	- void *modify_actions)
	-{
	- struct mlx5_modify_hdr *modify_hdr;
	- int err;
	-
	- modify_hdr = kzalloc(sizeof(*modify_hdr), GFP_KERNEL);
	- if (!modify_hdr)
	- return ERR_PTR(-ENOMEM);
	-
	- modify_hdr->ns_type = ns_type;
	- err = mlx5_cmd_modify_header_alloc(dev, ns_type, num_actions,
	- modify_actions, modify_hdr);
	- if (err) {
	- kfree(modify_hdr);
	- return ERR_PTR(err);
	- }
	-
	- return modify_hdr;
	-}
	-EXPORT_SYMBOL(mlx5_modify_header_alloc);
	-
	-void mlx5_modify_header_dealloc(struct mlx5_core_dev *dev,
	- struct mlx5_modify_hdr *modify_hdr)
	-{
	- mlx5_cmd_modify_header_dealloc(dev, modify_hdr);
	- kfree(modify_hdr);
	-}
	-EXPORT_SYMBOL(mlx5_modify_header_dealloc);
	-
	-struct mlx5_pkt_reformat mlx5_packet_reformat_alloc(struct mlx5_core_dev dev,
	- struct mlx5_pkt_reformat_params *params,
	- enum mlx5_flow_namespace_type ns_type)
	-{
	- struct mlx5_pkt_reformat *pkt_reformat;
	- int err;
	-
	- pkt_reformat = kzalloc(sizeof(*pkt_reformat), GFP_KERNEL);
	- if (!pkt_reformat)
	- return ERR_PTR(-ENOMEM);
	-
	- pkt_reformat->ns_type = ns_type;
	- pkt_reformat->reformat_type = params->type;
	- err = mlx5_cmd_packet_reformat_alloc(dev, params, ns_type,
	- pkt_reformat);
	- if (err) {
	- kfree(pkt_reformat);
	- return ERR_PTR(err);
	- }
	-
	- return pkt_reformat;
	-}
	-EXPORT_SYMBOL(mlx5_packet_reformat_alloc);
	-
	-void mlx5_packet_reformat_dealloc(struct mlx5_core_dev *dev,
	- struct mlx5_pkt_reformat *pkt_reformat)
	-{
	- mlx5_cmd_packet_reformat_dealloc(dev, pkt_reformat);
	- kfree(pkt_reformat);
	-}
	-EXPORT_SYMBOL(mlx5_packet_reformat_dealloc);
	-
	diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fw.c b/sys/dev/mlx5/mlx5_core/mlx5_fw.c
	--- a/sys/dev/mlx5/mlx5_core/mlx5_fw.c
	+++ b/sys/dev/mlx5/mlx5_core/mlx5_fw.c
	@@ -241,6 +241,12 @@
	return err;
	}

	+ if (MLX5_CAP_GEN(dev, ipsec_offload)) {
	+ err = mlx5_core_get_caps(dev, MLX5_CAP_IPSEC);
	+ if (err)
	+ return err;
	+ }
	+
	err = mlx5_core_query_special_contexts(dev);
	if (err)
	return err;
	diff --git a/sys/dev/mlx5/mlx5_core/mlx5_main.c b/sys/dev/mlx5/mlx5_core/mlx5_main.c
	--- a/sys/dev/mlx5/mlx5_core/mlx5_main.c
	+++ b/sys/dev/mlx5/mlx5_core/mlx5_main.c
	@@ -28,6 +28,7 @@

	#include "opt_rss.h"
	#include "opt_ratelimit.h"
	+#include "opt_ipsec.h"

	#include <linux/kmod.h>
	#include <linux/module.h>
	@@ -64,6 +65,9 @@
	MODULE_DEPEND(mlx5, linuxkpi, 1, 1, 1);
	MODULE_DEPEND(mlx5, mlxfw, 1, 1, 1);
	MODULE_DEPEND(mlx5, firmware, 1, 1, 1);
	+#ifdef IPSEC_ACCEL
	+MODULE_DEPEND(mlx5, ipsec, 1, 1, 1);
	+#endif
	MODULE_VERSION(mlx5, 1);

	SYSCTL_NODE(_hw, OID_AUTO, mlx5, CTLFLAG_RW \| CTLFLAG_MPSAFE, 0,
	@@ -1203,7 +1207,7 @@
	goto err_stop_eqs;
	}

	- err = mlx5_init_fs(dev);
	+ err = mlx5_fs_core_init(dev);
	if (err) {
	mlx5_core_err(dev, "flow steering init %d\n", err);
	goto err_free_comp_eqs;
	@@ -1317,7 +1321,7 @@
	mlx5_eswitch_cleanup(dev->priv.eswitch);
	mlx5_diag_cnt_cleanup(dev);
	mlx5_mpfs_destroy(dev);
	- mlx5_cleanup_fs(dev);
	+ mlx5_fs_core_cleanup(dev);
	mlx5_wait_for_reclaim_vfs_pages(dev);
	free_comp_eqs(dev);
	mlx5_stop_eqs(dev);
	@@ -1680,10 +1684,17 @@

	mlx5_pagealloc_init(dev);

	+ pr_info("%s - MARK BLOCH WAS HERE\n", __func__);
	+ err = mlx5_fs_core_alloc(dev);
	+ if (err) {
	+ mlx5_core_err(dev, "Failed to alloc flow steering\n");
	+ goto clean_health;
	+ }
	+
	err = mlx5_load_one(dev, priv, true);
	if (err) {
	mlx5_core_err(dev, "mlx5_load_one failed %d\n", err);
	- goto clean_health;
	+ goto clean_fs;
	}

	mlx5_fwdump_prep(dev);
	@@ -1727,6 +1738,8 @@
	pci_save_state(pdev);
	return 0;

	+clean_fs:
	+ mlx5_fs_core_free(dev);
	clean_health:
	mlx5_pagealloc_cleanup(dev);
	mlx5_health_cleanup(dev);
	@@ -1758,6 +1771,7 @@
	(long long)(dev->priv.fw_pages * MLX5_ADAPTER_PAGE_SIZE));
	}

	+ mlx5_fs_core_free(dev);
	mlx5_pagealloc_cleanup(dev);
	mlx5_health_cleanup(dev);
	mlx5_fwdump_clean(dev);
	diff --git a/sys/dev/mlx5/mlx5_core/mlx5_tls.c b/sys/dev/mlx5/mlx5_core/mlx5_tls.c
	--- a/sys/dev/mlx5/mlx5_core/mlx5_tls.c
	+++ b/sys/dev/mlx5/mlx5_core/mlx5_tls.c
	@@ -34,66 +34,6 @@
	#include <dev/mlx5/mlx5_core/mlx5_core.h>
	#include <dev/mlx5/mlx5_core/transobj.h>

	-int mlx5_encryption_key_create(struct mlx5_core_dev *mdev, u32 pdn,
	- const void p_key, u32 key_len, u32 p_obj_id)
	-{
	- u32 in[MLX5_ST_SZ_DW(create_encryption_key_in)] = {};
	- u32 out[MLX5_ST_SZ_DW(create_encryption_key_out)] = {};
	- u64 general_obj_types;
	- int err;
	-
	- general_obj_types = MLX5_CAP_GEN_64(mdev, general_obj_types);
	- if (!(general_obj_types & MLX5_HCA_CAP_GENERAL_OBJ_TYPES_ENCRYPTION_KEY))
	- return -EINVAL;
	-
	- switch (key_len) {
	- case 128 / 8:
	- memcpy(MLX5_ADDR_OF(create_encryption_key_in, in,
	- encryption_key_object.key[4]), p_key, 128 / 8);
	- MLX5_SET(create_encryption_key_in, in, encryption_key_object.pd, pdn);
	- MLX5_SET(create_encryption_key_in, in, encryption_key_object.key_size,
	- MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_KEY_SIZE_128);
	- MLX5_SET(create_encryption_key_in, in, encryption_key_object.key_type,
	- MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_DEK);
	- break;
	- case 256 / 8:
	- memcpy(MLX5_ADDR_OF(create_encryption_key_in, in,
	- encryption_key_object.key[0]), p_key, 256 / 8);
	- MLX5_SET(create_encryption_key_in, in, encryption_key_object.pd, pdn);
	- MLX5_SET(create_encryption_key_in, in, encryption_key_object.key_size,
	- MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_KEY_SIZE_256);
	- MLX5_SET(create_encryption_key_in, in, encryption_key_object.key_type,
	- MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_DEK);
	- break;
	- default:
	- return -EINVAL;
	- }
	-
	- MLX5_SET(create_encryption_key_in, in, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJ);
	- MLX5_SET(create_encryption_key_in, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY);
	-
	- err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
	- if (err == 0)
	- *p_obj_id = MLX5_GET(create_encryption_key_out, out, obj_id);
	-
	- /* avoid leaking key on the stack */
	- memset(in, 0, sizeof(in));
	-
	- return err;
	-}
	-
	-int mlx5_encryption_key_destroy(struct mlx5_core_dev *mdev, u32 oid)
	-{
	- u32 in[MLX5_ST_SZ_DW(destroy_encryption_key_in)] = {};
	- u32 out[MLX5_ST_SZ_DW(destroy_encryption_key_out)] = {};
	-
	- MLX5_SET(destroy_encryption_key_in, in, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJ);
	- MLX5_SET(destroy_encryption_key_in, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY);
	- MLX5_SET(destroy_encryption_key_in, in, obj_id, oid);
	-
	- return mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
	-}
	-
	int mlx5_tls_open_tis(struct mlx5_core_dev mdev, int tc, int tdn, int pdn, u32 p_tisn)
	{
	u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {};
	diff --git a/sys/dev/mlx5/mlx5_core/wq.h b/sys/dev/mlx5/mlx5_core/wq.h
	--- a/sys/dev/mlx5/mlx5_core/wq.h
	+++ b/sys/dev/mlx5/mlx5_core/wq.h
	@@ -29,6 +29,7 @@
	#define __MLX5_WQ_H__

	#include <dev/mlx5/mlx5_ifc.h>
	+#include <dev/mlx5/cq.h>

	struct mlx5_wq_param {
	int linear;
	@@ -138,6 +139,22 @@
	*wq->db = cpu_to_be32(wq->cc & 0xffffff);
	}

	+static inline struct mlx5_cqe64 mlx5_cqwq_get_cqe(struct mlx5_cqwq wq)
	+{
	+ u32 ci = mlx5_cqwq_get_ci(wq);
	+ struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(wq, ci);
	+ u8 cqe_ownership_bit = cqe->op_own & MLX5_CQE_OWNER_MASK;
	+ u8 sw_ownership_val = mlx5_cqwq_get_wrap_cnt(wq) & 1;
	+
	+ if (cqe_ownership_bit != sw_ownership_val)
	+ return NULL;
	+
	+ /* ensure cqe content is read after cqe ownership bit */
	+ atomic_thread_fence_acq();
	+
	+ return cqe;
	+}
	+
	static inline int mlx5_wq_ll_is_full(struct mlx5_wq_ll *wq)
	{
	return wq->cur_sz == wq->sz_m1;
	diff --git a/sys/dev/mlx5/mlx5_en/en.h b/sys/dev/mlx5/mlx5_en/en.h
	--- a/sys/dev/mlx5/mlx5_en/en.h
	+++ b/sys/dev/mlx5/mlx5_en/en.h
	@@ -72,6 +72,7 @@
	#include <dev/mlx5/mlx5_core/wq.h>
	#include <dev/mlx5/mlx5_core/transobj.h>
	#include <dev/mlx5/mlx5_core/mlx5_core.h>
	+#include <dev/mlx5/mlx5_accel/ipsec.h>

	#define MLX5_SET_CFG(p, f, v) MLX5_SET(create_flow_group_in, p, f, v)

	@@ -963,7 +964,7 @@
	struct mlx5e_eth_addr_info {
	u8 addr [ETH_ALEN + 2];
	/* flow table rule per traffic type */
	- struct mlx5_flow_rule *ft_rule[MLX5E_NUM_TT];
	+ struct mlx5_flow_handle *ft_rule[MLX5E_NUM_TT];
	};

	#define MLX5E_ETH_ADDR_HASH_SIZE (1 << BITS_PER_BYTE)
	@@ -999,10 +1000,10 @@

	struct mlx5e_vlan_db {
	unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)];
	- struct mlx5_flow_rule *active_vlans_ft_rule[VLAN_N_VID];
	- struct mlx5_flow_rule *untagged_ft_rule;
	- struct mlx5_flow_rule *any_cvlan_ft_rule;
	- struct mlx5_flow_rule *any_svlan_ft_rule;
	+ struct mlx5_flow_handle *active_vlans_ft_rule[VLAN_N_VID];
	+ struct mlx5_flow_handle *untagged_ft_rule;
	+ struct mlx5_flow_handle *any_cvlan_ft_rule;
	+ struct mlx5_flow_handle *any_svlan_ft_rule;
	bool filter_disabled;
	};

	@@ -1011,7 +1012,7 @@
	u_int proto;
	u_int port;
	bool installed;
	- struct mlx5_flow_rule *vxlan_ft_rule;
	+ struct mlx5_flow_handle *vxlan_ft_rule;
	TAILQ_ENTRY(mlx5e_vxlan_db_el) link;
	};

	@@ -1034,19 +1035,20 @@
	struct mlx5e_accel_fs_tcp {
	struct mlx5_flow_namespace *ns;
	struct mlx5e_flow_table tables[MLX5E_ACCEL_FS_TCP_NUM_TYPES];
	- struct mlx5_flow_rule *default_rules[MLX5E_ACCEL_FS_TCP_NUM_TYPES];
	+ struct mlx5_flow_handle *default_rules[MLX5E_ACCEL_FS_TCP_NUM_TYPES];
	};

	struct mlx5e_flow_tables {
	struct mlx5_flow_namespace *ns;
	struct mlx5e_flow_table vlan;
	struct mlx5e_flow_table vxlan;
	- struct mlx5_flow_rule *vxlan_catchall_ft_rule;
	+ struct mlx5_flow_handle *vxlan_catchall_ft_rule;
	struct mlx5e_flow_table main;
	struct mlx5e_flow_table main_vxlan;
	- struct mlx5_flow_rule *main_vxlan_rule[MLX5E_NUM_TT];
	+ struct mlx5_flow_handle *main_vxlan_rule[MLX5E_NUM_TT];
	struct mlx5e_flow_table inner_rss;
	struct mlx5e_accel_fs_tcp accel_tcp;
	+ struct mlx5_flow_table *ipsec_ft;
	};

	struct mlx5e_xmit_args {
	@@ -1074,6 +1076,7 @@
	u32 xoff;
	};

	+struct mlx5e_ipsec;
	struct mlx5e_priv {
	struct mlx5_core_dev mdev; / must be first */

	@@ -1152,6 +1155,7 @@
	bool sw_is_port_buf_owner;

	struct pfil_head *pfil;
	+ struct mlx5e_ipsec *ipsec;
	struct mlx5e_channel channel[];
	};

	diff --git a/sys/dev/mlx5/mlx5_en/en_hw_tls_rx.h b/sys/dev/mlx5/mlx5_en/en_hw_tls_rx.h
	--- a/sys/dev/mlx5/mlx5_en/en_hw_tls_rx.h
	+++ b/sys/dev/mlx5/mlx5_en/en_hw_tls_rx.h
	@@ -63,7 +63,7 @@
	uint32_t tirn; /* HW TIR context number */
	uint32_t dek_index; /* HW TLS context number */
	struct mlx5e_tls_rx tls_rx; / parent pointer */
	- struct mlx5_flow_rule *flow_rule;
	+ struct mlx5_flow_handle *flow_rule;
	struct mtx mtx;
	struct completion progress_complete;
	uint32_t state; /* see MLX5E_TLS_RX_ST_XXX */
	diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_flow_table.c b/sys/dev/mlx5/mlx5_en/mlx5_en_flow_table.c
	--- a/sys/dev/mlx5/mlx5_en/mlx5_en_flow_table.c
	+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_flow_table.c
	@@ -145,17 +145,17 @@
	mlx5e_del_eth_addr_from_flow_table(struct mlx5e_priv *priv,
	struct mlx5e_eth_addr_info *ai)
	{
	- mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV6_IPSEC_ESP]);
	- mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV4_IPSEC_ESP]);
	- mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV6_IPSEC_AH]);
	- mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV4_IPSEC_AH]);
	- mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV6_TCP]);
	- mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV4_TCP]);
	- mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV6_UDP]);
	- mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV4_UDP]);
	- mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV6]);
	- mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV4]);
	- mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_ANY]);
	+ mlx5_del_flow_rules(&ai->ft_rule[MLX5E_TT_IPV6_IPSEC_ESP]);
	+ mlx5_del_flow_rules(&ai->ft_rule[MLX5E_TT_IPV4_IPSEC_ESP]);
	+ mlx5_del_flow_rules(&ai->ft_rule[MLX5E_TT_IPV6_IPSEC_AH]);
	+ mlx5_del_flow_rules(&ai->ft_rule[MLX5E_TT_IPV4_IPSEC_AH]);
	+ mlx5_del_flow_rules(&ai->ft_rule[MLX5E_TT_IPV6_TCP]);
	+ mlx5_del_flow_rules(&ai->ft_rule[MLX5E_TT_IPV4_TCP]);
	+ mlx5_del_flow_rules(&ai->ft_rule[MLX5E_TT_IPV6_UDP]);
	+ mlx5_del_flow_rules(&ai->ft_rule[MLX5E_TT_IPV4_UDP]);
	+ mlx5_del_flow_rules(&ai->ft_rule[MLX5E_TT_IPV6]);
	+ mlx5_del_flow_rules(&ai->ft_rule[MLX5E_TT_IPV4]);
	+ mlx5_del_flow_rules(&ai->ft_rule[MLX5E_TT_ANY]);
	}

	static int
	@@ -250,24 +250,30 @@
	static int
	mlx5e_add_eth_addr_rule_sub(struct mlx5e_priv *priv,
	struct mlx5e_eth_addr_info *ai, int type,
	- u32 mc, u32 mv)
	+ struct mlx5_flow_spec *spec)
	{
	struct mlx5_flow_destination dest = {};
	u8 mc_enable = 0;
	- struct mlx5_flow_rule **rule_p;
	+ struct mlx5_flow_handle **rule_p;
	struct mlx5_flow_table *ft = priv->fts.main.t;
	- u8 *mc_dmac = MLX5_ADDR_OF(fte_match_param, mc,
	+ u8 *mc_dmac = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
	outer_headers.dmac_47_16);
	- u8 *mv_dmac = MLX5_ADDR_OF(fte_match_param, mv,
	+ u8 *mv_dmac = MLX5_ADDR_OF(fte_match_param, spec->match_value,
	outer_headers.dmac_47_16);
	u32 *tirn = priv->tirn;
	u32 tt_vec;
	int err = 0;
	struct mlx5_flow_act flow_act = {
	- .actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG,
	- .flow_tag = MLX5_FS_ETH_FLOW_TAG,
	+ .action = MLX5_FLOW_RULE_FWD_ACTION_DEST,
	};
	+ u8 *mc;
	+ u8 *mv;

	+ mv = (u8 *)spec->match_value;
	+ mc = (u8 *)spec->match_criteria;
	+
	+ spec->flow_context.flow_tag = MLX5_FS_ETH_FLOW_TAG;
	+ spec->flow_context.flags = FLOW_CONTEXT_HAS_TAG;
	dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;

	switch (type) {
	@@ -291,12 +297,11 @@

	tt_vec = mlx5e_get_tt_vec(ai, type);

	+ spec->match_criteria_enable = mc_enable;
	if (tt_vec & BIT(MLX5E_TT_ANY)) {
	rule_p = &ai->ft_rule[MLX5E_TT_ANY];
	dest.tir_num = tirn[MLX5E_TT_ANY];
	- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
	- MLX5_FLOW_RULE_FWD_ACTION_DEST,
	- &flow_act, &dest);
	+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
	if (IS_ERR_OR_NULL(*rule_p))
	goto err_del_ai;
	}
	@@ -304,14 +309,13 @@
	mc_enable = MLX5_MATCH_OUTER_HEADERS;
	MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);

	+ spec->match_criteria_enable = mc_enable;
	if (tt_vec & BIT(MLX5E_TT_IPV4)) {
	rule_p = &ai->ft_rule[MLX5E_TT_IPV4];
	dest.tir_num = tirn[MLX5E_TT_IPV4];
	MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
	ETHERTYPE_IP);
	- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
	- MLX5_FLOW_RULE_FWD_ACTION_DEST,
	- &flow_act, &dest);
	+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
	if (IS_ERR_OR_NULL(*rule_p))
	goto err_del_ai;
	}
	@@ -321,9 +325,7 @@
	dest.tir_num = tirn[MLX5E_TT_IPV6];
	MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
	ETHERTYPE_IPV6);
	- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
	- MLX5_FLOW_RULE_FWD_ACTION_DEST,
	- &flow_act, &dest);
	+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
	if (IS_ERR_OR_NULL(*rule_p))
	goto err_del_ai;
	}
	@@ -336,9 +338,7 @@
	dest.tir_num = tirn[MLX5E_TT_IPV4_UDP];
	MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
	ETHERTYPE_IP);
	- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
	- MLX5_FLOW_RULE_FWD_ACTION_DEST,
	- &flow_act, &dest);
	+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
	if (IS_ERR_OR_NULL(*rule_p))
	goto err_del_ai;
	}
	@@ -348,9 +348,7 @@
	dest.tir_num = tirn[MLX5E_TT_IPV6_UDP];
	MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
	ETHERTYPE_IPV6);
	- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
	- MLX5_FLOW_RULE_FWD_ACTION_DEST,
	- &flow_act, &dest);
	+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
	if (IS_ERR_OR_NULL(*rule_p))
	goto err_del_ai;
	}
	@@ -362,9 +360,7 @@
	dest.tir_num = tirn[MLX5E_TT_IPV4_TCP];
	MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
	ETHERTYPE_IP);
	- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
	- MLX5_FLOW_RULE_FWD_ACTION_DEST,
	- &flow_act, &dest);
	+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
	if (IS_ERR_OR_NULL(*rule_p))
	goto err_del_ai;
	}
	@@ -374,9 +370,7 @@
	dest.tir_num = tirn[MLX5E_TT_IPV6_TCP];
	MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
	ETHERTYPE_IPV6);
	- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
	- MLX5_FLOW_RULE_FWD_ACTION_DEST,
	- &flow_act, &dest);
	+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
	if (IS_ERR_OR_NULL(*rule_p))
	goto err_del_ai;
	}
	@@ -388,9 +382,7 @@
	dest.tir_num = tirn[MLX5E_TT_IPV4_IPSEC_AH];
	MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
	ETHERTYPE_IP);
	- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
	- MLX5_FLOW_RULE_FWD_ACTION_DEST,
	- &flow_act, &dest);
	+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
	if (IS_ERR_OR_NULL(*rule_p))
	goto err_del_ai;
	}
	@@ -400,9 +392,7 @@
	dest.tir_num = tirn[MLX5E_TT_IPV6_IPSEC_AH];
	MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
	ETHERTYPE_IPV6);
	- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
	- MLX5_FLOW_RULE_FWD_ACTION_DEST,
	- &flow_act, &dest);
	+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
	if (IS_ERR_OR_NULL(*rule_p))
	goto err_del_ai;
	}
	@@ -414,9 +404,7 @@
	dest.tir_num = tirn[MLX5E_TT_IPV4_IPSEC_ESP];
	MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
	ETHERTYPE_IP);
	- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
	- MLX5_FLOW_RULE_FWD_ACTION_DEST,
	- &flow_act, &dest);
	+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
	if (IS_ERR_OR_NULL(*rule_p))
	goto err_del_ai;
	}
	@@ -426,9 +414,7 @@
	dest.tir_num = tirn[MLX5E_TT_IPV6_IPSEC_ESP];
	MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
	ETHERTYPE_IPV6);
	- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
	- MLX5_FLOW_RULE_FWD_ACTION_DEST,
	- &flow_act, &dest);
	+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
	if (IS_ERR_OR_NULL(*rule_p))
	goto err_del_ai;
	}
	@@ -447,23 +433,19 @@
	mlx5e_add_eth_addr_rule(struct mlx5e_priv *priv,
	struct mlx5e_eth_addr_info *ai, int type)
	{
	- u32 *match_criteria;
	- u32 *match_value;
	+ struct mlx5_flow_spec *spec;
	int err = 0;

	- match_value = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
	- match_criteria = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
	- if (!match_value \|\| !match_criteria) {
	+ spec = mlx5_vzalloc(sizeof(*spec));
	+ if (!spec) {
	mlx5_en_err(priv->ifp, "alloc failed\n");
	err = -ENOMEM;
	goto add_eth_addr_rule_out;
	}
	- err = mlx5e_add_eth_addr_rule_sub(priv, ai, type, match_criteria,
	- match_value);
	+ err = mlx5e_add_eth_addr_rule_sub(priv, ai, type, spec);

	add_eth_addr_rule_out:
	- kvfree(match_criteria);
	- kvfree(match_value);
	+ kvfree(spec);

	return (err);
	}
	@@ -471,51 +453,56 @@
	static void
	mlx5e_del_main_vxlan_rules(struct mlx5e_priv *priv)
	{
	- mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_IPSEC_ESP]);
	- mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_IPSEC_ESP]);
	- mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_IPSEC_AH]);
	- mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_IPSEC_AH]);
	- mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_TCP]);
	- mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_TCP]);
	- mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_UDP]);
	- mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_UDP]);
	- mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6]);
	- mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4]);
	- mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_ANY]);
	+ mlx5_del_flow_rules(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_IPSEC_ESP]);
	+ mlx5_del_flow_rules(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_IPSEC_ESP]);
	+ mlx5_del_flow_rules(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_IPSEC_AH]);
	+ mlx5_del_flow_rules(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_IPSEC_AH]);
	+ mlx5_del_flow_rules(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_TCP]);
	+ mlx5_del_flow_rules(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_TCP]);
	+ mlx5_del_flow_rules(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_UDP]);
	+ mlx5_del_flow_rules(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_UDP]);
	+ mlx5_del_flow_rules(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6]);
	+ mlx5_del_flow_rules(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4]);
	+ mlx5_del_flow_rules(&priv->fts.main_vxlan_rule[MLX5E_TT_ANY]);
	}

	static int
	-mlx5e_add_main_vxlan_rules_sub(struct mlx5e_priv priv, u32 mc, u32 *mv)
	+mlx5e_add_main_vxlan_rules_sub(struct mlx5e_priv *priv,
	+ struct mlx5_flow_spec *spec)
	{
	struct mlx5_flow_destination dest = {};
	- u8 mc_enable = 0;
	- struct mlx5_flow_rule **rule_p;
	+ struct mlx5_flow_handle **rule_p;
	struct mlx5_flow_table *ft = priv->fts.main_vxlan.t;
	u32 *tirn = priv->tirn_inner_vxlan;
	struct mlx5_flow_act flow_act = {
	- .actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG,
	- .flow_tag = MLX5_FS_ETH_FLOW_TAG,
	+ .action = MLX5_FLOW_RULE_FWD_ACTION_DEST,
	};
	int err = 0;
	+ u8 *mc;
	+ u8 *mv;
	+
	+ spec->flow_context.flags = FLOW_CONTEXT_HAS_TAG;
	+ spec->flow_context.flow_tag = MLX5_FS_ETH_FLOW_TAG;
	+
	+ mc = (u8 *)spec->match_criteria;
	+ mv = (u8 *)spec->match_value;

	dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;

	- mc_enable = MLX5_MATCH_INNER_HEADERS;
	+ spec->match_criteria_enable = MLX5_MATCH_INNER_HEADERS;
	MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.ethertype);

	rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV4];
	dest.tir_num = tirn[MLX5E_TT_IPV4];
	MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IP);
	- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
	- MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
	+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
	if (IS_ERR_OR_NULL(*rule_p))
	goto err_del_ai;

	rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV6];
	dest.tir_num = tirn[MLX5E_TT_IPV6];
	MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IPV6);
	- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
	- MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
	+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
	if (IS_ERR_OR_NULL(*rule_p))
	goto err_del_ai;

	@@ -525,16 +512,14 @@
	rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_UDP];
	dest.tir_num = tirn[MLX5E_TT_IPV4_UDP];
	MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IP);
	- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
	- MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
	+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
	if (IS_ERR_OR_NULL(*rule_p))
	goto err_del_ai;

	rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_UDP];
	dest.tir_num = tirn[MLX5E_TT_IPV6_UDP];
	MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IPV6);
	- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
	- MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
	+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
	if (IS_ERR_OR_NULL(*rule_p))
	goto err_del_ai;

	@@ -543,16 +528,14 @@
	rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_TCP];
	dest.tir_num = tirn[MLX5E_TT_IPV4_TCP];
	MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IP);
	- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
	- MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
	+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
	if (IS_ERR_OR_NULL(*rule_p))
	goto err_del_ai;

	rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_TCP];
	dest.tir_num = tirn[MLX5E_TT_IPV6_TCP];
	MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IPV6);
	- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
	- MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
	+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
	if (IS_ERR_OR_NULL(*rule_p))
	goto err_del_ai;

	@@ -561,16 +544,14 @@
	rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_IPSEC_AH];
	dest.tir_num = tirn[MLX5E_TT_IPV4_IPSEC_AH];
	MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IP);
	- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
	- MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
	+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
	if (IS_ERR_OR_NULL(*rule_p))
	goto err_del_ai;

	rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_IPSEC_AH];
	dest.tir_num = tirn[MLX5E_TT_IPV6_IPSEC_AH];
	MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IPV6);
	- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
	- MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
	+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
	if (IS_ERR_OR_NULL(*rule_p))
	goto err_del_ai;

	@@ -579,8 +560,7 @@
	rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_IPSEC_ESP];
	dest.tir_num = tirn[MLX5E_TT_IPV4_IPSEC_ESP];
	MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IP);
	- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
	- MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
	+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
	if (IS_ERR_OR_NULL(*rule_p))
	goto err_del_ai;

	@@ -588,18 +568,16 @@
	dest.tir_num = tirn[MLX5E_TT_IPV6_IPSEC_ESP];
	MLX5_SET(fte_match_param, mv, inner_headers.ethertype,
	ETHERTYPE_IPV6);
	- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
	- MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
	+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
	if (IS_ERR_OR_NULL(*rule_p))
	goto err_del_ai;

	- mc_enable = 0;
	+ spec->match_criteria_enable = 0;
	memset(mv, 0, MLX5_ST_SZ_BYTES(fte_match_param));
	memset(mc, 0, MLX5_ST_SZ_BYTES(fte_match_param));
	rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_ANY];
	dest.tir_num = tirn[MLX5E_TT_ANY];
	- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
	- MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
	+ *rule_p = mlx5_add_flow_rules(ft, NULL, &flow_act, &dest, 1);
	if (IS_ERR_OR_NULL(*rule_p))
	goto err_del_ai;

	@@ -616,22 +594,19 @@
	static int
	mlx5e_add_main_vxlan_rules(struct mlx5e_priv *priv)
	{
	- u32 *match_criteria;
	- u32 *match_value;
	+ struct mlx5_flow_spec *spec;
	int err = 0;

	- match_value = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
	- match_criteria = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
	- if (match_value == NULL \|\| match_criteria == NULL) {
	+ spec = mlx5_vzalloc(sizeof(*spec));
	+ if (!spec) {
	mlx5_en_err(priv->ifp, "alloc failed\n");
	err = -ENOMEM;
	goto add_main_vxlan_rules_out;
	}
	- err = mlx5e_add_main_vxlan_rules_sub(priv, match_criteria, match_value);
	+ err = mlx5e_add_main_vxlan_rules_sub(priv, spec);

	add_main_vxlan_rules_out:
	- kvfree(match_criteria);
	- kvfree(match_value);
	+ kvfree(spec);

	return (err);
	}
	@@ -689,22 +664,27 @@
	static int
	mlx5e_add_vlan_rule_sub(struct mlx5e_priv *priv,
	enum mlx5e_vlan_rule_type rule_type, u16 vid,
	- u32 mc, u32 mv)
	+ struct mlx5_flow_spec *spec)
	{
	struct mlx5_flow_table *ft = priv->fts.vlan.t;
	struct mlx5_flow_destination dest = {};
	- u8 mc_enable = 0;
	- struct mlx5_flow_rule **rule_p;
	+ struct mlx5_flow_handle **rule_p;
	int err = 0;
	struct mlx5_flow_act flow_act = {
	- .actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG,
	- .flow_tag = MLX5_FS_ETH_FLOW_TAG,
	+ .action = MLX5_FLOW_RULE_FWD_ACTION_DEST,
	};
	+ u8 *mv;
	+ u8 *mc;

	+ mv = (u8 *)spec->match_value;
	+ mc = (u8 *)spec->match_criteria;
	+
	+ spec->flow_context.flow_tag = MLX5_FS_ETH_FLOW_TAG;
	+ spec->flow_context.flags = FLOW_CONTEXT_HAS_TAG;
	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
	dest.ft = priv->fts.vxlan.t;

	- mc_enable = MLX5_MATCH_OUTER_HEADERS;
	+ spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;

	switch (rule_type) {
	case MLX5E_VLAN_RULE_TYPE_UNTAGGED:
	@@ -731,11 +711,7 @@
	break;
	}

	- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
	- MLX5_FLOW_RULE_FWD_ACTION_DEST,
	- &flow_act,
	- &dest);
	-
	+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
	if (IS_ERR(*rule_p)) {
	err = PTR_ERR(*rule_p);
	*rule_p = NULL;
	@@ -749,24 +725,20 @@
	mlx5e_add_vlan_rule(struct mlx5e_priv *priv,
	enum mlx5e_vlan_rule_type rule_type, u16 vid)
	{
	- u32 *match_criteria;
	- u32 *match_value;
	+ struct mlx5_flow_spec *spec;
	int err = 0;

	- match_value = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
	- match_criteria = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
	- if (!match_value \|\| !match_criteria) {
	+ spec = mlx5_vzalloc(sizeof(*spec));
	+ if (!spec) {
	mlx5_en_err(priv->ifp, "alloc failed\n");
	err = -ENOMEM;
	goto add_vlan_rule_out;
	}

	- err = mlx5e_add_vlan_rule_sub(priv, rule_type, vid, match_criteria,
	- match_value);
	+ err = mlx5e_add_vlan_rule_sub(priv, rule_type, vid, spec);

	add_vlan_rule_out:
	- kvfree(match_criteria);
	- kvfree(match_value);
	+ kvfree(spec);

	return (err);
	}
	@@ -777,16 +749,16 @@
	{
	switch (rule_type) {
	case MLX5E_VLAN_RULE_TYPE_UNTAGGED:
	- mlx5_del_flow_rule(&priv->vlan.untagged_ft_rule);
	+ mlx5_del_flow_rules(&priv->vlan.untagged_ft_rule);
	break;
	case MLX5E_VLAN_RULE_TYPE_ANY_CTAG_VID:
	- mlx5_del_flow_rule(&priv->vlan.any_cvlan_ft_rule);
	+ mlx5_del_flow_rules(&priv->vlan.any_cvlan_ft_rule);
	break;
	case MLX5E_VLAN_RULE_TYPE_ANY_STAG_VID:
	- mlx5_del_flow_rule(&priv->vlan.any_svlan_ft_rule);
	+ mlx5_del_flow_rules(&priv->vlan.any_svlan_ft_rule);
	break;
	case MLX5E_VLAN_RULE_TYPE_MATCH_VID:
	- mlx5_del_flow_rule(&priv->vlan.active_vlans_ft_rule[vid]);
	+ mlx5_del_flow_rules(&priv->vlan.active_vlans_ft_rule[vid]);
	mlx5e_vport_context_update_vlans(priv);
	break;
	default:
	@@ -1520,11 +1492,16 @@
	{
	struct mlx5e_flow_table *ft = inner_vxlan ? &priv->fts.main_vxlan :
	&priv->fts.main;
	+ struct mlx5_flow_table_attr ft_attr = {};
	int err;

	ft->num_groups = 0;
	- ft->t = mlx5_create_flow_table(priv->fts.ns, 0,
	- inner_vxlan ? "vxlan_main" : "main", MLX5E_MAIN_TABLE_SIZE);
	+ ft_attr.max_fte = MLX5E_MAIN_TABLE_SIZE;
	+ if (priv->ipsec)
	+ ft_attr.level = inner_vxlan ? 10 : 12;
	+ else
	+ ft_attr.level = inner_vxlan ? 2 : 4;
	+ ft->t = mlx5_create_flow_table(priv->fts.ns, &ft_attr);

	if (IS_ERR(ft->t)) {
	err = PTR_ERR(ft->t);
	@@ -1645,11 +1622,13 @@
	mlx5e_create_vlan_flow_table(struct mlx5e_priv *priv)
	{
	struct mlx5e_flow_table *ft = &priv->fts.vlan;
	+ struct mlx5_flow_table_attr ft_attr = {};
	int err;

	ft->num_groups = 0;
	- ft->t = mlx5_create_flow_table(priv->fts.ns, 0, "vlan",
	- MLX5E_VLAN_TABLE_SIZE);
	+ ft_attr.max_fte = MLX5E_VLAN_TABLE_SIZE;
	+ ft_attr.level = (priv->ipsec) ? 8 : 0;
	+ ft->t = mlx5_create_flow_table(priv->fts.ns, &ft_attr);

	if (IS_ERR(ft->t)) {
	err = PTR_ERR(ft->t);
	@@ -1685,23 +1664,29 @@
	}

	static int
	-mlx5e_add_vxlan_rule_sub(struct mlx5e_priv priv, u32 mc, u32 *mv,
	+mlx5e_add_vxlan_rule_sub(struct mlx5e_priv priv, struct mlx5_flow_spec spec,
	struct mlx5e_vxlan_db_el *el)
	{
	struct mlx5_flow_table *ft = priv->fts.vxlan.t;
	struct mlx5_flow_destination dest = {};
	- u8 mc_enable;
	- struct mlx5_flow_rule **rule_p;
	+ struct mlx5_flow_handle **rule_p;
	int err = 0;
	struct mlx5_flow_act flow_act = {
	- .actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG,
	- .flow_tag = MLX5_FS_ETH_FLOW_TAG,
	+ .action = MLX5_FLOW_RULE_FWD_ACTION_DEST,
	};
	+ u8 *mc;
	+ u8 *mv;
	+
	+ mv = (u8 *)spec->match_value;
	+ mc = (u8 *)spec->match_criteria;
	+
	+ spec->flow_context.flow_tag = MLX5_FS_ETH_FLOW_TAG;
	+ spec->flow_context.flags = FLOW_CONTEXT_HAS_TAG;

	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
	dest.ft = priv->fts.main_vxlan.t;

	- mc_enable = MLX5_MATCH_OUTER_HEADERS;
	+ spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
	rule_p = &el->vxlan_ft_rule;
	MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);
	MLX5_SET(fte_match_param, mv, outer_headers.ethertype, el->proto);
	@@ -1710,8 +1695,7 @@
	MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.udp_dport);
	MLX5_SET(fte_match_param, mv, outer_headers.udp_dport, el->port);

	- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
	- MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
	+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);

	if (IS_ERR(*rule_p)) {
	err = PTR_ERR(*rule_p);
	@@ -1766,23 +1750,20 @@
	mlx5e_add_vxlan_rule_from_db(struct mlx5e_priv *priv,
	struct mlx5e_vxlan_db_el *el)
	{
	- u32 *match_criteria;
	- u32 *match_value;
	+ struct mlx5_flow_spec *spec;
	int err;

	- match_value = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
	- match_criteria = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
	- if (match_value == NULL \|\| match_criteria == NULL) {
	+ spec = mlx5_vzalloc(sizeof(*spec));
	+ if (!spec) {
	mlx5_en_err(priv->ifp, "alloc failed\n");
	err = -ENOMEM;
	goto add_vxlan_rule_out;
	}

	- err = mlx5e_add_vxlan_rule_sub(priv, match_criteria, match_value, el);
	+ err = mlx5e_add_vxlan_rule_sub(priv, spec, el);

	add_vxlan_rule_out:
	- kvfree(match_criteria);
	- kvfree(match_value);
	+ kvfree(spec);

	return (err);
	}
	@@ -1820,24 +1801,25 @@
	}

	static int
	-mlx5e_add_vxlan_catchall_rule_sub(struct mlx5e_priv priv, u32 mc, u32 *mv)
	+mlx5e_add_vxlan_catchall_rule_sub(struct mlx5e_priv *priv,
	+ struct mlx5_flow_spec *spec)
	{
	struct mlx5_flow_table *ft = priv->fts.vxlan.t;
	struct mlx5_flow_destination dest = {};
	- u8 mc_enable = 0;
	- struct mlx5_flow_rule **rule_p;
	+ struct mlx5_flow_handle **rule_p;
	int err = 0;
	struct mlx5_flow_act flow_act = {
	- .actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG,
	- .flow_tag = MLX5_FS_ETH_FLOW_TAG,
	+ .action = MLX5_FLOW_RULE_FWD_ACTION_DEST,
	};

	+ spec->flow_context.flow_tag = MLX5_FS_ETH_FLOW_TAG;
	+ spec->flow_context.flags = FLOW_CONTEXT_HAS_TAG;
	+
	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
	dest.ft = priv->fts.main.t;

	rule_p = &priv->fts.vxlan_catchall_ft_rule;
	- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
	- MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
	+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);

	if (IS_ERR(*rule_p)) {
	err = PTR_ERR(*rule_p);
	@@ -1852,24 +1834,20 @@
	static int
	mlx5e_add_vxlan_catchall_rule(struct mlx5e_priv *priv)
	{
	- u32 *match_criteria;
	- u32 *match_value;
	+ struct mlx5_flow_spec *spec;
	int err;

	- match_value = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
	- match_criteria = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
	- if (match_value == NULL \|\| match_criteria == NULL) {
	+ spec = mlx5_vzalloc(sizeof(*spec));
	+ if (!spec) {
	mlx5_en_err(priv->ifp, "alloc failed\n");
	err = -ENOMEM;
	goto add_vxlan_rule_out;
	}

	- err = mlx5e_add_vxlan_catchall_rule_sub(priv, match_criteria,
	- match_value);
	+ err = mlx5e_add_vxlan_catchall_rule_sub(priv, spec);

	add_vxlan_rule_out:
	- kvfree(match_criteria);
	- kvfree(match_value);
	+ kvfree(spec);

	return (err);
	}
	@@ -1913,7 +1891,7 @@
	}

	if (el->installed)
	- mlx5_del_flow_rule(&el->vxlan_ft_rule);
	+ mlx5_del_flow_rules(&el->vxlan_ft_rule);
	TAILQ_REMOVE(&priv->vxlan.head, el, link);
	kvfree(el);
	return (0);
	@@ -1927,7 +1905,7 @@
	TAILQ_FOREACH(el, &priv->vxlan.head, link) {
	if (!el->installed)
	continue;
	- mlx5_del_flow_rule(&el->vxlan_ft_rule);
	+ mlx5_del_flow_rules(&el->vxlan_ft_rule);
	el->installed = false;
	}
	}
	@@ -1935,7 +1913,7 @@
	static void
	mlx5e_del_vxlan_catchall_rule(struct mlx5e_priv *priv)
	{
	- mlx5_del_flow_rule(&priv->fts.vxlan_catchall_ft_rule);
	+ mlx5_del_flow_rules(&priv->fts.vxlan_catchall_ft_rule);
	}

	void
	@@ -2032,11 +2010,13 @@
	mlx5e_create_vxlan_flow_table(struct mlx5e_priv *priv)
	{
	struct mlx5e_flow_table *ft = &priv->fts.vxlan;
	+ struct mlx5_flow_table_attr ft_attr = {};
	int err;

	ft->num_groups = 0;
	- ft->t = mlx5_create_flow_table(priv->fts.ns, 0, "vxlan",
	- MLX5E_VXLAN_TABLE_SIZE);
	+ ft_attr.max_fte = MLX5E_VXLAN_TABLE_SIZE;
	+ ft_attr.level = (priv->ipsec) ? 9 : 1;
	+ ft->t = mlx5_create_flow_table(priv->fts.ns, &ft_attr);

	if (IS_ERR(ft->t)) {
	err = PTR_ERR(ft->t);
	@@ -2146,11 +2126,13 @@
	mlx5e_create_inner_rss_flow_table(struct mlx5e_priv *priv)
	{
	struct mlx5e_flow_table *ft = &priv->fts.inner_rss;
	+ struct mlx5_flow_table_attr ft_attr = {};
	int err;

	ft->num_groups = 0;
	- ft->t = mlx5_create_flow_table(priv->fts.ns, 0, "inner_rss",
	- MLX5E_INNER_RSS_TABLE_SIZE);
	+ ft_attr.max_fte = MLX5E_INNER_RSS_TABLE_SIZE;
	+ ft_attr.level = (priv->ipsec) ? 11 : 3;
	+ ft->t = mlx5_create_flow_table(priv->fts.ns, &ft_attr);

	if (IS_ERR(ft->t)) {
	err = PTR_ERR(ft->t);
	@@ -2200,9 +2182,13 @@
	priv->fts.ns = mlx5_get_flow_namespace(
	priv->mdev, MLX5_FLOW_NAMESPACE_KERNEL);

	+ err = mlx5e_accel_ipsec_fs_rx_tables_create(priv);
	+ if (err)
	+ return err;
	+
	err = mlx5e_create_vlan_flow_table(priv);
	if (err)
	- return (err);
	+ goto err_destroy_ipsec_flow_table;

	err = mlx5e_create_vxlan_flow_table(priv);
	if (err)
	@@ -2224,13 +2210,19 @@
	if (err)
	goto err_destroy_main_flow_table_false;

	+ err = mlx5e_accel_ipsec_fs_rx_catchall_rules(priv);
	+ if (err)
	+ goto err_destroy_vxlan_catchall_rule;
	+
	err = mlx5e_accel_fs_tcp_create(priv);
	if (err)
	- goto err_del_vxlan_catchall_rule;
	+ goto err_destroy_ipsec_catchall_rules;

	return (0);

	-err_del_vxlan_catchall_rule:
	+err_destroy_ipsec_catchall_rules:
	+ mlx5e_accel_ipsec_fs_rx_catchall_rules_destroy(priv);
	+err_destroy_vxlan_catchall_rule:
	mlx5e_del_vxlan_catchall_rule(priv);
	err_destroy_main_flow_table_false:
	mlx5e_destroy_main_flow_table(priv);
	@@ -2242,6 +2234,8 @@
	mlx5e_destroy_vxlan_flow_table(priv);
	err_destroy_vlan_flow_table:
	mlx5e_destroy_vlan_flow_table(priv);
	+err_destroy_ipsec_flow_table:
	+ mlx5e_accel_ipsec_fs_rx_tables_destroy(priv);

	return (err);
	}
	@@ -2250,12 +2244,14 @@
	mlx5e_close_flow_tables(struct mlx5e_priv *priv)
	{
	mlx5e_accel_fs_tcp_destroy(priv);
	+ mlx5e_accel_ipsec_fs_rx_catchall_rules_destroy(priv);
	mlx5e_del_vxlan_catchall_rule(priv);
	mlx5e_destroy_main_flow_table(priv);
	mlx5e_destroy_inner_rss_flow_table(priv);
	mlx5e_destroy_main_vxlan_flow_table(priv);
	mlx5e_destroy_vxlan_flow_table(priv);
	mlx5e_destroy_vlan_flow_table(priv);
	+ mlx5e_accel_ipsec_fs_rx_tables_destroy(priv);
	}

	int
	diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_hw_tls.c b/sys/dev/mlx5/mlx5_en/mlx5_en_hw_tls.c
	--- a/sys/dev/mlx5/mlx5_en/mlx5_en_hw_tls.c
	+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_hw_tls.c
	@@ -33,6 +33,7 @@
	#include <dev/mlx5/mlx5_en/en.h>

	#include <dev/mlx5/tls.h>
	+#include <dev/mlx5/crypto.h>

	#include <linux/delay.h>
	#include <sys/ktls.h>
	@@ -239,6 +240,7 @@

	/* try to allocate a DEK context ID */
	err = mlx5_encryption_key_create(priv->mdev, priv->pdn,
	+ MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_TLS,
	MLX5_ADDR_OF(sw_tls_cntx, ptag->crypto_params, key.key_data),
	MLX5_GET(sw_tls_cntx, ptag->crypto_params, key.key_len),
	&ptag->dek_index);
	diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_hw_tls_rx.c b/sys/dev/mlx5/mlx5_en/mlx5_en_hw_tls_rx.c
	--- a/sys/dev/mlx5/mlx5_en/mlx5_en_hw_tls_rx.c
	+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_hw_tls_rx.c
	@@ -31,6 +31,7 @@

	#include <dev/mlx5/mlx5_en/en.h>

	+#include <dev/mlx5/crypto.h>
	#include <dev/mlx5/tls.h>

	#include <dev/mlx5/fs.h>
	@@ -553,6 +554,7 @@

	/* try to allocate a DEK context ID */
	err = mlx5_encryption_key_create(priv->mdev, priv->pdn,
	+ MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_TLS,
	MLX5_ADDR_OF(sw_tls_rx_cntx, ptag->crypto_params, key.key_data),
	MLX5_GET(sw_tls_rx_cntx, ptag->crypto_params, key.key_len),
	&ptag->dek_index);
	@@ -661,7 +663,7 @@
	struct mlx5e_iq *iq;
	struct mlx5e_priv *priv;
	struct mlx5e_tls_rx_tag *ptag;
	- struct mlx5_flow_rule *flow_rule;
	+ struct mlx5_flow_handle *flow_rule;
	const struct tls_session_params *en;
	uint32_t value;
	int error;
	diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_main.c b/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
	--- a/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
	+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
	@@ -26,6 +26,7 @@
	* $FreeBSD$
	*/

	+#include "opt_ipsec.h"
	#include "opt_kern_tls.h"
	#include "opt_rss.h"
	#include "opt_ratelimit.h"
	@@ -37,6 +38,8 @@
	#include <machine/atomic.h>

	#include <net/debugnet.h>
	+#include <netipsec/keydb.h>
	+#include <netipsec/ipsec_accel.h>

	static int mlx5e_get_wqe_sz(struct mlx5e_priv priv, u32 wqe_sz, u32 *nsegs);
	static if_snd_tag_query_t mlx5e_ul_snd_tag_query;
	@@ -3611,6 +3614,21 @@
	if_togglecapenable2(ifp, IFCAP2_BIT(IFCAP2_RXTLS4));
	if ((mask & IFCAP2_BIT(IFCAP2_RXTLS6)) != 0)
	if_togglecapenable2(ifp, IFCAP2_BIT(IFCAP2_RXTLS6));
	+#ifdef IPSEC_ACCEL
	+ if ((mask & IFCAP2_BIT(IFCAP2_IPSEC_ACCEL)) != 0) {
	+ bool was_enabled = (if_getcapenable2(ifp) &
	+ IFCAP2_BIT(IFCAP2_IPSEC_ACCEL)) != 0;
	+ if_togglecapenable2(ifp, IFCAP2_BIT(IFCAP2_IPSEC_ACCEL));
	+ mlx5e_close_locked(ifp);
	+ if (was_enabled) {
	+ ipsec_accel_on_ifdown(priv->ifp);
	+ mlx5e_ipsec_cleanup(priv);
	+ } else {
	+ mlx5e_ipsec_init(priv);
	+ }
	+ mlx5e_open_locked(ifp);
	+ }
	+#endif
	out:
	PRIV_UNLOCK(priv);
	break;
	@@ -4452,6 +4470,10 @@
	if_setcapabilitiesbit(ifp, IFCAP_VXLAN_HWCSUM \| IFCAP_VXLAN_HWTSO, 0);
	if_setcapabilities2bit(ifp, IFCAP2_BIT(IFCAP2_RXTLS4) \|
	IFCAP2_BIT(IFCAP2_RXTLS6), 0);
	+
	+ if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PACKET_OFFLOAD)
	+ if_setcapabilities2bit(ifp, IFCAP2_BIT(IFCAP2_IPSEC_ACCEL) , 0);
	+
	if_setsndtagallocfn(ifp, mlx5e_snd_tag_alloc);
	#ifdef RATELIMIT
	if_setratelimitqueryfn(ifp, mlx5e_rl_query);
	@@ -4551,10 +4573,18 @@
	goto err_rl_init;
	}

	+ if ((if_getcapenable2(ifp) & IFCAP2_BIT(IFCAP2_IPSEC_ACCEL)) != 0) {
	+ err = mlx5e_ipsec_init(priv);
	+ if (err) {
	+ if_printf(ifp, "%s: mlx5e_tls_init failed\n", __func__);
	+ goto err_tls_init;
	+ }
	+ }
	+
	err = mlx5e_open_drop_rq(priv, &priv->drop_rq);
	if (err) {
	if_printf(ifp, "%s: mlx5e_open_drop_rq failed (%d)\n", __func__, err);
	- goto err_tls_init;
	+ goto err_ipsec_init;
	}

	err = mlx5e_open_rqts(priv);
	@@ -4730,6 +4760,9 @@
	err_open_drop_rq:
	mlx5e_close_drop_rq(&priv->drop_rq);

	+err_ipsec_init:
	+ mlx5e_ipsec_cleanup(priv);
	+
	err_tls_init:
	mlx5e_tls_cleanup(priv);

	@@ -4838,10 +4871,14 @@
	ether_ifdetach(ifp);

	mlx5e_tls_rx_cleanup(priv);
	+#ifdef IPSEC_ACCEL
	+ ipsec_accel_on_ifdown(priv->ifp);
	+#endif
	mlx5e_close_flow_tables(priv);
	mlx5e_close_tirs(priv);
	mlx5e_close_rqts(priv);
	mlx5e_close_drop_rq(&priv->drop_rq);
	+ mlx5e_ipsec_cleanup(priv);
	mlx5e_tls_cleanup(priv);
	mlx5e_rl_cleanup(priv);

	@@ -4956,6 +4993,7 @@
	module_init_order(mlx5e_init, SI_ORDER_SIXTH);
	module_exit_order(mlx5e_cleanup, SI_ORDER_SIXTH);

	+MODULE_DEPEND(mlx5en, ipsec, 1, 1, 1);
	MODULE_DEPEND(mlx5en, linuxkpi, 1, 1, 1);
	MODULE_DEPEND(mlx5en, mlx5, 1, 1, 1);
	MODULE_VERSION(mlx5en, 1);
	diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c b/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
	--- a/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
	+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
	@@ -30,6 +30,7 @@

	#include <dev/mlx5/mlx5_en/en.h>
	#include <machine/in_cksum.h>
	+#include <dev/mlx5/mlx5_accel/ipsec.h>

	static inline int
	mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq,
	@@ -71,6 +72,9 @@
	/* get IP header aligned */
	m_adj(mb, MLX5E_NET_IP_ALIGN);

	+ err = mlx5_accel_ipsec_rx_tag_add(rq->ifp, mb);
	+ if (err)
	+ goto err_free_mbuf;
	err = -bus_dmamap_load_mbuf_sg(rq->dma_tag, rq->mbuf[ix].dma_map,
	mb, segs, &nsegs, BUS_DMA_NOWAIT);
	if (err != 0)
	@@ -419,6 +423,8 @@
	default:
	break;
	}
	+
	+ mlx5e_accel_ipsec_handle_rx(mb, cqe);
	}

	static inline void
	@@ -564,7 +570,9 @@
	("Filter returned %d!\n", rv));
	}
	}
	- if ((MHLEN - MLX5E_NET_IP_ALIGN) >= byte_cnt &&
	+ if (!mlx5e_accel_ipsec_flow(cqe) /* tag is already assigned
	+ to rq->mbuf */ &&
	+ MHLEN - MLX5E_NET_IP_ALIGN >= byte_cnt &&
	(mb = m_gethdr(M_NOWAIT, MT_DATA)) != NULL) {
	/* set maximum mbuf length */
	mb->m_len = MHLEN - MLX5E_NET_IP_ALIGN;
	diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c b/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c
	--- a/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c
	+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c
	@@ -32,6 +32,7 @@

	#include <dev/mlx5/mlx5_en/en.h>
	#include <machine/atomic.h>
	+#include <dev/mlx5/mlx5_accel/ipsec.h>

	static inline bool
	mlx5e_do_send_cqe_inline(struct mlx5e_sq *sq)
	@@ -746,6 +747,8 @@
	/* get pointer to mbuf */
	mb = *mbp;

	+ mlx5e_accel_ipsec_handle_tx(mb, wqe);
	+
	/* Send a copy of the frame to the BPF listener, if any */
	if (ifp != NULL && if_getbpf(ifp) != NULL)
	ETHER_BPF_MTAP(ifp, mb);
	diff --git a/sys/dev/mlx5/mlx5_ib/mlx5_ib.h b/sys/dev/mlx5/mlx5_ib/mlx5_ib.h
	--- a/sys/dev/mlx5/mlx5_ib/mlx5_ib.h
	+++ b/sys/dev/mlx5/mlx5_ib/mlx5_ib.h
	@@ -172,7 +172,7 @@
	struct list_head list;
	struct ib_flow ibflow;
	struct mlx5_ib_flow_prio *prio;
	- struct mlx5_flow_rule *rule;
	+ struct mlx5_flow_handle *rule;
	};

	struct mlx5_ib_flow_db {
	diff --git a/sys/dev/mlx5/mlx5_ib/mlx5_ib_cq.c b/sys/dev/mlx5/mlx5_ib/mlx5_ib_cq.c
	--- a/sys/dev/mlx5/mlx5_ib/mlx5_ib_cq.c
	+++ b/sys/dev/mlx5/mlx5_ib/mlx5_ib_cq.c
	@@ -202,7 +202,7 @@
	case MLX5_CQE_RESP_WR_IMM:
	wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
	wc->wc_flags = IB_WC_WITH_IMM;
	- wc->ex.imm_data = cqe->imm_inval_pkey;
	+ wc->ex.imm_data = cqe->immediate;
	break;
	case MLX5_CQE_RESP_SEND:
	wc->opcode = IB_WC_RECV;
	@@ -214,12 +214,12 @@
	case MLX5_CQE_RESP_SEND_IMM:
	wc->opcode = IB_WC_RECV;
	wc->wc_flags = IB_WC_WITH_IMM;
	- wc->ex.imm_data = cqe->imm_inval_pkey;
	+ wc->ex.imm_data = cqe->immediate;
	break;
	case MLX5_CQE_RESP_SEND_INV:
	wc->opcode = IB_WC_RECV;
	wc->wc_flags = IB_WC_WITH_INVALIDATE;
	- wc->ex.invalidate_rkey = be32_to_cpu(cqe->imm_inval_pkey);
	+ wc->ex.invalidate_rkey = be32_to_cpu(cqe->inval_rkey);
	break;
	}
	wc->src_qp = be32_to_cpu(cqe->flags_rqpn) & 0xffffff;
	@@ -227,7 +227,7 @@
	g = (be32_to_cpu(cqe->flags_rqpn) >> 28) & 3;
	wc->wc_flags \|= g ? IB_WC_GRH : 0;
	if (unlikely(is_qp1(qp->ibqp.qp_type))) {
	- u16 pkey = be32_to_cpu(cqe->imm_inval_pkey) & 0xffff;
	+ u16 pkey = be32_to_cpu(cqe->pkey) & 0xffff;

	ib_find_cached_pkey(&dev->ib_dev, qp->port, pkey,
	&wc->pkey_index);
	diff --git a/sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c b/sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c
	--- a/sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c
	+++ b/sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c
	@@ -2073,13 +2073,13 @@
	mutex_lock(&dev->flow_db.lock);

	list_for_each_entry_safe(iter, tmp, &handler->list, list) {
	- mlx5_del_flow_rule(&iter->rule);
	+ mlx5_del_flow_rules(&iter->rule);
	put_flow_table(dev, iter->prio, true);
	list_del(&iter->list);
	kfree(iter);
	}

	- mlx5_del_flow_rule(&handler->rule);
	+ mlx5_del_flow_rules(&handler->rule);
	put_flow_table(dev, handler->prio, true);
	mutex_unlock(&dev->flow_db.lock);

	@@ -2108,6 +2108,7 @@
	enum flow_table_type ft_type)
	{
	bool dont_trap = flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP;
	+ struct mlx5_flow_table_attr ft_attr = {};
	struct mlx5_flow_namespace *ns = NULL;
	struct mlx5_ib_flow_prio *prio;
	struct mlx5_flow_table *ft;
	@@ -2156,10 +2157,11 @@

	ft = prio->flow_table;
	if (!ft) {
	- ft = mlx5_create_auto_grouped_flow_table(ns, priority, "bypass",
	- num_entries,
	- num_groups,
	- 0);
	+ ft_attr.prio = priority;
	+ ft_attr.max_fte = num_entries;
	+ ft_attr.autogroup.max_num_groups = num_groups;
	+
	+ ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr);

	if (!IS_ERR(ft)) {
	prio->refcount = 0;
	@@ -2182,10 +2184,8 @@
	struct mlx5_flow_spec *spec;
	const void ib_flow = (const void )flow_attr + sizeof(*flow_attr);
	unsigned int spec_index;
	- struct mlx5_flow_act flow_act = {
	- .actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG,
	- .flow_tag = MLX5_FS_DEFAULT_FLOW_TAG,
	- };
	+ struct mlx5_flow_act flow_act = {};
	+
	u32 action;
	int err = 0;

	@@ -2199,6 +2199,9 @@
	goto free;
	}

	+ spec->flow_context.flags = FLOW_CONTEXT_HAS_TAG;
	+ spec->flow_context.flow_tag = MLX5_FS_DEFAULT_FLOW_TAG;
	+
	INIT_LIST_HEAD(&handler->list);

	for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) {
	@@ -2212,12 +2215,8 @@

	spec->match_criteria_enable = get_match_criteria_enable(spec->match_criteria);
	action = dst ? MLX5_FLOW_RULE_FWD_ACTION_DEST : 0;
	- handler->rule = mlx5_add_flow_rule(ft, spec->match_criteria_enable,
	- spec->match_criteria,
	- spec->match_value,
	- action,
	- &flow_act,
	- dst);
	+ flow_act.action = action;
	+ handler->rule = mlx5_add_flow_rules(ft, spec, &flow_act, dst, 1);

	if (IS_ERR(handler->rule)) {
	err = PTR_ERR(handler->rule);
	@@ -2248,7 +2247,7 @@
	handler_dst = create_flow_rule(dev, ft_prio,
	flow_attr, dst);
	if (IS_ERR(handler_dst)) {
	- mlx5_del_flow_rule(&handler->rule);
	+ mlx5_del_flow_rules(&handler->rule);
	ft_prio->refcount--;
	kfree(handler);
	handler = handler_dst;
	@@ -2311,7 +2310,7 @@
	&leftovers_specs[LEFTOVERS_UC].flow_attr,
	dst);
	if (IS_ERR(handler_ucast)) {
	- mlx5_del_flow_rule(&handler->rule);
	+ mlx5_del_flow_rules(&handler->rule);
	ft_prio->refcount--;
	kfree(handler);
	handler = handler_ucast;
	@@ -2354,7 +2353,7 @@
	return handler_rx;

	err_tx:
	- mlx5_del_flow_rule(&handler_rx->rule);
	+ mlx5_del_flow_rules(&handler_rx->rule);
	ft_rx->refcount--;
	kfree(handler_rx);
	err:
	diff --git a/sys/dev/mlx5/mlx5_ifc.h b/sys/dev/mlx5/mlx5_ifc.h
	--- a/sys/dev/mlx5/mlx5_ifc.h
	+++ b/sys/dev/mlx5/mlx5_ifc.h
	@@ -64,7 +64,7 @@
	MLX5_EVENT_TYPE_FPGA_ERROR = 0x20,
	MLX5_EVENT_TYPE_FPGA_QP_ERROR = 0x21,
	MLX5_EVENT_TYPE_DEVICE_TRACER = 0x26,
	- MLX5_EVENT_TYPE_CODING_GENERAL_OBJ_EVENT = 0x27,
	+ MLX5_EVENT_TYPE_OBJECT_CHANGE = 0x27,
	};

	enum {
	@@ -323,8 +323,13 @@
	MLX5_ICMD_CMDS_OPCODE_ICMD_OPCODE_INIT_OCSD = 0xf004
	};

	+enum {
	+ MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_IPSEC = 1ULL << 0x13,
	+};
	+
	enum {
	MLX5_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY = 0xc,
	+ MLX5_GENERAL_OBJECT_TYPES_IPSEC = 0x13,
	};

	enum {
	@@ -337,7 +342,8 @@
	};

	enum {
	- MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_DEK = 0x1,
	+ MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_TLS = 0x1,
	+ MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_IPSEC = 0x2,
	};

	struct mlx5_ifc_flow_table_fields_supported_bits {
	@@ -464,39 +470,70 @@

	u8 reserved_at_340[0x440];
	};
	+
	struct mlx5_ifc_flow_table_prop_layout_bits {
	u8 ft_support[0x1];
	- u8 flow_tag[0x1];
	+ u8 reserved_at_1[0x1];
	u8 flow_counter[0x1];
	u8 flow_modify_en[0x1];
	u8 modify_root[0x1];
	- u8 identified_miss_table[0x1];
	+ u8 identified_miss_table_mode[0x1];
	u8 flow_table_modify[0x1];
	- u8 encap[0x1];
	+ u8 reformat[0x1];
	u8 decap[0x1];
	- u8 reset_root_to_default[0x1];
	- u8 reserved_at_a[0x16];
	-
	- u8 reserved_at_20[0x2];
	+ u8 reserved_at_9[0x1];
	+ u8 pop_vlan[0x1];
	+ u8 push_vlan[0x1];
	+ u8 reserved_at_c[0x1];
	+ u8 pop_vlan_2[0x1];
	+ u8 push_vlan_2[0x1];
	+ u8 reformat_and_vlan_action[0x1];
	+ u8 reserved_at_10[0x1];
	+ u8 sw_owner[0x1];
	+ u8 reformat_l3_tunnel_to_l2[0x1];
	+ u8 reformat_l2_to_l3_tunnel[0x1];
	+ u8 reformat_and_modify_action[0x1];
	+ u8 ignore_flow_level[0x1];
	+ u8 reserved_at_16[0x1];
	+ u8 table_miss_action_domain[0x1];
	+ u8 termination_table[0x1];
	+ u8 reformat_and_fwd_to_table[0x1];
	+ u8 reserved_at_1a[0x2];
	+ u8 ipsec_encrypt[0x1];
	+ u8 ipsec_decrypt[0x1];
	+ u8 sw_owner_v2[0x1];
	+ u8 reserved_at_1f[0x1];
	+ u8 termination_table_raw_traffic[0x1];
	+ u8 reserved_at_21[0x1];
	u8 log_max_ft_size[0x6];
	u8 log_max_modify_header_context[0x8];
	u8 max_modify_header_actions[0x8];
	u8 max_ft_level[0x8];

	- u8 reserved_at_40[0x20];
	-
	- u8 reserved_at_60[0x18];
	+ u8 reformat_add_esp_trasport[0x1];
	+ u8 reformat_l2_to_l3_esp_tunnel[0x1];
	+ u8 reformat_add_esp_transport_over_udp[0x1];
	+ u8 reformat_del_esp_trasport[0x1];
	+ u8 reformat_l3_esp_tunnel_to_l2[0x1];
	+ u8 reformat_del_esp_transport_over_udp[0x1];
	+ u8 execute_aso[0x1];
	+ u8 reserved_at_47[0x19];
	+ u8 reserved_at_60[0x2];
	+ u8 reformat_insert[0x1];
	+ u8 reformat_remove[0x1];
	+ u8 macsec_encrypt[0x1];
	+ u8 macsec_decrypt[0x1];
	+ u8 reserved_at_66[0x2];
	+ u8 reformat_add_macsec[0x1];
	+ u8 reformat_remove_macsec[0x1];
	+ u8 reserved_at_6a[0xe];
	u8 log_max_ft_num[0x8];
	-
	u8 reserved_at_80[0x10];
	u8 log_max_flow_counter[0x8];
	u8 log_max_destination[0x8];
	-
	u8 reserved_at_a0[0x18];
	u8 log_max_flow[0x8];
	-
	u8 reserved_at_c0[0x40];
	-
	struct mlx5_ifc_flow_table_fields_supported_bits ft_field_support;

	struct mlx5_ifc_flow_table_fields_supported_bits ft_field_bitmask_support;
	@@ -524,15 +561,18 @@
	MLX5_FLOW_CONTEXT_DEST_TYPE_FLOW_TABLE = 0x1,
	MLX5_FLOW_CONTEXT_DEST_TYPE_TIR = 0x2,
	MLX5_FLOW_CONTEXT_DEST_TYPE_QP = 0x3,
	+ MLX5_FLOW_CONTEXT_DEST_TYPE_TABLE_TYPE = 0xA,
	};

	struct mlx5_ifc_dest_format_struct_bits {
	- u8 destination_type[0x8];
	- u8 destination_id[0x18];
	+ u8 destination_type[0x8];
	+ u8 destination_id[0x18];

	- u8 reserved_0[0x8];
	- u8 destination_table_type[0x8];
	- u8 reserved_at_1[0x10];
	+ u8 destination_eswitch_owner_vhca_id_valid[0x1];
	+ u8 packet_reformat[0x1];
	+ u8 reserved_at_22[0x6];
	+ u8 destination_table_type[0x8];
	+ u8 destination_eswitch_owner_vhca_id[0x10];
	};

	struct mlx5_ifc_ipv4_layout_bits {
	@@ -586,11 +626,25 @@
	union mlx5_ifc_ipv6_layout_ipv4_layout_auto_bits dst_ipv4_dst_ipv6;
	};

	+struct mlx5_ifc_nvgre_key_bits {
	+ u8 hi[0x18];
	+ u8 lo[0x8];
	+};
	+
	+union mlx5_ifc_gre_key_bits {
	+ struct mlx5_ifc_nvgre_key_bits nvgre;
	+ u8 key[0x20];
	+};
	+
	struct mlx5_ifc_fte_match_set_misc_bits {
	- u8 reserved_0[0x8];
	+ u8 gre_c_present[0x1];
	+ u8 reserved_at_1[0x1];
	+ u8 gre_k_present[0x1];
	+ u8 gre_s_present[0x1];
	+ u8 source_vhca_port[0x4];
	u8 source_sqn[0x18];

	- u8 reserved_1[0x10];
	+ u8 source_eswitch_owner_vhca_id[0x10];
	u8 source_port[0x10];

	u8 outer_second_prio[0x3];
	@@ -600,35 +654,163 @@
	u8 inner_second_cfi[0x1];
	u8 inner_second_vid[0xc];

	- u8 outer_second_vlan_tag[0x1];
	- u8 inner_second_vlan_tag[0x1];
	- u8 reserved_2[0xe];
	+ u8 outer_second_cvlan_tag[0x1];
	+ u8 inner_second_cvlan_tag[0x1];
	+ u8 outer_second_svlan_tag[0x1];
	+ u8 inner_second_svlan_tag[0x1];
	+ u8 reserved_at_64[0xc];
	u8 gre_protocol[0x10];

	- u8 gre_key_h[0x18];
	- u8 gre_key_l[0x8];
	+ union mlx5_ifc_gre_key_bits gre_key;

	u8 vxlan_vni[0x18];
	- u8 reserved_3[0x8];
	+ u8 bth_opcode[0x8];

	u8 geneve_vni[0x18];
	- u8 reserved4[0x7];
	+ u8 reserved_at_d8[0x6];
	+ u8 geneve_tlv_option_0_exist[0x1];
	u8 geneve_oam[0x1];

	- u8 reserved_5[0xc];
	+ u8 reserved_at_e0[0xc];
	u8 outer_ipv6_flow_label[0x14];

	- u8 reserved_6[0xc];
	+ u8 reserved_at_100[0xc];
	u8 inner_ipv6_flow_label[0x14];

	- u8 reserved_7[0xa];
	+ u8 reserved_at_120[0xa];
	u8 geneve_opt_len[0x6];
	u8 geneve_protocol_type[0x10];

	- u8 reserved_8[0x8];
	+ u8 reserved_at_140[0x8];
	u8 bth_dst_qp[0x18];
	+ u8 inner_esp_spi[0x20];
	+ u8 outer_esp_spi[0x20];
	+ u8 reserved_at_1a0[0x60];
	+};

	- u8 reserved_9[0xa0];
	+struct mlx5_ifc_fte_match_mpls_bits {
	+ u8 mpls_label[0x14];
	+ u8 mpls_exp[0x3];
	+ u8 mpls_s_bos[0x1];
	+ u8 mpls_ttl[0x8];
	+};
	+
	+struct mlx5_ifc_fte_match_set_misc2_bits {
	+ struct mlx5_ifc_fte_match_mpls_bits outer_first_mpls;
	+
	+ struct mlx5_ifc_fte_match_mpls_bits inner_first_mpls;
	+
	+ struct mlx5_ifc_fte_match_mpls_bits outer_first_mpls_over_gre;
	+
	+ struct mlx5_ifc_fte_match_mpls_bits outer_first_mpls_over_udp;
	+
	+ u8 metadata_reg_c_7[0x20];
	+
	+ u8 metadata_reg_c_6[0x20];
	+
	+ u8 metadata_reg_c_5[0x20];
	+
	+ u8 metadata_reg_c_4[0x20];
	+
	+ u8 metadata_reg_c_3[0x20];
	+
	+ u8 metadata_reg_c_2[0x20];
	+
	+ u8 metadata_reg_c_1[0x20];
	+
	+ u8 metadata_reg_c_0[0x20];
	+
	+ u8 metadata_reg_a[0x20];
	+
	+ u8 reserved_at_1a0[0x8];
	+
	+ u8 macsec_syndrome[0x8];
	+ u8 ipsec_syndrome[0x8];
	+ u8 reserved_at_1b8[0x8];
	+
	+ u8 reserved_at_1c0[0x40];
	+};
	+
	+struct mlx5_ifc_fte_match_set_misc3_bits {
	+ u8 inner_tcp_seq_num[0x20];
	+
	+ u8 outer_tcp_seq_num[0x20];
	+
	+ u8 inner_tcp_ack_num[0x20];
	+
	+ u8 outer_tcp_ack_num[0x20];
	+
	+ u8 reserved_at_80[0x8];
	+ u8 outer_vxlan_gpe_vni[0x18];
	+
	+ u8 outer_vxlan_gpe_next_protocol[0x8];
	+ u8 outer_vxlan_gpe_flags[0x8];
	+ u8 reserved_at_b0[0x10];
	+
	+ u8 icmp_header_data[0x20];
	+
	+ u8 icmpv6_header_data[0x20];
	+
	+ u8 icmp_type[0x8];
	+ u8 icmp_code[0x8];
	+ u8 icmpv6_type[0x8];
	+ u8 icmpv6_code[0x8];
	+
	+ u8 geneve_tlv_option_0_data[0x20];
	+
	+ u8 gtpu_teid[0x20];
	+
	+ u8 gtpu_msg_type[0x8];
	+ u8 gtpu_msg_flags[0x8];
	+ u8 reserved_at_170[0x10];
	+
	+ u8 gtpu_dw_2[0x20];
	+
	+ u8 gtpu_first_ext_dw_0[0x20];
	+
	+ u8 gtpu_dw_0[0x20];
	+
	+ u8 reserved_at_1e0[0x20];
	+};
	+
	+struct mlx5_ifc_fte_match_set_misc4_bits {
	+ u8 prog_sample_field_value_0[0x20];
	+
	+ u8 prog_sample_field_id_0[0x20];
	+
	+ u8 prog_sample_field_value_1[0x20];
	+
	+ u8 prog_sample_field_id_1[0x20];
	+
	+ u8 prog_sample_field_value_2[0x20];
	+
	+ u8 prog_sample_field_id_2[0x20];
	+
	+ u8 prog_sample_field_value_3[0x20];
	+
	+ u8 prog_sample_field_id_3[0x20];
	+
	+ u8 reserved_at_100[0x100];
	+};
	+
	+struct mlx5_ifc_fte_match_set_misc5_bits {
	+ u8 macsec_tag_0[0x20];
	+
	+ u8 macsec_tag_1[0x20];
	+
	+ u8 macsec_tag_2[0x20];
	+
	+ u8 macsec_tag_3[0x20];
	+
	+ u8 tunnel_header_0[0x20];
	+
	+ u8 tunnel_header_1[0x20];
	+
	+ u8 tunnel_header_2[0x20];
	+
	+ u8 tunnel_header_3[0x20];
	+
	+ u8 reserved_at_100[0x100];
	};

	struct mlx5_ifc_cmd_pas_bits {
	@@ -866,6 +1048,20 @@
	u8 reserved_1[0x7200];
	};

	+struct mlx5_ifc_port_selection_cap_bits {
	+ u8 reserved_at_0[0x10];
	+ u8 port_select_flow_table[0x1];
	+ u8 reserved_at_11[0x1];
	+ u8 port_select_flow_table_bypass[0x1];
	+ u8 reserved_at_13[0xd];
	+
	+ u8 reserved_at_20[0x1e0];
	+
	+ struct mlx5_ifc_flow_table_prop_layout_bits flow_table_properties_port_selection;
	+
	+ u8 reserved_at_400[0x7c00];
	+};
	+
	struct mlx5_ifc_pddr_module_info_bits {
	u8 cable_technology[0x8];
	u8 cable_breakout[0x8];
	@@ -1157,7 +1353,15 @@
	u8 reserved_0[0x20];

	u8 hca_cap_2[0x1];
	- u8 reserved_at_21[0x1f];
	+ u8 create_lag_when_not_master_up[0x1];
	+ u8 dtor[0x1];
	+ u8 event_on_vhca_state_teardown_request[0x1];
	+ u8 event_on_vhca_state_in_use[0x1];
	+ u8 event_on_vhca_state_active[0x1];
	+ u8 event_on_vhca_state_allocated[0x1];
	+ u8 event_on_vhca_state_invalid[0x1];
	+ u8 reserved_at_28[0x8];
	+ u8 vhca_id[0x10];

	u8 reserved_at_40[0x40];

	@@ -1407,7 +1611,8 @@

	u8 reserved_at_460[0x3];
	u8 log_max_uctx[0x5];
	- u8 reserved_at_468[0x3];
	+ u8 reserved_at_468[0x2];
	+ u8 ipsec_offload[0x1];
	u8 log_max_umem[0x5];
	u8 max_num_eqs[0x10];

	@@ -1491,11 +1696,27 @@
	u8 reserved_at_260[0x5a0];
	};

	-enum mlx5_flow_destination_type {
	- MLX5_FLOW_DESTINATION_TYPE_VPORT = 0x0,
	- MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE = 0x1,
	- MLX5_FLOW_DESTINATION_TYPE_TIR = 0x2,
	- MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE = 0xA,
	+enum mlx5_ifc_flow_destination_type {
	+ MLX5_IFC_FLOW_DESTINATION_TYPE_VPORT = 0x0,
	+ MLX5_IFC_FLOW_DESTINATION_TYPE_FLOW_TABLE = 0x1,
	+ MLX5_IFC_FLOW_DESTINATION_TYPE_TIR = 0x2,
	+ MLX5_IFC_FLOW_DESTINATION_TYPE_FLOW_SAMPLER = 0x6,
	+ MLX5_IFC_FLOW_DESTINATION_TYPE_UPLINK = 0x8,
	+ MLX5_IFC_FLOW_DESTINATION_TYPE_TABLE_TYPE = 0xA,
	+};
	+
	+enum mlx5_flow_table_miss_action {
	+ MLX5_FLOW_TABLE_MISS_ACTION_DEF,
	+ MLX5_FLOW_TABLE_MISS_ACTION_FWD,
	+ MLX5_FLOW_TABLE_MISS_ACTION_SWITCH_DOMAIN,
	+};
	+
	+struct mlx5_ifc_extended_dest_format_bits {
	+ struct mlx5_ifc_dest_format_struct_bits destination_entry;
	+
	+ u8 packet_reformat_id[0x20];
	+
	+ u8 reserved_at_60[0x20];
	};

	union mlx5_ifc_dest_format_struct_flow_counter_list_auto_bits {
	@@ -1505,13 +1726,21 @@
	};

	struct mlx5_ifc_fte_match_param_bits {
	- struct mlx5_ifc_fte_match_set_lyr_2_4_bits outer_headers;
	+ struct mlx5_ifc_fte_match_set_lyr_2_4_bits outer_headers;
	+
	+ struct mlx5_ifc_fte_match_set_misc_bits misc_parameters;
	+
	+ struct mlx5_ifc_fte_match_set_lyr_2_4_bits inner_headers;
	+
	+ struct mlx5_ifc_fte_match_set_misc2_bits misc_parameters_2;

	- struct mlx5_ifc_fte_match_set_misc_bits misc_parameters;
	+ struct mlx5_ifc_fte_match_set_misc3_bits misc_parameters_3;

	- struct mlx5_ifc_fte_match_set_lyr_2_4_bits inner_headers;
	+ struct mlx5_ifc_fte_match_set_misc4_bits misc_parameters_4;

	- u8 reserved_0[0xa00];
	+ struct mlx5_ifc_fte_match_set_misc5_bits misc_parameters_5;
	+
	+ u8 reserved_at_e00[0x200];
	};

	enum {
	@@ -2313,43 +2542,85 @@
	u8 atomic_resp[32][0x8];
	};

	+struct mlx5_ifc_vlan_bits {
	+ u8 ethtype[0x10];
	+ u8 prio[0x3];
	+ u8 cfi[0x1];
	+ u8 vid[0xc];
	+};
	+
	enum {
	- MLX5_FLOW_CONTEXT_ACTION_ALLOW = 0x1,
	- MLX5_FLOW_CONTEXT_ACTION_DROP = 0x2,
	- MLX5_FLOW_CONTEXT_ACTION_FWD_DEST = 0x4,
	- MLX5_FLOW_CONTEXT_ACTION_COUNT = 0x8,
	- MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT = 0x10,
	- MLX5_FLOW_CONTEXT_ACTION_MOD_HDR = 0x40,
	+ MLX5_FLOW_METER_COLOR_RED = 0x0,
	+ MLX5_FLOW_METER_COLOR_YELLOW = 0x1,
	+ MLX5_FLOW_METER_COLOR_GREEN = 0x2,
	+ MLX5_FLOW_METER_COLOR_UNDEFINED = 0x3,
	+};
	+
	+enum {
	+ MLX5_EXE_ASO_FLOW_METER = 0x2,
	+};
	+
	+struct mlx5_ifc_exe_aso_ctrl_flow_meter_bits {
	+ u8 return_reg_id[0x4];
	+ u8 aso_type[0x4];
	+ u8 reserved_at_8[0x14];
	+ u8 action[0x1];
	+ u8 init_color[0x2];
	+ u8 meter_id[0x1];
	+};
	+
	+union mlx5_ifc_exe_aso_ctrl {
	+ struct mlx5_ifc_exe_aso_ctrl_flow_meter_bits exe_aso_ctrl_flow_meter;
	+};
	+
	+struct mlx5_ifc_execute_aso_bits {
	+ u8 valid[0x1];
	+ u8 reserved_at_1[0x7];
	+ u8 aso_object_id[0x18];
	+
	+ union mlx5_ifc_exe_aso_ctrl exe_aso_ctrl;
	+};
	+
	+enum {
	+ MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_IPSEC = 0x0,
	};

	struct mlx5_ifc_flow_context_bits {
	- u8 reserved_0[0x20];
	+ struct mlx5_ifc_vlan_bits push_vlan;

	u8 group_id[0x20];

	- u8 reserved_1[0x8];
	+ u8 reserved_at_40[0x8];
	u8 flow_tag[0x18];

	- u8 reserved_2[0x10];
	+ u8 reserved_at_60[0x10];
	u8 action[0x10];

	- u8 reserved_3[0x8];
	+ u8 extended_destination[0x1];
	+ u8 reserved_at_81[0x1];
	+ u8 flow_source[0x2];
	+ u8 encrypt_decrypt_type[0x4];
	u8 destination_list_size[0x18];

	- u8 reserved_4[0x8];
	+ u8 reserved_at_a0[0x8];
	u8 flow_counter_list_size[0x18];

	u8 packet_reformat_id[0x20];

	- u8 modify_header_id[0x20];
	+ u8 modify_header_id[0x20];

	- u8 reserved_6[0x100];
	+ struct mlx5_ifc_vlan_bits push_vlan_2;
	+
	+ u8 encrypt_decrypt_obj_id[0x20];
	+ u8 reserved_at_140[0xc0];

	struct mlx5_ifc_fte_match_param_bits match_value;

	- u8 reserved_7[0x600];
	+ struct mlx5_ifc_execute_aso_bits execute_aso[4];
	+
	+ u8 reserved_at_1300[0x500];

	- union mlx5_ifc_dest_format_struct_flow_counter_list_auto_bits destination[0];
	+ union mlx5_ifc_dest_format_struct_flow_counter_list_auto_bits destination[];
	};

	enum {
	@@ -3011,21 +3282,27 @@
	};

	struct mlx5_ifc_flow_table_context_bits {
	- u8 reformat_en[0x1];
	- u8 decap_en[0x1];
	- u8 reserved_at_2[0x2];
	- u8 table_miss_action[0x4];
	- u8 level[0x8];
	- u8 reserved_at_10[0x8];
	- u8 log_size[0x8];
	+ u8 reformat_en[0x1];
	+ u8 decap_en[0x1];
	+ u8 sw_owner[0x1];
	+ u8 termination_table[0x1];
	+ u8 table_miss_action[0x4];
	+ u8 level[0x8];
	+ u8 reserved_at_10[0x8];
	+ u8 log_size[0x8];

	- u8 reserved_at_20[0x8];
	- u8 table_miss_id[0x18];
	+ u8 reserved_at_20[0x8];
	+ u8 table_miss_id[0x18];

	- u8 reserved_at_40[0x8];
	- u8 lag_master_next_table_id[0x18];
	+ u8 reserved_at_40[0x8];
	+ u8 lag_master_next_table_id[0x18];
	+
	+ u8 reserved_at_60[0x60];
	+
	+ u8 sw_owner_icm_root_1[0x40];
	+
	+ u8 sw_owner_icm_root_0[0x40];

	- u8 reserved_at_60[0xe0];
	};

	struct mlx5_ifc_esw_vport_context_bits {
	@@ -3983,28 +4260,32 @@
	};

	struct mlx5_ifc_set_flow_table_root_in_bits {
	- u8 opcode[0x10];
	- u8 reserved_0[0x10];
	-
	- u8 reserved_1[0x10];
	- u8 op_mod[0x10];
	+ u8 opcode[0x10];
	+ u8 reserved_at_10[0x10];

	- u8 other_vport[0x1];
	- u8 reserved_2[0xf];
	- u8 vport_number[0x10];
	+ u8 reserved_at_20[0x10];
	+ u8 op_mod[0x10];

	- u8 reserved_3[0x20];
	+ u8 other_vport[0x1];
	+ u8 reserved_at_41[0xf];
	+ u8 vport_number[0x10];

	- u8 table_type[0x8];
	- u8 reserved_4[0x18];
	+ u8 reserved_at_60[0x20];

	- u8 reserved_5[0x8];
	- u8 table_id[0x18];
	+ u8 table_type[0x8];
	+ u8 reserved_at_88[0x7];
	+ u8 table_of_other_vport[0x1];
	+ u8 table_vport_number[0x10];

	- u8 reserved_6[0x8];
	- u8 underlay_qpn[0x18];
	+ u8 reserved_at_a0[0x8];
	+ u8 table_id[0x18];

	- u8 reserved_7[0x120];
	+ u8 reserved_at_c0[0x8];
	+ u8 underlay_qpn[0x18];
	+ u8 table_eswitch_owner_vhca_id_valid[0x1];
	+ u8 reserved_at_e1[0xf];
	+ u8 table_eswitch_owner_vhca_id[0x10];
	+ u8 reserved_at_100[0x100];
	};

	struct mlx5_ifc_set_fte_out_bits {
	@@ -4017,34 +4298,35 @@
	};

	struct mlx5_ifc_set_fte_in_bits {
	- u8 opcode[0x10];
	- u8 reserved_0[0x10];
	+ u8 opcode[0x10];
	+ u8 reserved_at_10[0x10];

	- u8 reserved_1[0x10];
	- u8 op_mod[0x10];
	+ u8 reserved_at_20[0x10];
	+ u8 op_mod[0x10];

	- u8 other_vport[0x1];
	- u8 reserved_2[0xf];
	- u8 vport_number[0x10];
	+ u8 other_vport[0x1];
	+ u8 reserved_at_41[0xf];
	+ u8 vport_number[0x10];

	- u8 reserved_3[0x20];
	+ u8 reserved_at_60[0x20];

	- u8 table_type[0x8];
	- u8 reserved_4[0x18];
	+ u8 table_type[0x8];
	+ u8 reserved_at_88[0x18];

	- u8 reserved_5[0x8];
	- u8 table_id[0x18];
	+ u8 reserved_at_a0[0x8];
	+ u8 table_id[0x18];

	- u8 reserved_6[0x18];
	- u8 modify_enable_mask[0x8];
	+ u8 ignore_flow_level[0x1];
	+ u8 reserved_at_c1[0x17];
	+ u8 modify_enable_mask[0x8];

	- u8 reserved_7[0x20];
	+ u8 reserved_at_e0[0x20];

	- u8 flow_index[0x20];
	+ u8 flow_index[0x20];

	- u8 reserved_8[0xe0];
	+ u8 reserved_at_120[0xe0];

	- struct mlx5_ifc_flow_context_bits flow_context;
	+ struct mlx5_ifc_flow_context_bits flow_context;
	};

	struct mlx5_ifc_set_driver_version_out_bits {
	@@ -4325,6 +4607,23 @@
	MLX5_QUERY_VPORT_STATE_IN_OP_MOD_UPLINK = 0x2,
	};

	+enum {
	+ MLX5_FLOW_CONTEXT_ACTION_ALLOW = 0x1,
	+ MLX5_FLOW_CONTEXT_ACTION_DROP = 0x2,
	+ MLX5_FLOW_CONTEXT_ACTION_FWD_DEST = 0x4,
	+ MLX5_FLOW_CONTEXT_ACTION_COUNT = 0x8,
	+ MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT = 0x10,
	+ MLX5_FLOW_CONTEXT_ACTION_DECAP = 0x20,
	+ MLX5_FLOW_CONTEXT_ACTION_MOD_HDR = 0x40,
	+ MLX5_FLOW_CONTEXT_ACTION_VLAN_POP = 0x80,
	+ MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH = 0x100,
	+ MLX5_FLOW_CONTEXT_ACTION_VLAN_POP_2 = 0x400,
	+ MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2 = 0x800,
	+ MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT = 0x1000,
	+ MLX5_FLOW_CONTEXT_ACTION_CRYPTO_ENCRYPT = 0x2000,
	+ MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO = 0x4000,
	+};
	+
	struct mlx5_ifc_query_vport_state_in_bits {
	u8 opcode[0x10];
	u8 reserved_0[0x10];
	@@ -5545,6 +5844,12 @@
	MLX5_REFORMAT_TYPE_L2_TO_L2_TUNNEL = 0x2,
	MLX5_REFORMAT_TYPE_L3_TUNNEL_TO_L2 = 0x3,
	MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL = 0x4,
	+ MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV4 = 0x5,
	+ MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_UDPV4 = 0x7,
	+ MLX5_REFORMAT_TYPE_DEL_ESP_TRANSPORT = 0x8,
	+ MLX5_REFORMAT_TYPE_DEL_ESP_TRANSPORT_OVER_UDP = 0xa,
	+ MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV6 = 0xb,
	+ MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_UDPV6 = 0xc,
	};

	struct mlx5_ifc_alloc_packet_reformat_context_in_bits {
	@@ -6255,6 +6560,11 @@
	struct mlx5_ifc_hca_vport_context_bits hca_vport_context;
	};

	+enum {
	+ MLX5_MODIFY_FLOW_TABLE_MISS_TABLE_ID = (1UL << 0),
	+ MLX5_MODIFY_FLOW_TABLE_LAG_NEXT_TABLE_ID = (1UL << 15),
	+};
	+
	struct mlx5_ifc_modify_flow_table_out_bits {
	u8 status[0x8];
	u8 reserved_at_8[0x18];
	@@ -7898,24 +8208,24 @@
	};

	struct mlx5_ifc_create_flow_table_in_bits {
	- u8 opcode[0x10];
	- u8 reserved_at_10[0x10];
	+ u8 opcode[0x10];
	+ u8 uid[0x10];

	- u8 reserved_at_20[0x10];
	- u8 op_mod[0x10];
	+ u8 reserved_at_20[0x10];
	+ u8 op_mod[0x10];

	- u8 other_vport[0x1];
	- u8 reserved_at_41[0xf];
	- u8 vport_number[0x10];
	+ u8 other_vport[0x1];
	+ u8 reserved_at_41[0xf];
	+ u8 vport_number[0x10];

	- u8 reserved_at_60[0x20];
	+ u8 reserved_at_60[0x20];

	- u8 table_type[0x8];
	- u8 reserved_at_88[0x18];
	+ u8 table_type[0x8];
	+ u8 reserved_at_88[0x18];

	- u8 reserved_at_a0[0x20];
	+ u8 reserved_at_a0[0x20];

	- struct mlx5_ifc_flow_table_context_bits flow_table_context;
	+ struct mlx5_ifc_flow_table_context_bits flow_table_context;
	};

	struct mlx5_ifc_create_flow_group_out_bits {
	@@ -7931,46 +8241,54 @@
	};

	enum {
	- MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS = 0x0,
	- MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS = 0x1,
	- MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_INNER_HEADERS = 0x2,
	+ MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS = 0x0,
	+ MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS = 0x1,
	+ MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_INNER_HEADERS = 0x2,
	+ MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS_2 = 0x3,
	};

	struct mlx5_ifc_create_flow_group_in_bits {
	- u8 opcode[0x10];
	- u8 reserved_0[0x10];
	+ u8 opcode[0x10];
	+ u8 reserved_at_10[0x10];

	- u8 reserved_1[0x10];
	- u8 op_mod[0x10];
	+ u8 reserved_at_20[0x10];
	+ u8 op_mod[0x10];

	- u8 other_vport[0x1];
	- u8 reserved_2[0xf];
	- u8 vport_number[0x10];
	+ u8 other_vport[0x1];
	+ u8 reserved_at_41[0xf];
	+ u8 vport_number[0x10];

	- u8 reserved_3[0x20];
	+ u8 reserved_at_60[0x20];

	- u8 table_type[0x8];
	- u8 reserved_4[0x18];
	+ u8 table_type[0x8];
	+ u8 reserved_at_88[0x4];
	+ u8 group_type[0x4];
	+ u8 reserved_at_90[0x10];

	- u8 reserved_5[0x8];
	- u8 table_id[0x18];
	+ u8 reserved_at_a0[0x8];
	+ u8 table_id[0x18];

	- u8 reserved_6[0x20];
	+ u8 source_eswitch_owner_vhca_id_valid[0x1];

	- u8 start_flow_index[0x20];
	+ u8 reserved_at_c1[0x1f];

	- u8 reserved_7[0x20];
	+ u8 start_flow_index[0x20];

	- u8 end_flow_index[0x20];
	+ u8 reserved_at_100[0x20];

	- u8 reserved_8[0xa0];
	+ u8 end_flow_index[0x20];

	- u8 reserved_9[0x18];
	- u8 match_criteria_enable[0x8];
	+ u8 reserved_at_140[0x10];
	+ u8 match_definer_id[0x10];

	- struct mlx5_ifc_fte_match_param_bits match_criteria;
	+ u8 reserved_at_160[0x80];
	+
	+ u8 reserved_at_1e0[0x18];
	+ u8 match_criteria_enable[0x8];
	+
	+ struct mlx5_ifc_fte_match_param_bits match_criteria;

	- u8 reserved_10[0xe00];
	+ u8 reserved_at_1200[0xe00];
	};

	struct mlx5_ifc_create_encryption_key_out_bits {
	@@ -11637,5 +11955,120 @@

	#define MLX5_FC_BULK_NUM_FCS(fc_enum) (MLX5_FC_BULK_SIZE_FACTOR * (fc_enum))

	+struct mlx5_ifc_ipsec_cap_bits {
	+ u8 ipsec_full_offload[0x1];
	+ u8 ipsec_crypto_offload[0x1];
	+ u8 ipsec_esn[0x1];
	+ u8 ipsec_crypto_esp_aes_gcm_256_encrypt[0x1];
	+ u8 ipsec_crypto_esp_aes_gcm_128_encrypt[0x1];
	+ u8 ipsec_crypto_esp_aes_gcm_256_decrypt[0x1];
	+ u8 ipsec_crypto_esp_aes_gcm_128_decrypt[0x1];
	+ u8 reserved_at_7[0x4];
	+ u8 log_max_ipsec_offload[0x5];
	+ u8 reserved_at_10[0x10];
	+
	+ u8 min_log_ipsec_full_replay_window[0x8];
	+ u8 max_log_ipsec_full_replay_window[0x8];
	+ u8 reserved_at_30[0x7d0];
	+};
	+
	+enum {
	+ MLX5_IPSEC_OBJECT_ICV_LEN_16B,
	+};
	+
	+enum {
	+ MLX5_IPSEC_ASO_REG_C_0_1 = 0x0,
	+ MLX5_IPSEC_ASO_REG_C_2_3 = 0x1,
	+ MLX5_IPSEC_ASO_REG_C_4_5 = 0x2,
	+ MLX5_IPSEC_ASO_REG_C_6_7 = 0x3,
	+};
	+
	+enum {
	+ MLX5_IPSEC_ASO_MODE = 0x0,
	+ MLX5_IPSEC_ASO_REPLAY_PROTECTION = 0x1,
	+ MLX5_IPSEC_ASO_INC_SN = 0x2,
	+};
	+
	+enum {
	+ MLX5_IPSEC_ASO_REPLAY_WIN_32BIT = 0x0,
	+ MLX5_IPSEC_ASO_REPLAY_WIN_64BIT = 0x1,
	+ MLX5_IPSEC_ASO_REPLAY_WIN_128BIT = 0x2,
	+ MLX5_IPSEC_ASO_REPLAY_WIN_256BIT = 0x3,
	+};
	+
	+struct mlx5_ifc_ipsec_aso_bits {
	+ u8 valid[0x1];
	+ u8 reserved_at_201[0x1];
	+ u8 mode[0x2];
	+ u8 window_sz[0x2];
	+ u8 soft_lft_arm[0x1];
	+ u8 hard_lft_arm[0x1];
	+ u8 remove_flow_enable[0x1];
	+ u8 esn_event_arm[0x1];
	+ u8 reserved_at_20a[0x16];
	+
	+ u8 remove_flow_pkt_cnt[0x20];
	+
	+ u8 remove_flow_soft_lft[0x20];
	+
	+ u8 reserved_at_260[0x80];
	+
	+ u8 mode_parameter[0x20];
	+
	+ u8 replay_protection_window[0x100];
	+};
	+
	+struct mlx5_ifc_ipsec_obj_bits {
	+ u8 modify_field_select[0x40];
	+ u8 full_offload[0x1];
	+ u8 reserved_at_41[0x1];
	+ u8 esn_en[0x1];
	+ u8 esn_overlap[0x1];
	+ u8 reserved_at_44[0x2];
	+ u8 icv_length[0x2];
	+ u8 reserved_at_48[0x4];
	+ u8 aso_return_reg[0x4];
	+ u8 reserved_at_50[0x10];
	+
	+ u8 esn_msb[0x20];
	+
	+ u8 reserved_at_80[0x8];
	+ u8 dekn[0x18];
	+
	+ u8 salt[0x20];
	+
	+ u8 implicit_iv[0x40];

	+ u8 reserved_at_100[0x8];
	+ u8 ipsec_aso_access_pd[0x18];
	+ u8 reserved_at_120[0xe0];
	+
	+ struct mlx5_ifc_ipsec_aso_bits ipsec_aso;
	+};
	+
	+struct mlx5_ifc_create_ipsec_obj_in_bits {
	+ struct mlx5_ifc_general_obj_in_cmd_hdr_bits general_obj_in_cmd_hdr;
	+ struct mlx5_ifc_ipsec_obj_bits ipsec_object;
	+};
	+
	+enum {
	+ MLX5_MODIFY_IPSEC_BITMASK_ESN_OVERLAP = 1 << 0,
	+ MLX5_MODIFY_IPSEC_BITMASK_ESN_MSB = 1 << 1,
	+};
	+
	+struct mlx5_ifc_query_ipsec_obj_out_bits {
	+ struct mlx5_ifc_general_obj_out_cmd_hdr_bits general_obj_out_cmd_hdr;
	+ struct mlx5_ifc_ipsec_obj_bits ipsec_object;
	+};
	+
	+struct mlx5_ifc_modify_ipsec_obj_in_bits {
	+ struct mlx5_ifc_general_obj_in_cmd_hdr_bits general_obj_in_cmd_hdr;
	+ struct mlx5_ifc_ipsec_obj_bits ipsec_object;
	+};
	+
	+enum {
	+ MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_PURPOSE_TLS = 0x1,
	+ MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_PURPOSE_IPSEC = 0x2,
	+ MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_PURPOSE_MACSEC = 0x4,
	+};
	#endif /* MLX5_IFC_H */
	diff --git a/sys/dev/mlx5/mlx5_lib/aso.h b/sys/dev/mlx5/mlx5_lib/aso.h
	new file mode 100644
	--- /dev/null
	+++ b/sys/dev/mlx5/mlx5_lib/aso.h
	@@ -0,0 +1,92 @@
	+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
	+/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */
	+
	+#ifndef __MLX5_LIB_ASO_H__
	+#define __MLX5_LIB_ASO_H__
	+
	+#include <dev/mlx5/qp.h>
	+#include <dev/mlx5/mlx5_core/mlx5_core.h>
	+
	+#define MLX5_ASO_WQEBBS \
	+ (DIV_ROUND_UP(sizeof(struct mlx5_aso_wqe), MLX5_SEND_WQE_BB))
	+#define MLX5_ASO_WQEBBS_DATA \
	+ (DIV_ROUND_UP(sizeof(struct mlx5_aso_wqe_data), MLX5_SEND_WQE_BB))
	+#define ASO_CTRL_READ_EN BIT(0)
	+#define MLX5_WQE_CTRL_WQE_OPC_MOD_SHIFT 24
	+#define MLX5_MACSEC_ASO_DS_CNT (DIV_ROUND_UP(sizeof(struct mlx5_aso_wqe), MLX5_SEND_WQE_DS))
	+
	+#define ASO_CTRL_READ_EN BIT(0)
	+struct mlx5_wqe_aso_ctrl_seg {
	+ __be32 va_h;
	+ __be32 va_l; /* include read_enable */
	+ __be32 l_key;
	+ u8 data_mask_mode;
	+ u8 condition_1_0_operand;
	+ u8 condition_1_0_offset;
	+ u8 data_offset_condition_operand;
	+ __be32 condition_0_data;
	+ __be32 condition_0_mask;
	+ __be32 condition_1_data;
	+ __be32 condition_1_mask;
	+ __be64 bitwise_data;
	+ __be64 data_mask;
	+};
	+
	+struct mlx5_wqe_aso_data_seg {
	+ __be32 bytewise_data[16];
	+};
	+
	+struct mlx5_aso_wqe {
	+ struct mlx5_wqe_ctrl_seg ctrl;
	+ struct mlx5_wqe_aso_ctrl_seg aso_ctrl;
	+};
	+
	+struct mlx5_aso_wqe_data {
	+ struct mlx5_wqe_ctrl_seg ctrl;
	+ struct mlx5_wqe_aso_ctrl_seg aso_ctrl;
	+ struct mlx5_wqe_aso_data_seg aso_data;
	+};
	+
	+enum {
	+ MLX5_ASO_LOGICAL_AND,
	+ MLX5_ASO_LOGICAL_OR,
	+};
	+
	+enum {
	+ MLX5_ASO_ALWAYS_FALSE,
	+ MLX5_ASO_ALWAYS_TRUE,
	+ MLX5_ASO_EQUAL,
	+ MLX5_ASO_NOT_EQUAL,
	+ MLX5_ASO_GREATER_OR_EQUAL,
	+ MLX5_ASO_LESSER_OR_EQUAL,
	+ MLX5_ASO_LESSER,
	+ MLX5_ASO_GREATER,
	+ MLX5_ASO_CYCLIC_GREATER,
	+ MLX5_ASO_CYCLIC_LESSER,
	+};
	+
	+enum {
	+ MLX5_ASO_DATA_MASK_MODE_BITWISE_64BIT,
	+ MLX5_ASO_DATA_MASK_MODE_BYTEWISE_64BYTE,
	+ MLX5_ASO_DATA_MASK_MODE_CALCULATED_64BYTE,
	+};
	+
	+enum {
	+ MLX5_ACCESS_ASO_OPC_MOD_IPSEC = 0x0,
	+ MLX5_ACCESS_ASO_OPC_MOD_FLOW_METER = 0x2,
	+ MLX5_ACCESS_ASO_OPC_MOD_MACSEC = 0x5,
	+};
	+
	+struct mlx5_aso;
	+
	+struct mlx5_aso_wqe mlx5_aso_get_wqe(struct mlx5_aso aso);
	+void mlx5_aso_build_wqe(struct mlx5_aso *aso, u8 ds_cnt,
	+ struct mlx5_aso_wqe *aso_wqe,
	+ u32 obj_id, u32 opc_mode);
	+void mlx5_aso_post_wqe(struct mlx5_aso *aso, bool with_data,
	+ struct mlx5_wqe_ctrl_seg *doorbell_cseg);
	+int mlx5_aso_poll_cq(struct mlx5_aso *aso, bool with_data);
	+
	+struct mlx5_aso mlx5_aso_create(struct mlx5_core_dev mdev, u32 pdn);
	+void mlx5_aso_destroy(struct mlx5_aso *aso);
	+#endif /* __MLX5_LIB_ASO_H__ */
	diff --git a/sys/dev/mlx5/mlx5_lib/mlx5_aso.c b/sys/dev/mlx5/mlx5_lib/mlx5_aso.c
	new file mode 100644
	--- /dev/null
	+++ b/sys/dev/mlx5/mlx5_lib/mlx5_aso.c
	@@ -0,0 +1,428 @@
	+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
	+// Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
	+
	+#include <linux/printk.h>
	+#include <dev/mlx5/driver.h>
	+#include <dev/mlx5/mlx5_core/transobj.h>
	+#include "aso.h"
	+#include <dev/mlx5/mlx5_core/wq.h>
	+#include <dev/mlx5/cq.h>
	+
	+struct mlx5_aso_cq {
	+ /* data path - accessed per cqe */
	+ struct mlx5_cqwq wq;
	+
	+ /* data path - accessed per napi poll */
	+ struct mlx5_core_cq mcq;
	+
	+ /* control */
	+ struct mlx5_core_dev *mdev;
	+ struct mlx5_wq_ctrl wq_ctrl;
	+} ____cacheline_aligned_in_smp;
	+
	+struct mlx5_aso {
	+ /* data path */
	+ u16 cc;
	+ u16 pc;
	+
	+ struct mlx5_wqe_ctrl_seg *doorbell_cseg;
	+ struct mlx5_aso_cq cq;
	+
	+ /* read only */
	+ struct mlx5_wq_cyc wq;
	+ void __iomem *uar_map;
	+ u32 sqn;
	+
	+ /* control path */
	+ struct mlx5_wq_ctrl wq_ctrl;
	+
	+} ____cacheline_aligned_in_smp;
	+
	+static void mlx5_aso_free_cq(struct mlx5_aso_cq *cq)
	+{
	+ mlx5_wq_destroy(&cq->wq_ctrl);
	+}
	+
	+static int mlx5_aso_alloc_cq(struct mlx5_core_dev *mdev, int numa_node,
	+ void cqc_data, struct mlx5_aso_cq cq)
	+{
	+ struct mlx5_core_cq *mcq = &cq->mcq;
	+ struct mlx5_wq_param param;
	+ int err;
	+ u32 i;
	+
	+ param.linear = 1;
	+ err = mlx5_cqwq_create(mdev, &param, cqc_data, &cq->wq, &cq->wq_ctrl);
	+ if (err)
	+ return err;
	+
	+ mcq->cqe_sz = 64;
	+ mcq->set_ci_db = cq->wq_ctrl.db.db;
	+ mcq->arm_db = cq->wq_ctrl.db.db + 1;
	+
	+ for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) {
	+ struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i);
	+
	+ cqe->op_own = 0xf1;
	+ }
	+
	+ cq->mdev = mdev;
	+
	+ return 0;
	+}
	+
	+static int create_aso_cq(struct mlx5_aso_cq cq, void cqc_data)
	+{
	+ u32 out[MLX5_ST_SZ_DW(create_cq_out)];
	+ struct mlx5_core_dev *mdev = cq->mdev;
	+ struct mlx5_core_cq *mcq = &cq->mcq;
	+ int inlen, eqn, irqn_not_used;
	+ void in, cqc;
	+ int err;
	+
	+ err = mlx5_vector2eqn(mdev, 0, &eqn, &irqn_not_used);
	+ if (err)
	+ return err;
	+
	+ inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
	+ sizeof(u64) * cq->wq_ctrl.buf.npages;
	+ in = kvzalloc(inlen, GFP_KERNEL);
	+ if (!in)
	+ return -ENOMEM;
	+
	+ cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
	+
	+ memcpy(cqc, cqc_data, MLX5_ST_SZ_BYTES(cqc));
	+
	+ mlx5_fill_page_array(&cq->wq_ctrl.buf, (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas));
	+
	+ MLX5_SET(cqc, cqc, cq_period_mode, 0);
	+ MLX5_SET(cqc, cqc, c_eqn, eqn);
	+ MLX5_SET(cqc, cqc, uar_page, mdev->priv.uar->index);
	+ MLX5_SET(cqc, cqc, log_page_size, cq->wq_ctrl.buf.page_shift -
	+ MLX5_ADAPTER_PAGE_SHIFT);
	+ MLX5_SET64(cqc, cqc, dbr_addr, cq->wq_ctrl.db.dma);
	+
	+ err = mlx5_core_create_cq(mdev, mcq, in, inlen, out, sizeof(out));
	+
	+ kvfree(in);
	+
	+ return err;
	+}
	+
	+static void mlx5_aso_destroy_cq(struct mlx5_aso_cq *cq)
	+{
	+ mlx5_core_destroy_cq(cq->mdev, &cq->mcq);
	+ mlx5_wq_destroy(&cq->wq_ctrl);
	+}
	+
	+static int mlx5_aso_create_cq(struct mlx5_core_dev *mdev, int numa_node,
	+ struct mlx5_aso_cq *cq)
	+{
	+ void *cqc_data;
	+ int err;
	+
	+ cqc_data = kvzalloc(MLX5_ST_SZ_BYTES(cqc), GFP_KERNEL);
	+ if (!cqc_data)
	+ return -ENOMEM;
	+
	+ MLX5_SET(cqc, cqc_data, log_cq_size, 1);
	+ MLX5_SET(cqc, cqc_data, uar_page, mdev->priv.uar->index);
	+ if (MLX5_CAP_GEN(mdev, cqe_128_always) && cache_line_size() >= 128)
	+ MLX5_SET(cqc, cqc_data, cqe_sz, CQE_STRIDE_128_PAD);
	+
	+ err = mlx5_aso_alloc_cq(mdev, numa_node, cqc_data, cq);
	+ if (err) {
	+ mlx5_core_err(mdev, "Failed to alloc aso wq cq, err=%d\n", err);
	+ goto err_out;
	+ }
	+
	+ err = create_aso_cq(cq, cqc_data);
	+ if (err) {
	+ mlx5_core_err(mdev, "Failed to create aso wq cq, err=%d\n", err);
	+ goto err_free_cq;
	+ }
	+
	+ kvfree(cqc_data);
	+ return 0;
	+
	+err_free_cq:
	+ mlx5_aso_free_cq(cq);
	+err_out:
	+ kvfree(cqc_data);
	+ return err;
	+}
	+
	+static int mlx5_aso_alloc_sq(struct mlx5_core_dev *mdev, int numa_node,
	+ void sqc_data, struct mlx5_aso sq)
	+{
	+ void *sqc_wq = MLX5_ADDR_OF(sqc, sqc_data, wq);
	+ struct mlx5_wq_cyc *wq = &sq->wq;
	+ struct mlx5_wq_param param;
	+ int err;
	+
	+ sq->uar_map = mdev->priv.uar->map;
	+
	+ param.linear = 1;
	+ err = mlx5_wq_cyc_create(mdev, &param, sqc_wq, wq, &sq->wq_ctrl);
	+ if (err)
	+ return err;
	+
	+ wq->db = &wq->db[MLX5_SND_DBR];
	+
	+ return 0;
	+}
	+
	+static int create_aso_sq(struct mlx5_core_dev *mdev, int pdn,
	+ void sqc_data, struct mlx5_aso sq)
	+{
	+ void in, sqc, *wq;
	+ int inlen, err;
	+ u8 ts_format;
	+
	+ inlen = MLX5_ST_SZ_BYTES(create_sq_in) +
	+ sizeof(u64) * sq->wq_ctrl.buf.npages;
	+ in = kvzalloc(inlen, GFP_KERNEL);
	+ if (!in)
	+ return -ENOMEM;
	+
	+ sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
	+ wq = MLX5_ADDR_OF(sqc, sqc, wq);
	+
	+ memcpy(sqc, sqc_data, MLX5_ST_SZ_BYTES(sqc));
	+ MLX5_SET(sqc, sqc, cqn, sq->cq.mcq.cqn);
	+
	+ MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST);
	+ MLX5_SET(sqc, sqc, flush_in_error_en, 1);
	+
	+ ts_format = mlx5_get_sq_default_ts(mdev);
	+ MLX5_SET(sqc, sqc, ts_format, ts_format);
	+
	+ MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
	+ MLX5_SET(wq, wq, uar_page, mdev->priv.uar->index);
	+ MLX5_SET(wq, wq, log_wq_pg_sz, sq->wq_ctrl.buf.page_shift -
	+ MLX5_ADAPTER_PAGE_SHIFT);
	+ MLX5_SET64(wq, wq, dbr_addr, sq->wq_ctrl.db.dma);
	+
	+ mlx5_fill_page_array(&sq->wq_ctrl.buf, (__be64 *)MLX5_ADDR_OF(wq, wq, pas));
	+
	+ err = mlx5_core_create_sq(mdev, in, inlen, &sq->sqn);
	+
	+ kvfree(in);
	+
	+ return err;
	+}
	+
	+static int mlx5_aso_set_sq_rdy(struct mlx5_core_dev *mdev, u32 sqn)
	+{
	+ void in, sqc;
	+ int inlen, err;
	+
	+ inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
	+ in = kvzalloc(inlen, GFP_KERNEL);
	+ if (!in)
	+ return -ENOMEM;
	+
	+ MLX5_SET(modify_sq_in, in, sq_state, MLX5_SQC_STATE_RST);
	+ MLX5_SET(modify_sq_in, in, sqn, sqn);
	+ sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
	+ MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RDY);
	+
	+ err = mlx5_core_modify_sq(mdev, in, inlen);
	+
	+ kvfree(in);
	+
	+ return err;
	+}
	+
	+static int mlx5_aso_create_sq_rdy(struct mlx5_core_dev *mdev, u32 pdn,
	+ void sqc_data, struct mlx5_aso sq)
	+{
	+ int err;
	+
	+ err = create_aso_sq(mdev, pdn, sqc_data, sq);
	+ if (err)
	+ return err;
	+
	+ err = mlx5_aso_set_sq_rdy(mdev, sq->sqn);
	+ if (err)
	+ mlx5_core_destroy_sq(mdev, sq->sqn);
	+
	+ return err;
	+}
	+
	+static void mlx5_aso_free_sq(struct mlx5_aso *sq)
	+{
	+ mlx5_wq_destroy(&sq->wq_ctrl);
	+}
	+
	+static void mlx5_aso_destroy_sq(struct mlx5_aso *sq)
	+{
	+ mlx5_core_destroy_sq(sq->cq.mdev, sq->sqn);
	+ mlx5_aso_free_sq(sq);
	+}
	+
	+static int mlx5_aso_create_sq(struct mlx5_core_dev *mdev, int numa_node,
	+ u32 pdn, struct mlx5_aso *sq)
	+{
	+ void sqc_data, wq;
	+ int err;
	+
	+ sqc_data = kvzalloc(MLX5_ST_SZ_BYTES(sqc), GFP_KERNEL);
	+ if (!sqc_data)
	+ return -ENOMEM;
	+
	+ wq = MLX5_ADDR_OF(sqc, sqc_data, wq);
	+ MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
	+ MLX5_SET(wq, wq, pd, pdn);
	+ MLX5_SET(wq, wq, log_wq_sz, 1);
	+
	+ err = mlx5_aso_alloc_sq(mdev, numa_node, sqc_data, sq);
	+ if (err) {
	+ mlx5_core_err(mdev, "Failed to alloc aso wq sq, err=%d\n", err);
	+ goto err_out;
	+ }
	+
	+ err = mlx5_aso_create_sq_rdy(mdev, pdn, sqc_data, sq);
	+ if (err) {
	+ mlx5_core_err(mdev, "Failed to open aso wq sq, err=%d\n", err);
	+ goto err_free_asosq;
	+ }
	+
	+ mlx5_core_dbg(mdev, "aso sq->sqn = 0x%x\n", sq->sqn);
	+
	+ kvfree(sqc_data);
	+ return 0;
	+
	+err_free_asosq:
	+ mlx5_aso_free_sq(sq);
	+err_out:
	+ kvfree(sqc_data);
	+ return err;
	+}
	+
	+struct mlx5_aso mlx5_aso_create(struct mlx5_core_dev mdev, u32 pdn)
	+{
	+ int numa_node = dev_to_node(&mdev->pdev->dev);
	+ struct mlx5_aso *aso;
	+ int err;
	+
	+ aso = kzalloc(sizeof(*aso), GFP_KERNEL);
	+ if (!aso)
	+ return ERR_PTR(-ENOMEM);
	+
	+ err = mlx5_aso_create_cq(mdev, numa_node, &aso->cq);
	+ if (err)
	+ goto err_cq;
	+
	+ err = mlx5_aso_create_sq(mdev, numa_node, pdn, aso);
	+ if (err)
	+ goto err_sq;
	+
	+ return aso;
	+
	+err_sq:
	+ mlx5_aso_destroy_cq(&aso->cq);
	+err_cq:
	+ kfree(aso);
	+ return ERR_PTR(err);
	+}
	+
	+void mlx5_aso_destroy(struct mlx5_aso *aso)
	+{
	+ mlx5_aso_destroy_sq(aso);
	+ mlx5_aso_destroy_cq(&aso->cq);
	+ kfree(aso);
	+}
	+
	+void mlx5_aso_build_wqe(struct mlx5_aso *aso, u8 ds_cnt,
	+ struct mlx5_aso_wqe *aso_wqe,
	+ u32 obj_id, u32 opc_mode)
	+{
	+ struct mlx5_wqe_ctrl_seg *cseg = &aso_wqe->ctrl;
	+
	+ cseg->opmod_idx_opcode = cpu_to_be32((opc_mode << MLX5_WQE_CTRL_WQE_OPC_MOD_SHIFT) \|
	+ (aso->pc << MLX5_WQE_CTRL_WQE_INDEX_SHIFT) \|
	+ MLX5_OPCODE_ACCESS_ASO);
	+ cseg->qpn_ds = cpu_to_be32((aso->sqn << MLX5_WQE_CTRL_QPN_SHIFT) \| ds_cnt);
	+ cseg->fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
	+ cseg->general_id = cpu_to_be32(obj_id);
	+}
	+
	+struct mlx5_aso_wqe mlx5_aso_get_wqe(struct mlx5_aso aso)
	+{
	+ struct mlx5_aso_wqe *wqe;
	+ u16 pi;
	+
	+ pi = mlx5_wq_cyc_ctr2ix(&aso->wq, aso->pc);
	+ wqe = mlx5_wq_cyc_get_wqe(&aso->wq, pi);
	+ memset(wqe, 0, sizeof(*wqe));
	+ return wqe;
	+}
	+
	+void mlx5_aso_post_wqe(struct mlx5_aso *aso, bool with_data,
	+ struct mlx5_wqe_ctrl_seg *doorbell_cseg)
	+{
	+ doorbell_cseg->fm_ce_se \|= MLX5_WQE_CTRL_CQ_UPDATE;
	+ /* ensure wqe is visible to device before updating doorbell record */
	+ wmb();
	+
	+ if (with_data)
	+ aso->pc += MLX5_ASO_WQEBBS_DATA;
	+ else
	+ aso->pc += MLX5_ASO_WQEBBS;
	+ *aso->wq.db = cpu_to_be32(aso->pc);
	+
	+ /* ensure doorbell record is visible to device before ringing the
	+ * doorbell
	+ */
	+ wmb();
	+
	+ mlx5_write64((__be32 *)doorbell_cseg, aso->uar_map + MLX5_BF_OFFSET, NULL);
	+
	+ /* Ensure doorbell is written on uar_page before poll_cq */
	+ WRITE_ONCE(doorbell_cseg, NULL);
	+}
	+
	+int mlx5_aso_poll_cq(struct mlx5_aso *aso, bool with_data)
	+{
	+ struct mlx5_aso_cq *cq = &aso->cq;
	+ struct mlx5_cqe64 *cqe;
	+
	+ cqe = mlx5_cqwq_get_cqe(&cq->wq);
	+ if (!cqe)
	+ return -ETIMEDOUT;
	+
	+ /* sq->cc must be updated only after mlx5_cqwq_update_db_record(),
	+ * otherwise a cq overrun may occur
	+ */
	+ mlx5_cqwq_pop(&cq->wq);
	+
	+ if (unlikely(get_cqe_opcode(cqe) != MLX5_CQE_REQ)) {
	+ struct mlx5_err_cqe *err_cqe;
	+
	+ mlx5_core_err(cq->mdev, "Bad OP in ASOSQ CQE: 0x%x\n",
	+ get_cqe_opcode(cqe));
	+
	+ err_cqe = (struct mlx5_err_cqe *)cqe;
	+ mlx5_core_err(cq->mdev, "vendor_err_synd=%x\n",
	+ err_cqe->vendor_err_synd);
	+ mlx5_core_err(cq->mdev, "syndrome=%x\n",
	+ err_cqe->syndrome);
	+ print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET,
	+ 16, 1, err_cqe,
	+ sizeof(*err_cqe), false);
	+ }
	+
	+ mlx5_cqwq_update_db_record(&cq->wq);
	+
	+ /* ensure cq space is freed before enabling more cqes */
	+ wmb();
	+
	+ if (with_data)
	+ aso->cc += MLX5_ASO_WQEBBS_DATA;
	+ else
	+ aso->cc += MLX5_ASO_WQEBBS;
	+
	+ return 0;
	+}
	diff --git a/sys/dev/mlx5/qp.h b/sys/dev/mlx5/qp.h
	--- a/sys/dev/mlx5/qp.h
	+++ b/sys/dev/mlx5/qp.h
	@@ -146,6 +146,8 @@
	#define MLX5_SEND_WQE_DS 16
	#define MLX5_SEND_WQE_BB 64
	#define MLX5_SEND_WQEBB_NUM_DS (MLX5_SEND_WQE_BB / MLX5_SEND_WQE_DS)
	+#define MLX5_WQE_CTRL_QPN_SHIFT 8
	+#define MLX5_WQE_CTRL_WQE_INDEX_SHIFT 8

	enum {
	MLX5_SEND_WQE_MAX_WQEBBS = 16,
	@@ -194,7 +196,10 @@
	u8 signature;
	u8 rsvd[2];
	u8 fm_ce_se;
	- __be32 imm;
	+ union {
	+ __be32 imm;
	+ __be32 general_id;
	+ };
	};

	#define MLX5_WQE_CTRL_DS_MASK 0x3f
	@@ -228,6 +233,10 @@
	MLX5_ETH_WQE_SWP_OUTER_L4_TYPE = 1 << 5,
	};

	+enum {
	+ MLX5_ETH_WQE_FT_META_IPSEC = BIT(0),
	+};
	+
	struct mlx5_wqe_eth_seg {
	u8 swp_outer_l4_offset;
	u8 swp_outer_l3_offset;
	@@ -236,7 +245,7 @@
	u8 cs_flags;
	u8 swp_flags;
	__be16 mss;
	- __be32 rsvd2;
	+ __be32 flow_table_metadata;
	union {
	struct {
	__be16 inline_hdr_sz;
	diff --git a/sys/dev/mlx5/tls.h b/sys/dev/mlx5/tls.h
	--- a/sys/dev/mlx5/tls.h
	+++ b/sys/dev/mlx5/tls.h
	@@ -30,9 +30,6 @@

	struct mlx5_core_dev;

	-int mlx5_encryption_key_create(struct mlx5_core_dev *mdev, u32 pdn,
	- const void p_key, u32 key_len, u32 p_obj_id);
	-int mlx5_encryption_key_destroy(struct mlx5_core_dev *mdev, u32 oid);
	int mlx5_tls_open_tis(struct mlx5_core_dev mdev, int tc, int tdn, int pdn, u32 p_tisn);
	void mlx5_tls_close_tis(struct mlx5_core_dev *mdev, u32 tisn);
	int mlx5_tls_open_tir(struct mlx5_core_dev mdev, int tdn, int rqtn, u32 p_tirn);
	diff --git a/sys/modules/mlx5/Makefile b/sys/modules/mlx5/Makefile
	--- a/sys/modules/mlx5/Makefile
	+++ b/sys/modules/mlx5/Makefile
	@@ -1,21 +1,25 @@
	# $FreeBSD$
	.PATH: ${.CURDIR}/../../dev/mlx5/mlx5_core \
	${.CURDIR}/../../dev/mlx5/mlx5_lib \
	+ ${.CURDIR}/../../dev/mlx5/mlx5_accel

	KMOD=mlx5
	SRCS= \
	mlx5_alloc.c \
	+mlx5_aso.c \
	mlx5_cmd.c \
	+mlx5_crypto.c \
	mlx5_cq.c \
	mlx5_diag_cnt.c \
	mlx5_diagnostics.c \
	mlx5_eq.c \
	mlx5_eswitch.c \
	-mlx5_fc_cmd.c \
	+mlx5_fs_chains.c \
	mlx5_fs_cmd.c \
	-mlx5_fs_tcp.c \
	-mlx5_fs_tree.c \
	+mlx5_fs_core.c \
	mlx5_fs_counters.c \
	+mlx5_fs_ft_pool.c \
	+mlx5_fs_tcp.c \
	mlx5_fw.c \
	mlx5_fwdump.c \
	mlx5_health.c \
	@@ -37,6 +41,10 @@
	mlx5_vsc.c \
	mlx5_wq.c \
	mlx5_gid.c \
	+mlx5_ipsec_fs.c \
	+mlx5_ipsec_offload.c \
	+mlx5_ipsec.c \
	+mlx5_ipsec_rxtx.c \
	device_if.h bus_if.h vnode_if.h pci_if.h pci_iov_if.h \
	opt_inet.h opt_inet6.h opt_rss.h opt_ratelimit.h

File Metadata

Mime Type: text/plain
Expires: Sun, May 31, 12:44 AM (6 h, 43 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 33640704
Default Alt Text: D44220.id.diff (512 KB)

D44220.id.diffNo OneTemporaryActions

D44220.id.diffView Options

File Metadata

Event Timeline

D44220.id.diff
No OneTemporary
Actions

D44220.id.diff
View Options