Page MenuHomeFreeBSD
Files F158234531

D44220.id135396.diff
No OneTemporary
Actions

D44220.id135396.diff
View Options

This file is larger than 256 KB, so syntax highlighting was skipped.
diff --git a/sys/dev/mlx5/cq.h b/sys/dev/mlx5/cq.h
--- a/sys/dev/mlx5/cq.h
+++ b/sys/dev/mlx5/cq.h
@@ -113,6 +113,12 @@
} params;
};
+enum {
+ CQE_STRIDE_64 = 0,
+ CQE_STRIDE_128 = 1,
+ CQE_STRIDE_128_PAD = 2,
+};
+
static inline int cqe_sz_to_mlx_sz(u8 size)
{
return size == 64 ? CQE_SIZE_64 : CQE_SIZE_128;
diff --git a/sys/dev/mlx5/crypto.h b/sys/dev/mlx5/crypto.h
new file mode 100644
--- /dev/null
+++ b/sys/dev/mlx5/crypto.h
@@ -0,0 +1,37 @@
+/*-
+ * Copyright (c) 2023, NVIDIA Technologies. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef __MLX5_CRYPTO_H__
+#define __MLX5_CRYPTO_H__
+
+struct mlx5_core_dev;
+
+int mlx5_encryption_key_create(struct mlx5_core_dev *, u32 pdn, u32 key_type,
+ const void *p_key, u32 key_len, u32 *p_obj_id);
+int mlx5_encryption_key_destroy(struct mlx5_core_dev *mdev, u32 oid);
+
+#endif /* __MLX5_CRYPTO_H__ */
diff --git a/sys/dev/mlx5/device.h b/sys/dev/mlx5/device.h
--- a/sys/dev/mlx5/device.h
+++ b/sys/dev/mlx5/device.h
@@ -150,12 +150,14 @@
tmp; \
})
-#define MLX5_BY_PASS_NUM_REGULAR_PRIOS 8
-#define MLX5_BY_PASS_NUM_DONT_TRAP_PRIOS 8
+#define MLX5_RDMA_RX_NUM_COUNTERS_PRIOS 2
+#define MLX5_RDMA_TX_NUM_COUNTERS_PRIOS 1
+#define MLX5_BY_PASS_NUM_REGULAR_PRIOS 16
+#define MLX5_BY_PASS_NUM_DONT_TRAP_PRIOS 16
#define MLX5_BY_PASS_NUM_MULTICAST_PRIOS 1
#define MLX5_BY_PASS_NUM_PRIOS (MLX5_BY_PASS_NUM_REGULAR_PRIOS +\
- MLX5_BY_PASS_NUM_DONT_TRAP_PRIOS +\
- MLX5_BY_PASS_NUM_MULTICAST_PRIOS)
+ MLX5_BY_PASS_NUM_DONT_TRAP_PRIOS +\
+ MLX5_BY_PASS_NUM_MULTICAST_PRIOS)
/* insert a value to a struct */
#define MLX5_VSC_SET(typ, p, fld, v) do { \
@@ -393,6 +395,8 @@
MLX5_OPCODE_UMR = 0x25,
MLX5_OPCODE_QOS_REMAP = 0x2a,
+ MLX5_OPCODE_ACCESS_ASO = 0x2d,
+
MLX5_OPCODE_SIGNATURE_CANCELED = (1 << 15),
};
@@ -569,6 +573,11 @@
__be32 rsvd1[6];
};
+struct mlx5_eqe_obj_change {
+ u8 rsvd0[2];
+ __be16 obj_type;
+ __be32 obj_id;
+};
#define PORT_MODULE_EVENT_MODULE_STATUS_MASK 0xF
#define PORT_MODULE_EVENT_ERROR_TYPE_MASK 0xF
@@ -640,6 +649,7 @@
struct mlx5_eqe_dct dct;
struct mlx5_eqe_temp_warning temp_warning;
struct mlx5_eqe_xrq_err xrq_err;
+ struct mlx5_eqe_obj_change obj_change;
} __packed;
struct mlx5_eqe {
@@ -705,7 +715,12 @@
u8 l4_hdr_type_etc;
__be16 vlan_info;
__be32 srqn; /* [31:24]: lro_num_seg, [23:0]: srqn */
- __be32 imm_inval_pkey;
+ union {
+ __be32 immediate;
+ __be32 inval_rkey;
+ __be32 pkey;
+ __be32 ft_metadata;
+ };
u8 rsvd40[4];
__be32 byte_cnt;
__be64 timestamp;
@@ -921,6 +936,7 @@
MLX5_MATCH_OUTER_HEADERS = 1 << 0,
MLX5_MATCH_MISC_PARAMETERS = 1 << 1,
MLX5_MATCH_INNER_HEADERS = 1 << 2,
+ MLX5_MATCH_MISC_PARAMETERS_2 = 1 << 3,
};
@@ -990,12 +1006,19 @@
MLX5_CAP_VECTOR_CALC,
MLX5_CAP_QOS,
MLX5_CAP_DEBUG,
- MLX5_CAP_NVME,
- MLX5_CAP_DMC,
- MLX5_CAP_DEC,
+ MLX5_CAP_RESERVED_14,
+ MLX5_CAP_DEV_MEM,
+ MLX5_CAP_RESERVED_16,
MLX5_CAP_TLS,
+ MLX5_CAP_VDPA_EMULATION = 0x13,
MLX5_CAP_DEV_EVENT = 0x14,
+ MLX5_CAP_IPSEC,
+ MLX5_CAP_CRYPTO = 0x1a,
+ MLX5_CAP_DEV_SHAMPO = 0x1d,
+ MLX5_CAP_MACSEC = 0x1f,
MLX5_CAP_GENERAL_2 = 0x20,
+ MLX5_CAP_PORT_SELECTION = 0x25,
+ MLX5_CAP_ADV_VIRTUALIZATION = 0x26,
/* NUM OF CAP Types */
MLX5_CAP_NUM
};
@@ -1060,6 +1083,9 @@
#define MLX5_CAP_FLOWTABLE(mdev, cap) \
MLX5_GET(flow_table_nic_cap, mdev->hca_caps_cur[MLX5_CAP_FLOW_TABLE], cap)
+#define MLX5_CAP64_FLOWTABLE(mdev, cap) \
+ MLX5_GET64(flow_table_nic_cap, (mdev)->hca_caps_cur[MLX5_CAP_FLOW_TABLE], cap)
+
#define MLX5_CAP_FLOWTABLE_MAX(mdev, cap) \
MLX5_GET(flow_table_nic_cap, mdev->hca_caps_max[MLX5_CAP_FLOW_TABLE], cap)
@@ -1069,6 +1095,54 @@
#define MLX5_CAP_FLOWTABLE_NIC_RX_MAX(mdev, cap) \
MLX5_CAP_FLOWTABLE_MAX(mdev, flow_table_properties_nic_receive.cap)
+#define MLX5_CAP_FLOWTABLE_NIC_TX(mdev, cap) \
+ MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_transmit.cap)
+
+#define MLX5_CAP_FLOWTABLE_NIC_TX_MAX(mdev, cap) \
+ MLX5_CAP_FLOWTABLE_MAX(mdev, flow_table_properties_nic_transmit.cap)
+
+#define MLX5_CAP_FLOWTABLE_SNIFFER_RX(mdev, cap) \
+ MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_receive_sniffer.cap)
+
+#define MLX5_CAP_FLOWTABLE_SNIFFER_RX_MAX(mdev, cap) \
+ MLX5_CAP_FLOWTABLE_MAX(mdev, flow_table_properties_nic_receive_sniffer.cap)
+
+#define MLX5_CAP_FLOWTABLE_SNIFFER_TX(mdev, cap) \
+ MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_transmit_sniffer.cap)
+
+#define MLX5_CAP_FLOWTABLE_SNIFFER_TX_MAX(mdev, cap) \
+ MLX5_CAP_FLOWTABLE_MAX(mdev, flow_table_properties_nic_transmit_sniffer.cap)
+
+#define MLX5_CAP_FLOWTABLE_RDMA_RX(mdev, cap) \
+ MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_receive_rdma.cap)
+
+#define MLX5_CAP_FLOWTABLE_RDMA_RX_MAX(mdev, cap) \
+ MLX5_CAP_FLOWTABLE_MAX(mdev, flow_table_properties_nic_receive_rdma.cap)
+
+#define MLX5_CAP_FLOWTABLE_RDMA_TX(mdev, cap) \
+ MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_transmit_rdma.cap)
+
+#define MLX5_CAP_FLOWTABLE_RDMA_TX_MAX(mdev, cap) \
+ MLX5_CAP_FLOWTABLE_MAX(mdev, flow_table_properties_nic_transmit_rdma.cap)
+
+#define MLX5_CAP_FLOWTABLE_NIC_TX(mdev, cap) \
+ MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_transmit.cap)
+
+#define MLX5_CAP_FLOWTABLE_NIC_TX_MAX(mdev, cap) \
+ MLX5_CAP_FLOWTABLE_MAX(mdev, flow_table_properties_nic_transmit.cap)
+
+#define MLX5_CAP_FLOWTABLE_RDMA_RX(mdev, cap) \
+ MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_receive_rdma.cap)
+
+#define MLX5_CAP_FLOWTABLE_RDMA_RX_MAX(mdev, cap) \
+ MLX5_CAP_FLOWTABLE_MAX(mdev, flow_table_properties_nic_receive_rdma.cap)
+
+#define MLX5_CAP_FLOWTABLE_RDMA_TX(mdev, cap) \
+ MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_transmit_rdma.cap)
+
+#define MLX5_CAP_FLOWTABLE_RDMA_TX_MAX(mdev, cap) \
+ MLX5_CAP_FLOWTABLE_MAX(mdev, flow_table_properties_nic_transmit_rdma.cap)
+
#define MLX5_CAP_ESW_FLOWTABLE(mdev, cap) \
MLX5_GET(flow_table_eswitch_cap, \
mdev->hca_caps_cur[MLX5_CAP_ESWITCH_FLOW_TABLE], cap)
@@ -1095,14 +1169,46 @@
#define MLX5_CAP_ESW_INGRESS_ACL_MAX(mdev, cap) \
MLX5_CAP_ESW_FLOWTABLE_MAX(mdev, flow_table_properties_esw_acl_ingress.cap)
+#define MLX5_CAP_ESW_FT_FIELD_SUPPORT_2(mdev, cap) \
+ MLX5_CAP_ESW_FLOWTABLE(mdev, ft_field_support_2_esw_fdb.cap)
+
+#define MLX5_CAP_ESW_FT_FIELD_SUPPORT_2_MAX(mdev, cap) \
+ MLX5_CAP_ESW_FLOWTABLE_MAX(mdev, ft_field_support_2_esw_fdb.cap)
+
#define MLX5_CAP_ESW(mdev, cap) \
MLX5_GET(e_switch_cap, \
mdev->hca_caps_cur[MLX5_CAP_ESWITCH], cap)
+#define MLX5_CAP64_ESW_FLOWTABLE(mdev, cap) \
+ MLX5_GET64(flow_table_eswitch_cap, \
+ (mdev)->hca_caps_cur[MLX5_CAP_ESWITCH_FLOW_TABLE], cap)
+
#define MLX5_CAP_ESW_MAX(mdev, cap) \
MLX5_GET(e_switch_cap, \
mdev->hca_caps_max[MLX5_CAP_ESWITCH], cap)
+#define MLX5_CAP_PORT_SELECTION(mdev, cap) \
+ MLX5_GET(port_selection_cap, \
+ mdev->hca_caps_cur[MLX5_CAP_PORT_SELECTION], cap)
+
+#define MLX5_CAP_PORT_SELECTION_MAX(mdev, cap) \
+ MLX5_GET(port_selection_cap, \
+ mdev->hca_caps_max[MLX5_CAP_PORT_SELECTION], cap)
+
+#define MLX5_CAP_ADV_VIRTUALIZATION(mdev, cap) \
+ MLX5_GET(adv_virtualization_cap, \
+ mdev->hca_caps_cur[MLX5_CAP_ADV_VIRTUALIZATION], cap)
+
+#define MLX5_CAP_ADV_VIRTUALIZATION_MAX(mdev, cap) \
+ MLX5_GET(adv_virtualization_cap, \
+ mdev->hca_caps_max[MLX5_CAP_ADV_VIRTUALIZATION], cap)
+
+#define MLX5_CAP_FLOWTABLE_PORT_SELECTION(mdev, cap) \
+ MLX5_CAP_PORT_SELECTION(mdev, flow_table_properties_port_selection.cap)
+
+#define MLX5_CAP_FLOWTABLE_PORT_SELECTION_MAX(mdev, cap) \
+ MLX5_CAP_PORT_SELECTION_MAX(mdev, flow_table_properties_port_selection.cap)
+
#define MLX5_CAP_ODP(mdev, cap)\
MLX5_GET(odp_cap, mdev->hca_caps_cur[MLX5_CAP_ODP], cap)
@@ -1171,6 +1277,9 @@
#define MLX5_CAP_DEV_EVENT(mdev, cap)\
MLX5_ADDR_OF(device_event_cap, (mdev)->hca_caps_cur[MLX5_CAP_DEV_EVENT], cap)
+#define MLX5_CAP_IPSEC(mdev, cap) \
+ MLX5_GET(ipsec_cap, (mdev)->hca_caps_cur[MLX5_CAP_IPSEC], cap)
+
enum {
MLX5_CMD_STAT_OK = 0x0,
MLX5_CMD_STAT_INT_ERR = 0x1,
diff --git a/sys/dev/mlx5/doorbell.h b/sys/dev/mlx5/doorbell.h
--- a/sys/dev/mlx5/doorbell.h
+++ b/sys/dev/mlx5/doorbell.h
@@ -63,10 +63,12 @@
{
unsigned long flags;
- spin_lock_irqsave(doorbell_lock, flags);
+ if (doorbell_lock)
+ spin_lock_irqsave(doorbell_lock, flags);
__raw_writel((__force u32) val[0], dest);
__raw_writel((__force u32) val[1], dest + 4);
- spin_unlock_irqrestore(doorbell_lock, flags);
+ if (doorbell_lock)
+ spin_unlock_irqrestore(doorbell_lock, flags);
}
#endif
diff --git a/sys/dev/mlx5/driver.h b/sys/dev/mlx5/driver.h
--- a/sys/dev/mlx5/driver.h
+++ b/sys/dev/mlx5/driver.h
@@ -635,11 +635,13 @@
struct mlx5_pme_stats pme_stats;
+ struct mlx5_flow_steering *steering;
struct mlx5_eswitch *eswitch;
struct mlx5_bfreg_data bfregs;
struct mlx5_uars_page *uar;
struct mlx5_fc_stats fc_stats;
+ struct mlx5_ft_pool *ft_pool;
};
enum mlx5_device_state {
@@ -728,6 +730,10 @@
struct mlx5_flow_root_namespace *esw_ingress_root_ns;
struct mlx5_flow_root_namespace *sniffer_rx_root_ns;
struct mlx5_flow_root_namespace *sniffer_tx_root_ns;
+ struct mlx5_flow_root_namespace *nic_tx_root_ns;
+ struct mlx5_flow_root_namespace *rdma_tx_root_ns;
+ struct mlx5_flow_root_namespace *rdma_rx_root_ns;
+
u32 num_q_counter_allocated[MLX5_INTERFACE_NUMBER];
struct mlx5_crspace_regmap *dump_rege;
uint32_t *dump_data;
@@ -756,6 +762,7 @@
#ifdef CONFIG_MLX5_FPGA
struct mlx5_fpga_device *fpga;
#endif
+ struct xarray ipsec_sadb;
};
enum {
diff --git a/sys/dev/mlx5/fs.h b/sys/dev/mlx5/fs.h
--- a/sys/dev/mlx5/fs.h
+++ b/sys/dev/mlx5/fs.h
@@ -35,8 +35,33 @@
#include <dev/mlx5/device.h>
#include <dev/mlx5/driver.h>
+enum mlx5_flow_destination_type {
+ MLX5_FLOW_DESTINATION_TYPE_NONE,
+ MLX5_FLOW_DESTINATION_TYPE_VPORT,
+ MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE,
+ MLX5_FLOW_DESTINATION_TYPE_TIR,
+ MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER,
+ MLX5_FLOW_DESTINATION_TYPE_UPLINK,
+ MLX5_FLOW_DESTINATION_TYPE_PORT,
+ MLX5_FLOW_DESTINATION_TYPE_COUNTER,
+ MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM,
+ MLX5_FLOW_DESTINATION_TYPE_RANGE,
+ MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE,
+};
+
+enum {
+ MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO = 1 << 16,
+ MLX5_FLOW_CONTEXT_ACTION_ENCRYPT = 1 << 17,
+ MLX5_FLOW_CONTEXT_ACTION_DECRYPT = 1 << 18,
+ MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS = 1 << 19,
+};
+
enum {
- MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO = 1 << 16,
+ MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT = BIT(0),
+ MLX5_FLOW_TABLE_TUNNEL_EN_DECAP = BIT(1),
+ MLX5_FLOW_TABLE_TERMINATION = BIT(2),
+ MLX5_FLOW_TABLE_UNMANAGED = BIT(3),
+ MLX5_FLOW_TABLE_OTHER_VPORT = BIT(4),
};
/*Flow tag*/
@@ -63,54 +88,149 @@
enum mlx5_flow_namespace_type {
MLX5_FLOW_NAMESPACE_BYPASS,
+ MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC,
+ MLX5_FLOW_NAMESPACE_LAG,
MLX5_FLOW_NAMESPACE_OFFLOADS,
+ MLX5_FLOW_NAMESPACE_ETHTOOL,
MLX5_FLOW_NAMESPACE_KERNEL,
MLX5_FLOW_NAMESPACE_LEFTOVERS,
- MLX5_FLOW_NAMESPACE_SNIFFER_RX,
- MLX5_FLOW_NAMESPACE_SNIFFER_TX,
+ MLX5_FLOW_NAMESPACE_ANCHOR,
+ MLX5_FLOW_NAMESPACE_FDB_BYPASS,
MLX5_FLOW_NAMESPACE_FDB,
MLX5_FLOW_NAMESPACE_ESW_EGRESS,
MLX5_FLOW_NAMESPACE_ESW_INGRESS,
+ MLX5_FLOW_NAMESPACE_SNIFFER_RX,
+ MLX5_FLOW_NAMESPACE_SNIFFER_TX,
+ MLX5_FLOW_NAMESPACE_EGRESS,
+ MLX5_FLOW_NAMESPACE_EGRESS_IPSEC,
+ MLX5_FLOW_NAMESPACE_EGRESS_MACSEC,
+ MLX5_FLOW_NAMESPACE_RDMA_RX,
+ MLX5_FLOW_NAMESPACE_RDMA_RX_KERNEL,
+ MLX5_FLOW_NAMESPACE_RDMA_TX,
+ MLX5_FLOW_NAMESPACE_PORT_SEL,
+ MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS,
+ MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS,
+ MLX5_FLOW_NAMESPACE_RDMA_RX_IPSEC,
+ MLX5_FLOW_NAMESPACE_RDMA_TX_IPSEC,
+};
+
+enum {
+ FDB_BYPASS_PATH,
+ FDB_TC_OFFLOAD,
+ FDB_FT_OFFLOAD,
+ FDB_TC_MISS,
+ FDB_BR_OFFLOAD,
+ FDB_SLOW_PATH,
+ FDB_PER_VPORT,
};
struct mlx5_flow_table;
struct mlx5_flow_group;
struct mlx5_flow_rule;
struct mlx5_flow_namespace;
+struct mlx5_flow_handle;
+
+enum {
+ FLOW_CONTEXT_HAS_TAG = BIT(0),
+};
+
+struct mlx5_flow_context {
+ u32 flags;
+ u32 flow_tag;
+ u32 flow_source;
+};
struct mlx5_flow_spec {
u8 match_criteria_enable;
u32 match_criteria[MLX5_ST_SZ_DW(fte_match_param)];
u32 match_value[MLX5_ST_SZ_DW(fte_match_param)];
+ struct mlx5_flow_context flow_context;
+};
+
+enum {
+ MLX5_FLOW_DEST_VPORT_VHCA_ID = BIT(0),
+ MLX5_FLOW_DEST_VPORT_REFORMAT_ID = BIT(1),
+};
+
+enum mlx5_flow_dest_range_field {
+ MLX5_FLOW_DEST_RANGE_FIELD_PKT_LEN = 0,
};
struct mlx5_flow_destination {
- u32 type;
+ enum mlx5_flow_destination_type type;
union {
- u32 tir_num;
- struct mlx5_flow_table *ft;
- u32 vport_num;
+ u32 tir_num;
+ u32 ft_num;
+ struct mlx5_flow_table *ft;
+ u32 counter_id;
+ struct {
+ u16 num;
+ u16 vhca_id;
+ struct mlx5_pkt_reformat *pkt_reformat;
+ u8 flags;
+ } vport;
+ struct {
+ struct mlx5_flow_table *hit_ft;
+ struct mlx5_flow_table *miss_ft;
+ enum mlx5_flow_dest_range_field field;
+ u32 min;
+ u32 max;
+ } range;
+ u32 sampler_id;
};
};
-enum mlx5_flow_act_actions {
- MLX5_FLOW_ACT_ACTIONS_FLOW_TAG = 1 << 0,
- MLX5_FLOW_ACT_ACTIONS_MODIFY_HDR = 1 << 1,
- MLX5_FLOW_ACT_ACTIONS_PACKET_REFORMAT = 1 << 2,
- MLX5_FLOW_ACT_ACTIONS_COUNT = 1 << 3,
+struct mlx5_exe_aso {
+ u32 object_id;
+ u8 type;
+ u8 return_reg_id;
+ union {
+ u32 ctrl_data;
+ struct {
+ u8 meter_idx;
+ u8 init_color;
+ } flow_meter;
+ };
+};
+
+enum {
+ FLOW_ACT_NO_APPEND = BIT(0),
+ FLOW_ACT_IGNORE_FLOW_LEVEL = BIT(1),
+};
+
+struct mlx5_fs_vlan {
+ u16 ethtype;
+ u16 vid;
+ u8 prio;
+};
+
+#define MLX5_FS_VLAN_DEPTH 2
+
+enum mlx5_flow_act_crypto_type {
+ MLX5_FLOW_ACT_CRYPTO_TYPE_IPSEC,
+};
+
+enum mlx5_flow_act_crypto_op {
+ MLX5_FLOW_ACT_CRYPTO_OP_ENCRYPT,
+ MLX5_FLOW_ACT_CRYPTO_OP_DECRYPT,
};
-enum MLX5_FLOW_ACT_FLAGS {
- MLX5_FLOW_ACT_NO_APPEND = 1 << 0,
+struct mlx5_flow_act_crypto_params {
+ u32 obj_id;
+ u8 type; /* see enum mlx5_flow_act_crypto_type */
+ u8 op; /* see enum mlx5_flow_act_crypto_op */
};
struct mlx5_flow_act {
- u32 actions; /* See enum mlx5_flow_act_actions */
- u32 flags;
- u32 flow_tag;
- struct mlx5_modify_hdr *modify_hdr;
+ u32 action;
+ struct mlx5_modify_hdr *modify_hdr;
struct mlx5_pkt_reformat *pkt_reformat;
- struct mlx5_fc *counter;
+ struct mlx5_flow_act_crypto_params crypto;
+ u32 flags;
+ struct mlx5_fs_vlan vlan[MLX5_FS_VLAN_DEPTH];
+ struct ib_counters *counters;
+ struct mlx5_flow_group *fg;
+ struct mlx5_exe_aso exe_aso;
};
#define FT_NAME_STR_SZ 20
@@ -137,6 +257,28 @@
size - 1);
}
+struct mlx5_flow_namespace *
+mlx5_get_flow_vport_acl_namespace(struct mlx5_core_dev *dev,
+ enum mlx5_flow_namespace_type type,
+ int vport);
+
+struct mlx5_flow_table_attr {
+ int prio;
+ int max_fte;
+ u32 level;
+ u32 flags;
+ u16 uid;
+ struct mlx5_flow_table *next_ft;
+
+ struct {
+ int max_num_groups;
+ int num_reserved_entries;
+ } autogroup;
+};
+
+struct mlx5_flow_namespace *
+mlx5_get_fdb_sub_ns(struct mlx5_core_dev *dev, int n);
+
struct mlx5_flow_namespace *
mlx5_get_flow_namespace(struct mlx5_core_dev *dev,
enum mlx5_flow_namespace_type type);
@@ -147,24 +289,19 @@
*/
struct mlx5_flow_table *
mlx5_create_auto_grouped_flow_table(struct mlx5_flow_namespace *ns,
- int prio,
- const char *name,
- int num_flow_table_entries,
- int max_num_groups,
- int num_reserved_entries);
+ struct mlx5_flow_table_attr *ft_attr);
struct mlx5_flow_table *
mlx5_create_vport_flow_table(struct mlx5_flow_namespace *ns,
- u16 vport,
- int prio,
- const char *name,
- int num_flow_table_entries);
+ struct mlx5_flow_table_attr *ft_attr, u16 vport);
+
+struct mlx5_flow_table *mlx5_create_lag_demux_flow_table(
+ struct mlx5_flow_namespace *ns,
+ int prio, u32 level);
struct mlx5_flow_table *
mlx5_create_flow_table(struct mlx5_flow_namespace *ns,
- int prio,
- const char *name,
- int num_flow_table_entries);
+ struct mlx5_flow_table_attr *ft_attr);
int mlx5_destroy_flow_table(struct mlx5_flow_table *ft);
/* inbox should be set with the following values:
@@ -177,18 +314,17 @@
mlx5_create_flow_group(struct mlx5_flow_table *ft, u32 *in);
void mlx5_destroy_flow_group(struct mlx5_flow_group *fg);
-/* Single destination per rule.
- * Group ID is implied by the match criteria.
- */
-struct mlx5_flow_rule *
-mlx5_add_flow_rule(struct mlx5_flow_table *ft,
- u8 match_criteria_enable,
- u32 *match_criteria,
- u32 *match_value,
- u32 sw_action,
- struct mlx5_flow_act *flow_act,
- struct mlx5_flow_destination *dest);
-void mlx5_del_flow_rule(struct mlx5_flow_rule **);
+struct mlx5_flow_handle *
+mlx5_add_flow_rules(struct mlx5_flow_table *ft,
+ const struct mlx5_flow_spec *spec,
+ struct mlx5_flow_act *flow_act,
+ struct mlx5_flow_destination *dest,
+ int num_dest);
+void mlx5_del_flow_rules(struct mlx5_flow_handle **pp);
+
+int mlx5_modify_rule_destination(struct mlx5_flow_handle *handler,
+ struct mlx5_flow_destination *new_dest,
+ struct mlx5_flow_destination *old_dest);
/*The following API is for sniffer*/
typedef int (*rule_event_fn)(struct mlx5_flow_rule *rule,
@@ -294,4 +430,8 @@
u64 *packets, u64 *bytes);
u32 mlx5_fc_id(struct mlx5_fc *counter);
/******* End of Flow counters API ******/
+
+u32 mlx5_flow_table_id(struct mlx5_flow_table *ft);
+int mlx5_fs_add_rx_underlay_qpn(struct mlx5_core_dev *dev, u32 underlay_qpn);
+int mlx5_fs_remove_rx_underlay_qpn(struct mlx5_core_dev *dev, u32 underlay_qpn);
#endif
diff --git a/sys/dev/mlx5/mlx5_accel/ipsec.h b/sys/dev/mlx5/mlx5_accel/ipsec.h
--- a/sys/dev/mlx5/mlx5_accel/ipsec.h
+++ b/sys/dev/mlx5/mlx5_accel/ipsec.h
@@ -1,33 +1,26 @@
/*-
- * Copyright (c) 2017 Mellanox Technologies. All rights reserved.
+ * Copyright (c) 2023 NVIDIA corporation & affiliates.
*
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
*
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
*
* $FreeBSD$
*/
@@ -35,105 +28,245 @@
#ifndef __MLX5_ACCEL_IPSEC_H__
#define __MLX5_ACCEL_IPSEC_H__
-#ifdef CONFIG_MLX5_ACCEL
-
+#include <sys/mbuf.h>
#include <dev/mlx5/driver.h>
+#include <dev/mlx5/qp.h>
+#include <dev/mlx5/mlx5_core/mlx5_core.h>
+#include <dev/mlx5/mlx5_en/en.h>
+#include <dev/mlx5/mlx5_lib/aso.h>
+
+#define MLX5E_IPSEC_SADB_RX_BITS 10
+#define MLX5_IPSEC_METADATA_MARKER(ipsec_metadata) ((ipsec_metadata >> 31) & 0x1)
+
+struct mlx5e_priv;
+struct mlx5e_tx_wqe;
+struct mlx5e_ipsec_tx;
+struct mlx5e_ipsec_rx;
+
+struct aes_gcm_keymat {
+ u64 seq_iv;
-enum {
- MLX5_ACCEL_IPSEC_DEVICE = BIT(1),
- MLX5_ACCEL_IPSEC_IPV6 = BIT(2),
- MLX5_ACCEL_IPSEC_ESP = BIT(3),
- MLX5_ACCEL_IPSEC_LSO = BIT(4),
+ u32 salt;
+ u32 icv_len;
+
+ u32 key_len;
+ u32 aes_key[256 / 32];
+};
+
+struct mlx5e_ipsec_work {
+ struct work_struct work;
+ struct mlx5e_ipsec_sa_entry *sa_entry;
+ void *data;
};
-#define MLX5_IPSEC_SADB_IP_AH BIT(7)
-#define MLX5_IPSEC_SADB_IP_ESP BIT(6)
-#define MLX5_IPSEC_SADB_SA_VALID BIT(5)
-#define MLX5_IPSEC_SADB_SPI_EN BIT(4)
-#define MLX5_IPSEC_SADB_DIR_SX BIT(3)
-#define MLX5_IPSEC_SADB_IPV6 BIT(2)
+struct mlx5e_ipsec_dwork {
+ struct delayed_work dwork;
+ struct mlx5e_ipsec_sa_entry *sa_entry;
+};
-enum {
- MLX5_IPSEC_CMD_ADD_SA = 0,
- MLX5_IPSEC_CMD_DEL_SA = 1,
+struct mlx5e_ipsec_aso {
+ u8 __aligned(64) ctx[MLX5_ST_SZ_BYTES(ipsec_aso)];
+ dma_addr_t dma_addr;
+ struct mlx5_aso *aso;
+ /* Protect ASO WQ access, as it is global to whole IPsec */
+ spinlock_t lock;
};
-enum mlx5_accel_ipsec_enc_mode {
- MLX5_IPSEC_SADB_MODE_NONE = 0,
- MLX5_IPSEC_SADB_MODE_AES_GCM_128_AUTH_128 = 1,
- MLX5_IPSEC_SADB_MODE_AES_GCM_256_AUTH_128 = 3,
+struct mlx5_replay_esn {
+ u32 replay_window;
+ u32 esn;
+ u32 esn_msb;
+ u8 overlap : 1;
+ u8 trigger : 1;
};
-#define MLX5_IPSEC_DEV(mdev) (mlx5_accel_ipsec_device_caps(mdev) & \
- MLX5_ACCEL_IPSEC_DEVICE)
+struct mlx5_accel_esp_xfrm_attrs {
+ u32 spi;
+ struct aes_gcm_keymat aes_gcm;
-struct mlx5_accel_ipsec_sa {
- __be32 cmd;
- u8 key_enc[32];
- u8 key_auth[32];
- __be32 sip[4];
- __be32 dip[4];
union {
- struct {
- __be32 reserved;
- u8 salt_iv[8];
- __be32 salt;
- } __packed gcm;
- struct {
- u8 salt[16];
- } __packed cbc;
- };
- __be32 spi;
- __be32 sw_sa_handle;
- __be16 tfclen;
- u8 enc_mode;
- u8 sip_masklen;
- u8 dip_masklen;
- u8 flags;
- u8 reserved[2];
-} __packed;
-
-/**
- * mlx5_accel_ipsec_sa_cmd_exec - Execute an IPSec SADB command
- * @mdev: mlx5 device
- * @cmd: command to execute
- * May be called from atomic context. Returns context pointer, or error
- * Caller must eventually call mlx5_accel_ipsec_sa_cmd_wait from non-atomic
- * context, to cleanup the context pointer
- */
-void *mlx5_accel_ipsec_sa_cmd_exec(struct mlx5_core_dev *mdev,
- struct mlx5_accel_ipsec_sa *cmd);
-
-/**
- * mlx5_accel_ipsec_sa_cmd_wait - Wait for command execution completion
- * @context: Context pointer returned from call to mlx5_accel_ipsec_sa_cmd_exec
- * Sleeps (killable) until command execution is complete.
- * Returns the command result, or -EINTR if killed
- */
-int mlx5_accel_ipsec_sa_cmd_wait(void *context);
+ __be32 a4;
+ __be32 a6[4];
+ } saddr;
-u32 mlx5_accel_ipsec_device_caps(struct mlx5_core_dev *mdev);
+ union {
+ __be32 a4;
+ __be32 a6[4];
+ } daddr;
+
+ u8 dir : 2;
+ u8 encap : 1;
+ u8 drop : 1;
+ u8 family;
+ struct mlx5_replay_esn replay_esn;
+ u32 authsize;
+ u32 reqid;
+ u16 sport;
+ u16 dport;
+};
+
+enum mlx5_ipsec_cap {
+ MLX5_IPSEC_CAP_CRYPTO = 1 << 0,
+ MLX5_IPSEC_CAP_ESN = 1 << 1,
+ MLX5_IPSEC_CAP_PACKET_OFFLOAD = 1 << 2,
+ MLX5_IPSEC_CAP_ROCE = 1 << 3,
+ MLX5_IPSEC_CAP_PRIO = 1 << 4,
+ MLX5_IPSEC_CAP_TUNNEL = 1 << 5,
+ MLX5_IPSEC_CAP_ESPINUDP = 1 << 6,
+};
+
+struct mlx5e_ipsec {
+ struct mlx5_core_dev *mdev;
+ struct workqueue_struct *wq;
+ struct mlx5e_ipsec_tx *tx;
+ struct mlx5e_ipsec_rx *rx_ipv4;
+ struct mlx5e_ipsec_rx *rx_ipv6;
+ struct mlx5e_ipsec_aso *aso;
+ u32 pdn;
+ u32 mkey;
+};
+
+struct mlx5e_ipsec_rule {
+ struct mlx5_flow_handle *rule;
+ struct mlx5_flow_handle *kspi_rule;
+ struct mlx5_flow_handle *reqid_rule;
+ struct mlx5_modify_hdr *modify_hdr;
+ struct mlx5_pkt_reformat *pkt_reformat;
+ struct mlx5_fc *fc;
+};
-unsigned int mlx5_accel_ipsec_counters_count(struct mlx5_core_dev *mdev);
-int mlx5_accel_ipsec_counters_read(struct mlx5_core_dev *mdev, u64 *counters,
- unsigned int count);
+struct mlx5e_ipsec_esn_state {
+ u32 esn;
+ u32 esn_msb;
+ u8 overlap: 1;
+};
-int mlx5_accel_ipsec_init(struct mlx5_core_dev *mdev);
-void mlx5_accel_ipsec_cleanup(struct mlx5_core_dev *mdev);
+struct mlx5e_ipsec_sa_entry {
+ struct secasvar *savp;
+ if_t ifp;
+ struct mlx5e_ipsec *ipsec;
+ struct mlx5_accel_esp_xfrm_attrs attrs;
+ struct mlx5e_ipsec_rule ipsec_rule;
+ struct mlx5e_ipsec_dwork *dwork;
+ struct mlx5e_ipsec_work *work;
+ u32 ipsec_obj_id;
+ u32 enc_key_id;
+ u16 kspi; /* Stack allocated unique SA identifier */
+ struct mlx5e_ipsec_esn_state esn_state;
+};
+
+struct upspec {
+ u16 dport;
+ u16 sport;
+ u8 proto;
+};
-#else
+struct mlx5_accel_pol_xfrm_attrs {
+ union {
+ __be32 a4;
+ __be32 a6[4];
+ } saddr;
-#define MLX5_IPSEC_DEV(mdev) false
+ union {
+ __be32 a4;
+ __be32 a6[4];
+ } daddr;
-static inline int mlx5_accel_ipsec_init(struct mlx5_core_dev *mdev)
+ struct upspec upspec;
+
+ u8 family;
+ u8 action;
+ u8 dir : 2;
+ u32 reqid;
+ u32 prio;
+};
+
+struct mlx5e_ipsec_pol_entry {
+ struct secpolicy *sp;
+ struct mlx5e_ipsec *ipsec;
+ struct mlx5e_ipsec_rule ipsec_rule;
+ struct mlx5_accel_pol_xfrm_attrs attrs;
+};
+
+/* This function doesn't really belong here, but let's put it here for now */
+void mlx5_object_change_event(struct mlx5_core_dev *dev, struct mlx5_eqe *eqe);
+
+int mlx5e_ipsec_init(struct mlx5e_priv *priv);
+void mlx5e_ipsec_cleanup(struct mlx5e_priv *priv);
+
+int mlx5e_ipsec_aso_init(struct mlx5e_ipsec *ipsec);
+void mlx5e_ipsec_aso_cleanup(struct mlx5e_ipsec *ipsec);
+
+int mlx5_ipsec_create_sa_ctx(struct mlx5e_ipsec_sa_entry *sa_entry);
+void mlx5_ipsec_free_sa_ctx(struct mlx5e_ipsec_sa_entry *sa_entry);
+
+u32 mlx5_ipsec_device_caps(struct mlx5_core_dev *mdev);
+
+static inline struct mlx5_core_dev *
+mlx5e_ipsec_sa2dev(struct mlx5e_ipsec_sa_entry *sa_entry)
+{
+ return sa_entry->ipsec->mdev;
+}
+
+static inline struct mlx5_core_dev *
+mlx5e_ipsec_pol2dev(struct mlx5e_ipsec_pol_entry *pol_entry)
+{
+ return pol_entry->ipsec->mdev;
+}
+
+void mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry,
+ struct mlx5_accel_esp_xfrm_attrs *attrs,
+ u8 dir);
+
+int mlx5e_accel_ipsec_fs_init(struct mlx5e_ipsec *ipsec);
+void mlx5e_accel_ipsec_fs_cleanup(struct mlx5e_ipsec *ipsec);
+int mlx5e_accel_ipsec_fs_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry);
+void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_ipsec_sa_entry *sa_entry);
+void mlx5e_accel_ipsec_fs_modify(struct mlx5e_ipsec_sa_entry *sa_entry);
+struct ipsec_accel_out_tag;
+void mlx5e_accel_ipsec_handle_tx_wqe(struct mbuf *mb, struct mlx5e_tx_wqe *wqe,
+ struct ipsec_accel_out_tag *tag);
+int mlx5e_accel_ipsec_fs_add_pol(struct mlx5e_ipsec_pol_entry *pol_entry);
+void mlx5e_accel_ipsec_fs_del_pol(struct mlx5e_ipsec_pol_entry *pol_entry);
+static inline int mlx5e_accel_ipsec_get_metadata(unsigned int id)
{
- return 0;
+ return MLX5_ETH_WQE_FT_META_IPSEC << 23 | id;
}
+static inline void
+mlx5e_accel_ipsec_handle_tx(struct mbuf *mb, struct mlx5e_tx_wqe *wqe)
+{
+ struct ipsec_accel_out_tag *tag;
-static inline void mlx5_accel_ipsec_cleanup(struct mlx5_core_dev *mdev)
+ tag = (struct ipsec_accel_out_tag *)m_tag_find(mb,
+ PACKET_TAG_IPSEC_ACCEL_OUT, NULL);
+ if (tag != NULL)
+ mlx5e_accel_ipsec_handle_tx_wqe(mb, wqe, tag);
+}
+void mlx5e_accel_ipsec_fs_rx_tables_destroy(struct mlx5e_priv *priv);
+int mlx5e_accel_ipsec_fs_rx_tables_create(struct mlx5e_priv *priv);
+void mlx5e_accel_ipsec_fs_rx_catchall_rules_destroy(struct mlx5e_priv *priv);
+int mlx5e_accel_ipsec_fs_rx_catchall_rules(struct mlx5e_priv *priv);
+int mlx5_accel_ipsec_rx_tag_add(if_t ifp, struct mbuf *mb);
+int mlx5e_accel_ipsec_handle_rx_cqe(struct mbuf *mb, struct mlx5_cqe64 *cqe);
+static inline int mlx5e_accel_ipsec_flow(struct mlx5_cqe64 *cqe)
{
+ return MLX5_IPSEC_METADATA_MARKER(be32_to_cpu(cqe->ft_metadata));
}
-#endif
+static inline void mlx5e_accel_ipsec_handle_rx(struct mbuf *mb, struct mlx5_cqe64 *cqe)
+{
+ u32 ipsec_meta_data = be32_to_cpu(cqe->ft_metadata);
+
+ if (!MLX5_IPSEC_METADATA_MARKER(ipsec_meta_data)) {
+ struct m_tag *mtag;
+ mtag = m_tag_find(mb, PACKET_TAG_IPSEC_ACCEL_IN, NULL);
+ if (mtag != NULL)
+ m_tag_delete(mb, mtag);
+
+ return;
+ }
+
+ mlx5e_accel_ipsec_handle_rx_cqe(mb, cqe);
+}
#endif /* __MLX5_ACCEL_IPSEC_H__ */
diff --git a/sys/dev/mlx5/mlx5_accel/mlx5_ipsec.c b/sys/dev/mlx5/mlx5_accel/mlx5_ipsec.c
new file mode 100644
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_accel/mlx5_ipsec.c
@@ -0,0 +1,666 @@
+/*-
+ * Copyright (c) 2023 NVIDIA corporation & affiliates.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include "opt_ipsec.h"
+
+#include <sys/types.h>
+#include <netinet/in.h>
+#include <sys/socket.h>
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <net/if.h>
+#include <net/if_var.h>
+#include <net/pfkeyv2.h>
+#include <netipsec/key_var.h>
+#include <netipsec/keydb.h>
+#include <netipsec/ipsec.h>
+#include <netipsec/xform.h>
+#include <netipsec/ipsec_accel.h>
+#include <dev/mlx5/fs.h>
+#include <dev/mlx5/mlx5_en/en.h>
+#include <dev/mlx5/mlx5_accel/ipsec.h>
+
+#define MLX5_IPSEC_RESCHED msecs_to_jiffies(1000)
+
+static struct mlx5e_ipsec_sa_entry *to_ipsec_sa_entry(void *x)
+{
+ return (struct mlx5e_ipsec_sa_entry *)x;
+}
+
+static struct mlx5e_ipsec_pol_entry *to_ipsec_pol_entry(void *x)
+{
+ return (struct mlx5e_ipsec_pol_entry *)x;
+}
+
+static void mlx5e_ipsec_handle_counters(struct work_struct *_work)
+{
+ struct mlx5e_ipsec_dwork *dwork =
+ container_of(_work, struct mlx5e_ipsec_dwork, dwork.work);
+ struct mlx5e_ipsec_sa_entry *sa_entry = dwork->sa_entry;
+ struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule;
+ struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
+ u64 bytes, packets;
+
+ if (sa_entry->attrs.drop)
+ return;
+
+ mlx5_fc_query(mdev, ipsec_rule->fc, &packets, &bytes);
+#ifdef IPSEC_ACCEL
+ ipsec_accel_drv_sa_lifetime_update(sa_entry->savp, sa_entry->ifp,
+ bytes, packets);
+#endif
+
+ queue_delayed_work(sa_entry->ipsec->wq, &dwork->dwork, MLX5_IPSEC_RESCHED);
+}
+
+static int mlx5e_ipsec_create_dwork(struct mlx5e_ipsec_sa_entry *sa_entry)
+{
+ struct mlx5e_ipsec_dwork *dwork;
+
+ dwork = kzalloc(sizeof(*dwork), GFP_KERNEL);
+ if (!dwork)
+ return (ENOMEM);
+
+ dwork->sa_entry = sa_entry;
+ INIT_DELAYED_WORK(&dwork->dwork, mlx5e_ipsec_handle_counters);
+ sa_entry->dwork = dwork;
+ return 0;
+}
+
+static int mlx5_xform_ah_authsize(const struct auth_hash *esph)
+{
+ int alen;
+
+ if (esph == NULL)
+ return 0;
+
+ switch (esph->type) {
+ case CRYPTO_SHA2_256_HMAC:
+ case CRYPTO_SHA2_384_HMAC:
+ case CRYPTO_SHA2_512_HMAC:
+ alen = esph->hashsize / 2; /* RFC4868 2.3 */
+ break;
+
+ case CRYPTO_POLY1305:
+ case CRYPTO_AES_NIST_GMAC:
+ alen = esph->hashsize;
+ break;
+
+ default:
+ alen = AH_HMAC_HASHLEN;
+ break;
+ }
+
+ return alen;
+}
+
+void mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry,
+ struct mlx5_accel_esp_xfrm_attrs *attrs,
+ u8 dir)
+{
+ struct secasvar *savp = sa_entry->savp;
+ const struct auth_hash *esph = savp->tdb_authalgxform;
+ struct aes_gcm_keymat *aes_gcm = &attrs->aes_gcm;
+ struct secasindex *saidx = &savp->sah->saidx;
+ struct seckey *key_encap = savp->key_enc;
+ int key_len;
+
+ memset(attrs, 0, sizeof(*attrs));
+
+ /* subtract off the salt, RFC4106, 8.1 and RFC3686, 5.1 */
+ key_len = _KEYLEN(key_encap) - SAV_ISCTRORGCM(savp) * 4 - SAV_ISCHACHA(savp) * 4;
+
+ memcpy(aes_gcm->aes_key, key_encap->key_data, key_len);
+ aes_gcm->key_len = key_len;
+
+ /* salt and seq_iv */
+ aes_gcm->seq_iv = 0;
+ memcpy(&aes_gcm->salt, key_encap->key_data + key_len,
+ sizeof(aes_gcm->salt));
+
+ switch (savp->alg_enc) {
+ case SADB_X_EALG_AESGCM8:
+ attrs->authsize = 8 / 4; /* in dwords */
+ break;
+ case SADB_X_EALG_AESGCM12:
+ attrs->authsize = 12 / 4; /* in dwords */
+ break;
+ case SADB_X_EALG_AESGCM16:
+ attrs->authsize = 16 / 4; /* in dwords */
+ break;
+ default: break;
+ }
+
+ /* iv len */
+ aes_gcm->icv_len = mlx5_xform_ah_authsize(esph); //TBD: check if value make sense
+
+ attrs->dir = dir;
+ /* spi - host order */
+ attrs->spi = ntohl(savp->spi);
+ attrs->family = saidx->dst.sa.sa_family;
+ attrs->reqid = saidx->reqid;
+
+ if (saidx->src.sa.sa_family == AF_INET) {
+ attrs->saddr.a4 = saidx->src.sin.sin_addr.s_addr;
+ attrs->daddr.a4 = saidx->dst.sin.sin_addr.s_addr;
+ } else {
+ memcpy(&attrs->saddr.a6, &saidx->src.sin6.sin6_addr, 16);
+ memcpy(&attrs->daddr.a6, &saidx->dst.sin6.sin6_addr, 16);
+ }
+
+ if (savp->natt) {
+ attrs->encap = true;
+ attrs->sport = savp->natt->sport;
+ attrs->dport = savp->natt->dport;
+ }
+
+ if (savp->flags & SADB_X_SAFLAGS_ESN) {
+ /* We support replay window with ESN only */
+ attrs->replay_esn.trigger = true;
+ if (sa_entry->esn_state.esn_msb)
+ attrs->replay_esn.esn = sa_entry->esn_state.esn;
+ else
+ /* According to RFC4303, section "3.3.3. Sequence Number Generation",
+ * the first packet sent using a given SA will contain a sequence
+ * number of 1.
+ */
+ attrs->replay_esn.esn = max_t(u32, sa_entry->esn_state.esn, 1);
+ attrs->replay_esn.esn_msb = sa_entry->esn_state.esn_msb;
+ attrs->replay_esn.overlap = sa_entry->esn_state.overlap;
+
+ if (savp->replay) {
+ switch (savp->replay->wsize) {
+ case 4:
+ attrs->replay_esn.replay_window = MLX5_IPSEC_ASO_REPLAY_WIN_32BIT;
+ break;
+ case 8:
+ attrs->replay_esn.replay_window = MLX5_IPSEC_ASO_REPLAY_WIN_64BIT;
+ break;
+ case 16:
+ attrs->replay_esn.replay_window = MLX5_IPSEC_ASO_REPLAY_WIN_128BIT;
+ break;
+ case 32:
+ attrs->replay_esn.replay_window = MLX5_IPSEC_ASO_REPLAY_WIN_256BIT;
+ break;
+ default:
+ /* Do nothing */
+ break;
+ }
+ }
+ }
+}
+
+static int mlx5e_xfrm_validate_state(struct mlx5_core_dev *mdev,
+ struct secasvar *savp)
+{
+ struct secasindex *saidx = &savp->sah->saidx;
+ struct seckey *key_encp = savp->key_enc;
+ int keylen;
+
+ if (!(mlx5_ipsec_device_caps(mdev) &
+ MLX5_IPSEC_CAP_PACKET_OFFLOAD)) {
+ mlx5_core_err(mdev, "FULL offload is not supported\n");
+ return (EINVAL);
+ }
+ if (savp->alg_enc == SADB_EALG_NONE) {
+ mlx5_core_err(mdev, "Cannot offload authenticated xfrm states\n");
+ return (EINVAL);
+ }
+ if (savp->alg_enc != SADB_X_EALG_AESGCM16) {
+ mlx5_core_err(mdev, "Only IPSec aes-gcm-16 encryption protocol may be offloaded\n");
+ return (EINVAL);
+ }
+ if (savp->tdb_compalgxform) {
+ mlx5_core_err(mdev, "Cannot offload compressed xfrm states\n");
+ return (EINVAL);
+ }
+ if (savp->alg_auth != SADB_X_AALG_AES128GMAC && savp->alg_auth != SADB_X_AALG_AES256GMAC) {
+ mlx5_core_err(mdev, "Cannot offload xfrm states with AEAD key length other than 128/256 bits\n");
+ return (EINVAL);
+ }
+ if ((saidx->dst.sa.sa_family != AF_INET && saidx->dst.sa.sa_family != AF_INET6) ||
+ (saidx->src.sa.sa_family != AF_INET && saidx->src.sa.sa_family != AF_INET6)) {
+ mlx5_core_err(mdev, "Only IPv4/6 xfrm states may be offloaded\n");
+ return (EINVAL);
+ }
+ if (saidx->proto != IPPROTO_ESP) {
+ mlx5_core_err(mdev, "Only ESP xfrm state may be offloaded\n");
+ return (EINVAL);
+ }
+ /* subtract off the salt, RFC4106, 8.1 and RFC3686, 5.1 */
+ keylen = _KEYLEN(key_encp) - SAV_ISCTRORGCM(savp) * 4 - SAV_ISCHACHA(savp) * 4;
+ if (keylen != 128/8 && keylen != 256 / 8) {
+ mlx5_core_err(mdev, "Cannot offload xfrm states with AEAD key length other than 128/256 bit\n");
+ return (EINVAL);
+ }
+
+ if (saidx->mode != IPSEC_MODE_TRANSPORT) {
+ mlx5_core_err(mdev, "Only transport xfrm states may be offloaded in full offlaod mode\n");
+ return (EINVAL);
+ }
+
+ if (savp->natt) {
+ if (!(mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_ESPINUDP)) {
+ mlx5_core_err(mdev, "Encapsulation is not supported\n");
+ return (EINVAL);
+ }
+ }
+
+ if (savp->replay && savp->replay->wsize != 0 && savp->replay->wsize != 4 &&
+ savp->replay->wsize != 8 && savp->replay->wsize != 16 && savp->replay->wsize != 32) {
+ mlx5_core_err(mdev, "Unsupported replay window size %d\n", savp->replay->wsize);
+ return (EINVAL);
+ }
+
+ if (savp->flags & SADB_X_SAFLAGS_ESN &&
+ !(mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_ESN)) {
+ mlx5_core_err(mdev, "ESN is not supported\n");
+ return (EINVAL);
+ }
+ return 0;
+}
+
+static int if_sa_inout_install(struct ifnet *ifp, uint32_t drv_spi,
+ struct secasvar *sav, void **privp,
+ u8 dir)
+{
+ struct mlx5e_ipsec_sa_entry *sa_entry = NULL;
+ struct mlx5e_priv *priv = if_getsoftc(ifp);
+ struct mlx5_core_dev *mdev = priv->mdev;
+ struct mlx5e_ipsec *ipsec = priv->ipsec;
+ int err;
+
+ if (priv->gone != 0 || ipsec == NULL)
+ return (EOPNOTSUPP);
+
+ err = mlx5e_xfrm_validate_state(mdev, sav);
+ if (err)
+ return err;
+
+ sa_entry = kzalloc(sizeof(*sa_entry), GFP_KERNEL);
+ if (sa_entry == NULL)
+ return (ENOMEM);
+
+ sa_entry->kspi = drv_spi;
+ sa_entry->savp = sav;
+ sa_entry->ifp = ifp;
+ sa_entry->ipsec = ipsec;
+
+ mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &sa_entry->attrs, dir);
+
+ err = mlx5e_ipsec_create_dwork(sa_entry);
+ if (err)
+ goto err_xfrm;
+
+ /* create hw context */
+ err = mlx5_ipsec_create_sa_ctx(sa_entry);
+ if (err)
+ goto err_sa_ctx;
+
+ err = mlx5e_accel_ipsec_fs_add_rule(sa_entry);
+ if (err)
+ goto err_fs;
+
+ *privp = sa_entry;
+ if (sa_entry->dwork)
+ queue_delayed_work(ipsec->wq, &sa_entry->dwork->dwork, MLX5_IPSEC_RESCHED);
+
+ err = xa_insert(&mdev->ipsec_sadb, sa_entry->ipsec_obj_id, sa_entry, GFP_KERNEL);
+ if (err)
+ goto err_xa;
+
+ return 0;
+
+err_xa:
+ if (sa_entry->dwork)
+ cancel_delayed_work_sync(&sa_entry->dwork->dwork);
+ mlx5e_accel_ipsec_fs_del_rule(sa_entry);
+err_fs:
+ mlx5_ipsec_free_sa_ctx(sa_entry);
+err_sa_ctx:
+ kfree(sa_entry->dwork);
+err_xfrm:
+ kfree(sa_entry);
+ mlx5_en_err(ifp, "Device failed to offload this state");
+ return err;
+}
+
+static int mlx5e_if_sa_out_install(struct ifnet *ifp, void *sp, void *sav, uint32_t drv_spi, void **privp)
+{
+ return if_sa_inout_install(ifp, drv_spi, sav, privp, IPSEC_DIR_OUTBOUND);
+}
+
+static int mlx5e_if_sa_deinstall(struct ifnet *ifp, uint32_t dev_spi, void *priv)
+{
+ struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(priv);
+ struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
+ struct mlx5e_ipsec_sa_entry *old;
+
+ old = xa_erase(&mdev->ipsec_sadb, sa_entry->ipsec_obj_id);
+ WARN_ON(old != sa_entry);
+
+ if (sa_entry->dwork)
+ cancel_delayed_work_sync(&sa_entry->dwork->dwork);
+ mlx5e_accel_ipsec_fs_del_rule(sa_entry);
+ mlx5_ipsec_free_sa_ctx(sa_entry);
+ kfree(sa_entry->dwork);
+ kfree(sa_entry);
+ return 0;
+}
+
+static int mlx5e_if_sa_cnt(struct ifnet *ifp, void *sa, uint32_t drv_spi,
+ void *priv, struct seclifetime *lt)
+{
+ struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(priv);
+ struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule;
+ struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
+ u64 packets;
+
+ mlx5_fc_query(mdev, ipsec_rule->fc, &packets, &lt->bytes);
+ /* TODO: remove this casting once Kostia changes allocation type to be u64 */
+ lt->allocations = (uint32_t)packets;
+ return 0;
+}
+
+static int mlx5e_if_sa_in_install(struct ifnet *ifp, void *sav, void *dst_addr,
+ int sproto, uint32_t spi, uint32_t drv_spi,
+ void **privp)
+{
+ return if_sa_inout_install(ifp, drv_spi, sav, privp, IPSEC_DIR_INBOUND);
+}
+
+static int mlx5e_xfrm_validate_policy(struct mlx5_core_dev *mdev,
+ struct secpolicy *sp, struct inpcb *inp)
+{
+ struct secpolicyindex *spidx = &sp->spidx;
+
+ if (!(mlx5_ipsec_device_caps(mdev) &
+ MLX5_IPSEC_CAP_PACKET_OFFLOAD)) {
+ mlx5_core_err(mdev, "FULL offload is not supported\n");
+ return (EINVAL);
+ }
+
+ if (sp->tcount > 1) {
+ mlx5_core_err(mdev, "Can offload exactly one template, "
+ "not %d\n", sp->tcount);
+ return (EINVAL);
+ }
+
+ if (sp->policy == IPSEC_POLICY_BYPASS &&
+ !(mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PRIO)) {
+ mlx5_core_err(mdev, "Device does not support policy priority\n");
+ return (EINVAL);
+ }
+
+ if (sp->tcount > 0 && inp != NULL) {
+ mlx5_core_err(mdev, "Not valid input data\n");
+ return (EINVAL);
+ }
+
+ if (spidx->dir != IPSEC_DIR_INBOUND && spidx->dir != IPSEC_DIR_OUTBOUND) {
+ mlx5_core_err(mdev, "Wrong policy direction\n");
+ return (EINVAL);
+ }
+
+ if (sp->tcount > 0 && sp->req[0]->saidx.mode != IPSEC_MODE_TRANSPORT) {
+ mlx5_core_err(mdev, "Device supports transport mode only");
+ return (EINVAL);
+ }
+
+ if (sp->policy != IPSEC_POLICY_DISCARD &&
+ sp->policy != IPSEC_POLICY_IPSEC && sp->policy != IPSEC_POLICY_BYPASS) {
+ mlx5_core_err(mdev, "Offloaded policy must be specific on its action\n");
+ return (EINVAL);
+ }
+
+ if (sp->policy == IPSEC_POLICY_BYPASS && !inp) {
+ mlx5_core_err(mdev, "Missing port information for IKE bypass\n");
+ return (EINVAL);
+ }
+
+ if (inp != NULL) {
+ INP_RLOCK(inp);
+ if (inp->inp_socket == NULL || inp->inp_socket->so_proto->
+ pr_protocol != IPPROTO_UDP) {
+ mlx5_core_err(mdev, "Unsupported IKE bypass protocol %d\n",
+ inp->inp_socket == NULL ? -1 :
+ inp->inp_socket->so_proto->pr_protocol);
+ INP_RUNLOCK(inp);
+ return (EINVAL);
+ }
+ INP_RUNLOCK(inp);
+ }
+
+ /* TODO fill relevant bits */
+ return 0;
+}
+
+static void mlx5e_ipsec_build_accel_pol_attrs(struct mlx5e_ipsec_pol_entry *pol_entry,
+ struct mlx5_accel_pol_xfrm_attrs *attrs,
+ struct inpcb *inp)
+{
+ struct secpolicy *sp = pol_entry->sp;
+ struct secpolicyindex *spidx = &sp->spidx;
+
+ memset(attrs, 0, sizeof(*attrs));
+
+ if (!inp) {
+ if (spidx->src.sa.sa_family == AF_INET) {
+ attrs->saddr.a4 = spidx->src.sin.sin_addr.s_addr;
+ attrs->daddr.a4 = spidx->dst.sin.sin_addr.s_addr;
+ } else if (spidx->src.sa.sa_family == AF_INET6) {
+ memcpy(&attrs->saddr.a6, &spidx->src.sin6.sin6_addr, 16);
+ memcpy(&attrs->daddr.a6, &spidx->dst.sin6.sin6_addr, 16);
+ } else {
+ KASSERT(0, ("unsupported family %d", spidx->src.sa.sa_family));
+ }
+ attrs->family = spidx->src.sa.sa_family;
+ attrs->prio = 0;
+ attrs->action = sp->policy;
+ attrs->reqid = sp->req[0]->saidx.reqid;
+ } else {
+ INP_RLOCK(inp);
+ if ((inp->inp_vflag & INP_IPV4) != 0) {
+ attrs->saddr.a4 = inp->inp_laddr.s_addr;
+ attrs->daddr.a4 = inp->inp_faddr.s_addr;
+ attrs->family = AF_INET;
+ } else if ((inp->inp_vflag & INP_IPV6) != 0) {
+ memcpy(&attrs->saddr.a6, &inp->in6p_laddr, 16);
+ memcpy(&attrs->daddr.a6, &inp->in6p_laddr, 16);
+ attrs->family = AF_INET6;
+ } else {
+ KASSERT(0, ("unsupported family %d", inp->inp_vflag));
+ }
+ attrs->upspec.dport = inp->inp_fport;
+ attrs->upspec.sport = inp->inp_lport;
+ attrs->upspec.proto = inp->inp_ip_p;
+ INP_RUNLOCK(inp);
+
+ /* Give highest priority for PCB policies */
+ attrs->prio = 1;
+ attrs->action = IPSEC_POLICY_IPSEC;
+ }
+ attrs->dir = spidx->dir;
+}
+
+static int mlx5e_if_spd_install(struct ifnet *ifp, void *sp, void *inp1,
+ void **ifdatap)
+{
+ struct mlx5e_ipsec_pol_entry *pol_entry;
+ struct mlx5e_priv *priv;
+ int err;
+
+ priv = if_getsoftc(ifp);
+ if (priv->gone || !priv->ipsec)
+ return (EOPNOTSUPP);
+
+ err = mlx5e_xfrm_validate_policy(priv->mdev, sp, inp1);
+ if (err)
+ return err;
+
+ pol_entry = kzalloc(sizeof(*pol_entry), GFP_KERNEL);
+ if (!pol_entry)
+ return (ENOMEM);
+
+ pol_entry->sp = sp;
+ pol_entry->ipsec = priv->ipsec;
+
+ mlx5e_ipsec_build_accel_pol_attrs(pol_entry, &pol_entry->attrs, inp1);
+ err = mlx5e_accel_ipsec_fs_add_pol(pol_entry);
+ if (err)
+ goto err_pol;
+ *ifdatap = pol_entry;
+
+ return 0;
+
+err_pol:
+ kfree(pol_entry);
+ mlx5_en_err(ifp, "Device failed to offload this policy");
+ return err;
+}
+
+
+static int mlx5e_if_spd_deinstall(struct ifnet *ifp, void *sp, void *ifdata)
+{
+ struct mlx5e_ipsec_pol_entry *pol_entry = to_ipsec_pol_entry(ifdata);
+
+ mlx5e_accel_ipsec_fs_del_pol(pol_entry);
+ kfree(pol_entry);
+ return 0;
+}
+
+void mlx5e_ipsec_cleanup(struct mlx5e_priv *priv)
+{
+ struct mlx5e_ipsec *pipsec = priv->ipsec;
+ if (!pipsec)
+ return;
+
+ mlx5e_accel_ipsec_fs_cleanup(pipsec);
+ destroy_workqueue(pipsec->wq);
+ mlx5e_ipsec_aso_cleanup(pipsec);
+ kfree(pipsec);
+ priv->ipsec = NULL;
+}
+
+static int
+mlx5e_if_ipsec_hwassist(if_t ifnet, void *sav __unused,
+ uint32_t drv_spi __unused, void *priv __unused)
+{
+ return (if_gethwassist(ifnet) & (CSUM_TSO | CSUM_TCP | CSUM_UDP |
+ CSUM_IP | CSUM_IP6_TSO | CSUM_IP6_TCP | CSUM_IP6_UDP));
+}
+
+static const struct if_ipsec_accel_methods mlx5e_ipsec_funcs = {
+ .if_sa_in_install = mlx5e_if_sa_in_install,
+ .if_sa_out_install = mlx5e_if_sa_out_install,
+ .if_sa_deinstall = mlx5e_if_sa_deinstall,
+ .if_spdadd = mlx5e_if_spd_install,
+ .if_spddel = mlx5e_if_spd_deinstall,
+ .if_sa_cnt = mlx5e_if_sa_cnt,
+ .if_hwassist = mlx5e_if_ipsec_hwassist,
+};
+
+int mlx5e_ipsec_init(struct mlx5e_priv *priv)
+{
+ struct mlx5_core_dev *mdev = priv->mdev;
+ struct mlx5e_ipsec *pipsec;
+ if_t ifp = priv->ifp;
+ int ret;
+
+ mlx5_core_info(mdev, "ipsec "
+ "offload %d log_max_dek %d gen_obj_types %d "
+ "ipsec_encrypt %d ipsec_decrypt %d "
+ "esp_aes_gcm_128_encrypt %d esp_aes_gcm_128_decrypt %d "
+ "ipsec_full_offload %d "
+ "reformat_add_esp_trasport %d reformat_del_esp_trasport %d "
+ "decap %d "
+ "ignore_flow_level_tx %d ignore_flow_level_rx %d "
+ "reformat_natt_tx %d reformat_natt_rx %d "
+ "ipsec_esn %d\n",
+ MLX5_CAP_GEN(mdev, ipsec_offload) != 0,
+ MLX5_CAP_GEN(mdev, log_max_dek) != 0,
+ (MLX5_CAP_GEN_64(mdev, general_obj_types) &
+ MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_IPSEC) != 0,
+ MLX5_CAP_FLOWTABLE_NIC_TX(mdev, ipsec_encrypt) != 0,
+ MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ipsec_decrypt) != 0,
+ MLX5_CAP_IPSEC(mdev, ipsec_crypto_esp_aes_gcm_128_encrypt) != 0,
+ MLX5_CAP_IPSEC(mdev, ipsec_crypto_esp_aes_gcm_128_decrypt) != 0,
+ MLX5_CAP_IPSEC(mdev, ipsec_full_offload) != 0,
+ MLX5_CAP_FLOWTABLE_NIC_TX(mdev, reformat_add_esp_trasport) != 0,
+ MLX5_CAP_FLOWTABLE_NIC_RX(mdev, reformat_del_esp_trasport) != 0,
+ MLX5_CAP_FLOWTABLE_NIC_RX(mdev, decap) != 0,
+ MLX5_CAP_FLOWTABLE_NIC_TX(mdev, ignore_flow_level) != 0,
+ MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ignore_flow_level) != 0,
+ MLX5_CAP_FLOWTABLE_NIC_TX(mdev,
+ reformat_add_esp_transport_over_udp) != 0,
+ MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
+ reformat_del_esp_transport_over_udp) != 0,
+ MLX5_CAP_IPSEC(mdev, ipsec_esn) != 0);
+
+ if (!(mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PACKET_OFFLOAD)) {
+ mlx5_core_dbg(mdev, "Not an IPSec offload device\n");
+ return 0;
+ }
+
+ xa_init_flags(&mdev->ipsec_sadb, XA_FLAGS_ALLOC);
+
+ pipsec = kzalloc(sizeof(*pipsec), GFP_KERNEL);
+ if (pipsec == NULL)
+ return (ENOMEM);
+
+ pipsec->mdev = mdev;
+ pipsec->pdn = priv->pdn;
+ pipsec->mkey = priv->mr.key;
+
+ ret = mlx5e_ipsec_aso_init(pipsec);
+ if (ret)
+ goto err_ipsec_aso;
+
+ pipsec->wq = alloc_workqueue("mlx5e_ipsec", WQ_UNBOUND, 0);
+ if (pipsec->wq == NULL) {
+ ret = ENOMEM;
+ goto err_ipsec_wq;
+ }
+
+ ret = mlx5e_accel_ipsec_fs_init(pipsec);
+ if (ret)
+ goto err_ipsec_alloc;
+
+ if_setipsec_accel_methods(ifp, &mlx5e_ipsec_funcs);
+ priv->ipsec = pipsec;
+ mlx5_core_dbg(mdev, "IPSec attached to netdevice\n");
+ return 0;
+
+err_ipsec_alloc:
+ destroy_workqueue(pipsec->wq);
+err_ipsec_wq:
+ mlx5e_ipsec_aso_cleanup(pipsec);
+err_ipsec_aso:
+ kfree(pipsec);
+ mlx5_core_err(priv->mdev, "IPSec initialization failed, %d\n", ret);
+ return ret;
+}
diff --git a/sys/dev/mlx5/mlx5_accel/mlx5_ipsec_fs.c b/sys/dev/mlx5/mlx5_accel/mlx5_ipsec_fs.c
new file mode 100644
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_accel/mlx5_ipsec_fs.c
@@ -0,0 +1,2048 @@
+/*-
+ * Copyright (c) 2023 NVIDIA corporation & affiliates.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include "opt_ipsec.h"
+
+#include <sys/types.h>
+#include <netinet/in.h>
+#include <sys/socket.h>
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <net/pfkeyv2.h>
+#include <netipsec/key_var.h>
+#include <netipsec/keydb.h>
+#include <netipsec/ipsec.h>
+#include <netipsec/xform.h>
+#include <netipsec/ipsec_accel.h>
+#include <dev/mlx5/fs.h>
+#include <dev/mlx5/mlx5_en/en.h>
+#include <dev/mlx5/qp.h>
+#include <dev/mlx5/mlx5_accel/ipsec.h>
+#include <dev/mlx5/mlx5_core/fs_core.h>
+#include <dev/mlx5/mlx5_core/fs_chains.h>
+
+/*
+ * TX tables are organized differently for Eternet and for RoCE:
+ *
+ * +=========+
+ * Ethrent Tx | SA KSPI | match
+ * --------------------->|Flowtable|----->+ +
+ * | |\ | / \
+ * +=========+ | | / \ +=========+ +=========+
+ * miss | | / \ | Status | | |
+ * DROP<--------+ |---->|Encrypt|------>|Flowtable|---->| TX NS |
+ * | \ / | | | |
+ * | \ / +=========+ +=========+
+ * +=========+ +=========+ | \ / |
+ * Roce | Policy | match|SA ReqId |match| + |
+ * Tx |Flowtable|----->|Flowtable|---->+ |
+ * ---->|IP header| |ReqId+IP | |
+ * | | | header |--------------------------------+
+ * +=========+ +=========+ miss |
+ * | |
+ * | miss |
+ * +-------------------------------------------------------
+ *
+ * +=========+
+ * | RDMA |
+ * |Flowtable|
+ * | |
+ * Rx Tables and rules: +=========+
+ * + /
+ * +=========+ +=========+ / \ +=========+ +=========+ /match
+ * | Policy | | SA | / \ | Status | | RoCE |/
+ * ---->|Flowtable| match|Flowtable| match / \ |Flowtable|----->|Flowtable|
+ * |IP header|----->|IP header|----->|Decrypt|----->| | | Roce V2 |
+ * | | |+ESP+SPI | \ / | | | UDP port|\
+ * +=========+ +=========+ \ / +=========+ +=========+ \miss
+ * | | \ / \
+ * | | + +=========+
+ * | miss | miss | Ethernet|
+ * +--------------->---------------------------------------------------->| RX NS |
+ * | |
+ * +=========+
+ *
+ */
+
+#define NUM_IPSEC_FTE BIT(15)
+#define IPSEC_TUNNEL_DEFAULT_TTL 0x40
+
+struct mlx5e_ipsec_fc {
+ struct mlx5_fc *cnt;
+ struct mlx5_fc *drop;
+};
+
+struct mlx5e_ipsec_ft {
+ struct mutex mutex; /* Protect changes to this struct */
+ struct mlx5_flow_table *pol;
+ struct mlx5_flow_table *sa_kspi;
+ struct mlx5_flow_table *sa;
+ struct mlx5_flow_table *status;
+ u32 refcnt;
+};
+
+struct mlx5e_ipsec_tx_roce {
+ struct mlx5_flow_group *g;
+ struct mlx5_flow_table *ft;
+ struct mlx5_flow_handle *rule;
+ struct mlx5_flow_namespace *ns;
+};
+
+struct mlx5e_ipsec_miss {
+ struct mlx5_flow_group *group;
+ struct mlx5_flow_handle *rule;
+};
+
+struct mlx5e_ipsec_tx {
+ struct mlx5e_ipsec_ft ft;
+ struct mlx5e_ipsec_miss pol;
+ struct mlx5e_ipsec_miss kspi_miss;
+ struct mlx5e_ipsec_rule status;
+ struct mlx5e_ipsec_rule kspi_bypass_rule; /*rule for IPSEC bypass*/
+ struct mlx5_flow_namespace *ns;
+ struct mlx5e_ipsec_fc *fc;
+ struct mlx5_fs_chains *chains;
+ struct mlx5e_ipsec_tx_roce roce;
+};
+
+struct mlx5e_ipsec_rx_roce {
+ struct mlx5_flow_group *g;
+ struct mlx5_flow_table *ft;
+ struct mlx5_flow_handle *rule;
+ struct mlx5e_ipsec_miss roce_miss;
+
+ struct mlx5_flow_table *ft_rdma;
+ struct mlx5_flow_namespace *ns_rdma;
+};
+
+struct mlx5e_ipsec_rx {
+ struct mlx5e_ipsec_ft ft;
+ struct mlx5e_ipsec_miss pol;
+ struct mlx5e_ipsec_miss sa;
+ struct mlx5e_ipsec_rule status;
+ struct mlx5_flow_namespace *ns;
+ struct mlx5e_ipsec_fc *fc;
+ struct mlx5_fs_chains *chains;
+ struct mlx5e_ipsec_rx_roce roce;
+};
+
+static void setup_fte_reg_a_with_tag(struct mlx5_flow_spec *spec,
+ u16 kspi);
+static void setup_fte_reg_a_no_tag(struct mlx5_flow_spec *spec);
+
+static void setup_fte_no_frags(struct mlx5_flow_spec *spec)
+{
+ /* Non fragmented */
+ spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
+
+ MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.frag);
+ MLX5_SET(fte_match_param, spec->match_value, outer_headers.frag, 0);
+}
+
+static void setup_fte_esp(struct mlx5_flow_spec *spec)
+{
+ /* ESP header */
+ spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
+
+ MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol);
+ MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_ESP);
+}
+
+static void setup_fte_spi(struct mlx5_flow_spec *spec, u32 spi, bool encap)
+{
+ /* SPI number */
+ spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS;
+
+ if (encap) {
+ MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters.inner_esp_spi);
+ MLX5_SET(fte_match_param, spec->match_value, misc_parameters.inner_esp_spi, spi);
+ } else {
+ MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters.outer_esp_spi);
+ MLX5_SET(fte_match_param, spec->match_value, misc_parameters.outer_esp_spi, spi);
+ }
+}
+
+static struct mlx5_fs_chains *
+ipsec_chains_create(struct mlx5_core_dev *mdev, struct mlx5_flow_table *miss_ft,
+ enum mlx5_flow_namespace_type ns, int base_prio,
+ int base_level, struct mlx5_flow_table **root_ft)
+{
+ struct mlx5_chains_attr attr = {};
+ struct mlx5_fs_chains *chains;
+ struct mlx5_flow_table *ft;
+ int err;
+
+ attr.flags = MLX5_CHAINS_AND_PRIOS_SUPPORTED |
+ MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED;
+ attr.max_grp_num = 2;
+ attr.default_ft = miss_ft;
+ attr.ns = ns;
+ attr.fs_base_prio = base_prio;
+ attr.fs_base_level = base_level;
+ chains = mlx5_chains_create(mdev, &attr);
+ if (IS_ERR(chains))
+ return chains;
+
+ /* Create chain 0, prio 1, level 0 to connect chains to prev in fs_core */
+ ft = mlx5_chains_get_table(chains, 0, 1, 0);
+ if (IS_ERR(ft)) {
+ err = PTR_ERR(ft);
+ goto err_chains_get;
+ }
+
+ *root_ft = ft;
+ return chains;
+
+err_chains_get:
+ mlx5_chains_destroy(chains);
+ return ERR_PTR(err);
+}
+
+static void ipsec_chains_destroy(struct mlx5_fs_chains *chains)
+{
+ mlx5_chains_put_table(chains, 0, 1, 0);
+ mlx5_chains_destroy(chains);
+}
+
+static struct mlx5_flow_table *
+ipsec_chains_get_table(struct mlx5_fs_chains *chains, u32 prio)
+{
+ return mlx5_chains_get_table(chains, 0, prio + 1, 0);
+}
+
+static void ipsec_chains_put_table(struct mlx5_fs_chains *chains, u32 prio)
+{
+ mlx5_chains_put_table(chains, 0, prio + 1, 0);
+}
+
+static struct mlx5_flow_table *ipsec_rx_ft_create(struct mlx5_flow_namespace *ns,
+ int level, int prio,
+ int max_num_groups)
+{
+ struct mlx5_flow_table_attr ft_attr = {};
+
+ ft_attr.max_fte = NUM_IPSEC_FTE;
+ ft_attr.level = level;
+ ft_attr.prio = prio;
+ ft_attr.autogroup.max_num_groups = max_num_groups;
+ ft_attr.autogroup.num_reserved_entries = 1;
+
+ return mlx5_create_auto_grouped_flow_table(ns, &ft_attr);
+}
+
+static int ipsec_miss_create(struct mlx5_core_dev *mdev,
+ struct mlx5_flow_table *ft,
+ struct mlx5e_ipsec_miss *miss,
+ struct mlx5_flow_destination *dest)
+{
+ int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
+ struct mlx5_flow_act flow_act = {};
+ struct mlx5_flow_spec *spec;
+ u32 *flow_group_in;
+ int err = 0;
+
+ flow_group_in = kvzalloc(inlen, GFP_KERNEL);
+ spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
+ if (!flow_group_in || !spec) {
+ err = -ENOMEM;
+ goto out;
+ }
+
+ /* Create miss_group */
+ MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ft->max_fte - 1);
+ MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, ft->max_fte - 1);
+ miss->group = mlx5_create_flow_group(ft, flow_group_in);
+ if (IS_ERR(miss->group)) {
+ err = PTR_ERR(miss->group);
+ mlx5_core_err(mdev, "fail to create IPsec miss_group err=%d\n",
+ err);
+ goto out;
+ }
+
+ if (dest)
+ flow_act.action = MLX5_FLOW_RULE_FWD_ACTION_DEST;
+ else
+ flow_act.action = MLX5_FLOW_RULE_FWD_ACTION_DROP;
+ /* Create miss rule */
+ miss->rule = mlx5_add_flow_rules(ft, NULL, &flow_act, dest, 1);
+ if (IS_ERR(miss->rule)) {
+ mlx5_destroy_flow_group(miss->group);
+ err = PTR_ERR(miss->rule);
+ mlx5_core_err(mdev, "fail to create IPsec miss_rule err=%d\n",
+ err);
+ goto out;
+ }
+out:
+ kvfree(flow_group_in);
+ kvfree(spec);
+ return err;
+}
+
+static int setup_modify_header(struct mlx5_core_dev *mdev, u32 val, u8 dir,
+ struct mlx5_flow_act *flow_act)
+{
+ u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {};
+ enum mlx5_flow_namespace_type ns_type;
+ struct mlx5_modify_hdr *modify_hdr;
+
+ MLX5_SET(set_action_in, action, action_type, MLX5_ACTION_TYPE_SET);
+ switch (dir) {
+ case IPSEC_DIR_INBOUND:
+ MLX5_SET(set_action_in, action, field,
+ MLX5_ACTION_IN_FIELD_METADATA_REG_B);
+ ns_type = MLX5_FLOW_NAMESPACE_KERNEL;
+ break;
+ case IPSEC_DIR_OUTBOUND:
+ MLX5_SET(set_action_in, action, field,
+ MLX5_ACTION_IN_FIELD_METADATA_REG_C_0);
+ ns_type = MLX5_FLOW_NAMESPACE_EGRESS;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ MLX5_SET(set_action_in, action, data, val);
+ MLX5_SET(set_action_in, action, offset, 0);
+ MLX5_SET(set_action_in, action, length, 32);
+
+ modify_hdr = mlx5_modify_header_alloc(mdev, ns_type, 1, action);
+ if (IS_ERR(modify_hdr)) {
+ mlx5_core_err(mdev, "Failed to allocate modify_header %ld\n",
+ PTR_ERR(modify_hdr));
+ return PTR_ERR(modify_hdr);
+ }
+
+ flow_act->modify_hdr = modify_hdr;
+ flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
+ return 0;
+}
+
+static int
+setup_pkt_transport_reformat(struct mlx5_accel_esp_xfrm_attrs *attrs,
+ struct mlx5_pkt_reformat_params *reformat_params)
+{
+ struct udphdr *udphdr;
+ size_t bfflen = 16;
+ char *reformatbf;
+ __be32 spi;
+ void *hdr;
+
+ if (attrs->family == AF_INET)
+ if (attrs->encap)
+ reformat_params->type = MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_UDPV4;
+ else
+ reformat_params->type = MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV4;
+ else
+ reformat_params->type =
+ MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV6;
+
+ if (attrs->encap)
+ bfflen += sizeof(*udphdr);
+ reformatbf = kzalloc(bfflen, GFP_KERNEL);
+ if (!reformatbf)
+ return -ENOMEM;
+
+ hdr = reformatbf;
+ if (attrs->encap) {
+ udphdr = (struct udphdr *)reformatbf;
+ udphdr->uh_sport = attrs->sport;
+ udphdr->uh_dport = attrs->dport;
+ hdr += sizeof(*udphdr);
+ }
+
+ /* convert to network format */
+ spi = htonl(attrs->spi);
+ memcpy(hdr, &spi, 4);
+
+ reformat_params->param_0 = attrs->authsize;
+ reformat_params->size = bfflen;
+ reformat_params->data = reformatbf;
+
+ return 0;
+}
+
+static int setup_pkt_reformat(struct mlx5_core_dev *mdev,
+ struct mlx5_accel_esp_xfrm_attrs *attrs,
+ struct mlx5_flow_act *flow_act)
+{
+ enum mlx5_flow_namespace_type ns_type = MLX5_FLOW_NAMESPACE_EGRESS;
+ struct mlx5_pkt_reformat_params reformat_params = {};
+ struct mlx5_pkt_reformat *pkt_reformat;
+ int ret;
+
+ if (attrs->dir == IPSEC_DIR_INBOUND) {
+ if (attrs->encap)
+ reformat_params.type = MLX5_REFORMAT_TYPE_DEL_ESP_TRANSPORT_OVER_UDP;
+ else
+ reformat_params.type = MLX5_REFORMAT_TYPE_DEL_ESP_TRANSPORT;
+ ns_type = MLX5_FLOW_NAMESPACE_KERNEL;
+ goto cmd;
+ }
+
+ ret = setup_pkt_transport_reformat(attrs, &reformat_params);
+ if (ret)
+ return ret;
+cmd:
+ pkt_reformat =
+ mlx5_packet_reformat_alloc(mdev, &reformat_params, ns_type);
+ if (reformat_params.data)
+ kfree(reformat_params.data);
+ if (IS_ERR(pkt_reformat))
+ return PTR_ERR(pkt_reformat);
+
+ flow_act->pkt_reformat = pkt_reformat;
+ flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
+ return 0;
+}
+
+static void setup_fte_addr4(struct mlx5_flow_spec *spec, __be32 *saddr,
+ __be32 *daddr)
+{
+ spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
+
+ MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version);
+ MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 4);
+
+ memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
+ outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4), saddr, 4);
+ memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
+ outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4), daddr, 4);
+ MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
+ outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4);
+ MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
+ outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
+}
+
+static void setup_fte_addr6(struct mlx5_flow_spec *spec, __be32 *saddr,
+ __be32 *daddr)
+{
+ spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
+
+ MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version);
+ MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 6);
+
+ memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
+ outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), saddr, 16);
+ memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
+ outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), daddr, 16);
+ memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
+ outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), 0xff, 16);
+ memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
+ outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 0xff, 16);
+}
+
+static int rx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry)
+{
+ struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule;
+ struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs;
+ struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
+ struct mlx5e_ipsec *ipsec = sa_entry->ipsec;
+ struct mlx5_flow_destination dest[2] = {};
+ struct mlx5_flow_act flow_act = {};
+ struct mlx5_flow_handle *rule;
+ struct mlx5_flow_spec *spec;
+ struct mlx5e_ipsec_rx *rx;
+ struct mlx5_fc *counter;
+ int err;
+
+ rx = (attrs->family == AF_INET) ? ipsec->rx_ipv4 : ipsec->rx_ipv6;
+
+ spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
+ if (!spec)
+ return -ENOMEM;
+
+ if (attrs->family == AF_INET)
+ setup_fte_addr4(spec, &attrs->saddr.a4, &attrs->daddr.a4);
+ else
+ setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6);
+
+ if (!attrs->encap)
+ setup_fte_esp(spec);
+
+ setup_fte_spi(spec, attrs->spi, attrs->encap);
+ setup_fte_no_frags(spec);
+
+ if (!attrs->drop) {
+ err = setup_modify_header(mdev, sa_entry->kspi | BIT(31), IPSEC_DIR_INBOUND,
+ &flow_act);
+ if (err)
+ goto err_mod_header;
+ }
+
+ err = setup_pkt_reformat(mdev, attrs, &flow_act);
+ if (err)
+ goto err_pkt_reformat;
+
+ counter = mlx5_fc_create(mdev, false);
+ if (IS_ERR(counter)) {
+ err = PTR_ERR(counter);
+ goto err_add_cnt;
+ }
+
+ flow_act.crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_IPSEC;
+ flow_act.crypto.op = MLX5_FLOW_ACT_CRYPTO_OP_DECRYPT;
+ flow_act.crypto.obj_id = sa_entry->ipsec_obj_id;
+ flow_act.flags |= FLOW_ACT_NO_APPEND;
+
+ flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT |
+ MLX5_FLOW_CONTEXT_ACTION_COUNT;
+
+ if (attrs->drop)
+ flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP;
+ else
+ flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
+
+ dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
+ dest[0].ft = rx->ft.status;
+ dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
+ dest[1].counter_id = mlx5_fc_id(counter);
+
+ rule = mlx5_add_flow_rules(rx->ft.sa, spec, &flow_act, dest, 2);
+ if (IS_ERR(rule)) {
+ err = PTR_ERR(rule);
+ mlx5_core_err(mdev, "fail to add RX ipsec rule err=%d\n", err);
+ goto err_add_flow;
+ }
+
+ kvfree(spec);
+ ipsec_rule->rule = rule;
+ ipsec_rule->fc = counter;
+ ipsec_rule->modify_hdr = flow_act.modify_hdr;
+ ipsec_rule->pkt_reformat = flow_act.pkt_reformat;
+ return 0;
+
+err_add_flow:
+ mlx5_fc_destroy(mdev, counter);
+err_add_cnt:
+ mlx5_packet_reformat_dealloc(mdev, flow_act.pkt_reformat);
+err_pkt_reformat:
+ if (flow_act.modify_hdr)
+ mlx5_modify_header_dealloc(mdev, flow_act.modify_hdr);
+err_mod_header:
+ kvfree(spec);
+
+ return err;
+}
+
+static struct mlx5_flow_table *ipsec_tx_ft_create(struct mlx5_flow_namespace *ns,
+ int level, int prio,
+ int max_num_groups)
+{
+ struct mlx5_flow_table_attr ft_attr = {};
+
+ ft_attr.autogroup.num_reserved_entries = 1;
+ ft_attr.autogroup.max_num_groups = max_num_groups;
+ ft_attr.max_fte = NUM_IPSEC_FTE;
+ ft_attr.level = level;
+ ft_attr.prio = prio;
+
+ return mlx5_create_auto_grouped_flow_table(ns, &ft_attr);
+}
+
+static int ipsec_counter_rule_tx(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_tx *tx)
+{
+ struct mlx5_flow_destination dest = {};
+ struct mlx5_flow_act flow_act = {};
+ struct mlx5_flow_handle *fte;
+ int err;
+
+ /* create fte */
+ flow_act.action = MLX5_FLOW_CONTEXT_ACTION_COUNT |
+ MLX5_FLOW_CONTEXT_ACTION_ALLOW;
+
+ dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
+ dest.counter_id = mlx5_fc_id(tx->fc->cnt);
+ fte = mlx5_add_flow_rules(tx->ft.status, NULL, &flow_act, &dest, 1);
+ if (IS_ERR_OR_NULL(fte)) {
+ err = PTR_ERR(fte);
+ mlx5_core_err(mdev, "Fail to add ipsec tx counter rule err=%d\n", err);
+ goto err_rule;
+ }
+
+ tx->status.rule = fte;
+ return 0;
+
+err_rule:
+ return err;
+}
+
+static void tx_destroy_roce(struct mlx5e_ipsec_tx *tx) {
+ if (!tx->roce.ft)
+ return;
+
+ mlx5_del_flow_rules(&tx->roce.rule);
+ mlx5_destroy_flow_group(tx->roce.g);
+ mlx5_destroy_flow_table(tx->roce.ft);
+ tx->roce.ft = NULL;
+}
+
+/* IPsec TX flow steering */
+static void tx_destroy(struct mlx5e_ipsec_tx *tx)
+{
+ tx_destroy_roce(tx);
+ if (tx->chains) {
+ ipsec_chains_destroy(tx->chains);
+ } else {
+ mlx5_del_flow_rules(&tx->pol.rule);
+ mlx5_destroy_flow_group(tx->pol.group);
+ mlx5_destroy_flow_table(tx->ft.pol);
+ }
+ mlx5_destroy_flow_table(tx->ft.sa);
+ mlx5_del_flow_rules(&tx->kspi_miss.rule);
+ mlx5_destroy_flow_group(tx->kspi_miss.group);
+ mlx5_del_flow_rules(&tx->kspi_bypass_rule.rule);
+ mlx5_del_flow_rules(&tx->kspi_bypass_rule.kspi_rule);
+ mlx5_destroy_flow_table(tx->ft.sa_kspi);
+ mlx5_del_flow_rules(&tx->status.rule);
+ mlx5_destroy_flow_table(tx->ft.status);
+}
+
+static int ipsec_tx_roce_rule_setup(struct mlx5_core_dev *mdev,
+ struct mlx5e_ipsec_tx *tx)
+{
+ struct mlx5_flow_destination dst = {};
+ struct mlx5_flow_act flow_act = {};
+ struct mlx5_flow_handle *rule;
+ int err = 0;
+
+ flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
+ dst.type = MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE;
+ dst.ft = tx->ft.pol;
+ rule = mlx5_add_flow_rules(tx->roce.ft, NULL, &flow_act, &dst, 1);
+ if (IS_ERR(rule)) {
+ err = PTR_ERR(rule);
+ mlx5_core_err(mdev, "Fail to add TX roce ipsec rule err=%d\n",
+ err);
+ goto out;
+ }
+ tx->roce.rule = rule;
+
+out:
+ return err;
+}
+
+static int ipsec_tx_create_roce(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_tx *tx)
+{
+ struct mlx5_flow_table_attr ft_attr = {};
+ struct mlx5_flow_table *ft;
+ struct mlx5_flow_group *g;
+ int ix = 0;
+ int err;
+ u32 *in;
+
+ if (!tx->roce.ns)
+ return -EOPNOTSUPP;
+
+ in = kvzalloc(MLX5_ST_SZ_BYTES(create_flow_group_in), GFP_KERNEL);
+ if (!in)
+ return -ENOMEM;
+
+ ft_attr.max_fte = 1;
+ ft = mlx5_create_flow_table(tx->roce.ns, &ft_attr);
+ if (IS_ERR(ft)) {
+ err = PTR_ERR(ft);
+ mlx5_core_err(mdev, "Fail to create ipsec tx roce ft err=%d\n",
+ err);
+ goto fail_table;
+ }
+ tx->roce.ft = ft;
+
+ MLX5_SET_CFG(in, start_flow_index, ix);
+ ix += 1;
+ MLX5_SET_CFG(in, end_flow_index, ix - 1);
+ g = mlx5_create_flow_group(ft, in);
+ if (IS_ERR(g)) {
+ err = PTR_ERR(g);
+ mlx5_core_err(mdev, "Fail to create ipsec tx roce group err=%d\n",
+ err);
+ goto fail_group;
+ }
+ tx->roce.g = g;
+
+ err = ipsec_tx_roce_rule_setup(mdev, tx);
+ if (err) {
+ mlx5_core_err(mdev, "Fail to create RoCE IPsec tx rules err=%d\n", err);
+ goto fail_rule;
+ }
+
+ kvfree(in);
+ return 0;
+
+fail_rule:
+ mlx5_destroy_flow_group(tx->roce.g);
+fail_group:
+ mlx5_destroy_flow_table(tx->roce.ft);
+ tx->roce.ft = NULL;
+fail_table:
+ kvfree(in);
+ return err;
+}
+
+/*
+ * Setting a rule in KSPI table for values that should bypass IPSEC.
+ *
+ * mdev - mlx5 core device
+ * tx - IPSEC TX
+ * return - 0 for success errno for failure
+ */
+static int tx_create_kspi_bypass_rules(struct mlx5_core_dev *mdev,
+ struct mlx5e_ipsec_tx *tx)
+{
+ struct mlx5_flow_destination dest = {};
+ struct mlx5_flow_act flow_act = {};
+ struct mlx5_flow_act flow_act_kspi = {};
+ struct mlx5_flow_handle *rule;
+ struct mlx5_flow_spec *spec;
+ int err;
+
+ spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
+ if (!spec)
+ return -ENOMEM;
+
+ dest.ft = tx->ft.status;
+ dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
+ flow_act_kspi.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
+
+ setup_fte_reg_a_with_tag(spec, IPSEC_ACCEL_DRV_SPI_BYPASS);
+ rule = mlx5_add_flow_rules(tx->ft.sa_kspi, spec, &flow_act_kspi,
+ &dest, 1);
+ if (IS_ERR(rule)) {
+ err = PTR_ERR(rule);
+ mlx5_core_err(mdev, "Fail to add ipsec kspi bypass rule err=%d\n",
+ err);
+ goto err_add_kspi_rule;
+ }
+ tx->kspi_bypass_rule.kspi_rule = rule;
+
+ /* set the rule for packets withoiut ipsec tag. */
+ flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
+ memset(spec, 0, sizeof(*spec));
+ setup_fte_reg_a_no_tag(spec);
+ rule = mlx5_add_flow_rules(tx->ft.sa_kspi, spec, &flow_act, &dest, 1);
+ if (IS_ERR(rule)) {
+ err = PTR_ERR(rule);
+ mlx5_core_err(mdev, "Fail to add ipsec kspi bypass rule err=%d\n", err);
+ goto err_add_rule;
+ }
+ tx->kspi_bypass_rule.rule = rule;
+
+ kvfree(spec);
+ return 0;
+err_add_rule:
+ mlx5_del_flow_rules(&tx->kspi_bypass_rule.kspi_rule);
+err_add_kspi_rule:
+ kvfree(spec);
+ return err;
+}
+
+
+static int tx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_tx *tx)
+{
+ struct mlx5_flow_destination dest = {};
+ struct mlx5_flow_table *ft;
+ int err;
+
+ /*
+ * Tx flow is different for ethernet traffic then for RoCE packets
+ * For Ethernet packets we start in SA KSPI table that matches KSPI of SA rule
+ * to the KSPI in the packet metadata
+ * For RoCE traffic we start in Policy table, then move to SA table
+ * which matches either reqid of the SA rule to reqid reported by policy table
+ * or ip header fields of SA to the packet IP header fields.
+ * Tables are ordered by their level so we set kspi
+ * with level 0 to have it first one for ethernet traffic.
+ * For RoCE the RoCE TX table direct the packets to policy table explicitly
+ */
+ ft = ipsec_tx_ft_create(tx->ns, 0, 0, 4);
+ if (IS_ERR(ft))
+ return PTR_ERR(ft);
+ tx->ft.sa_kspi = ft;
+
+ ft = ipsec_tx_ft_create(tx->ns, 2, 0, 4);
+ if (IS_ERR(ft)) {
+ err = PTR_ERR(ft);
+ goto err_reqid_ft;
+ }
+ tx->ft.sa = ft;
+
+ if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PRIO) {
+ tx->chains = ipsec_chains_create(
+ mdev, tx->ft.sa, MLX5_FLOW_NAMESPACE_EGRESS_IPSEC, 0, 1,
+ &tx->ft.pol);
+ if (IS_ERR(tx->chains)) {
+ err = PTR_ERR(tx->chains);
+ goto err_pol_ft;
+ }
+ } else {
+ ft = ipsec_tx_ft_create(tx->ns, 1, 0, 2);
+ if (IS_ERR(ft)) {
+ err = PTR_ERR(ft);
+ goto err_pol_ft;
+ }
+ tx->ft.pol = ft;
+ dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
+ dest.ft = tx->ft.sa;
+ err = ipsec_miss_create(mdev, tx->ft.pol, &tx->pol, &dest);
+ if (err)
+ goto err_pol_miss;
+ }
+
+ ft = ipsec_tx_ft_create(tx->ns, 2, 0, 1);
+ if (IS_ERR(ft)) {
+ err = PTR_ERR(ft);
+ goto err_status_ft;
+ }
+ tx->ft.status = ft;
+
+ /* set miss rule for kspi table with drop action*/
+ err = ipsec_miss_create(mdev, tx->ft.sa_kspi, &tx->kspi_miss, NULL);
+ if (err)
+ goto err_kspi_miss;
+
+ err = tx_create_kspi_bypass_rules(mdev, tx);
+ if (err)
+ goto err_kspi_rule;
+
+ err = ipsec_counter_rule_tx(mdev, tx);
+ if (err)
+ goto err_status_rule;
+
+ err = ipsec_tx_create_roce(mdev, tx);
+ if (err)
+ goto err_counter_rule;
+
+ return 0;
+
+err_counter_rule:
+ mlx5_del_flow_rules(&tx->status.rule);
+err_status_rule:
+ mlx5_del_flow_rules(&tx->kspi_bypass_rule.rule);
+ mlx5_del_flow_rules(&tx->kspi_bypass_rule.kspi_rule);
+err_kspi_rule:
+ mlx5_destroy_flow_table(tx->ft.status);
+err_status_ft:
+ if (tx->chains) {
+ ipsec_chains_destroy(tx->chains);
+ } else {
+ mlx5_del_flow_rules(&tx->pol.rule);
+ mlx5_destroy_flow_group(tx->pol.group);
+ }
+err_pol_miss:
+ if (!tx->chains)
+ mlx5_destroy_flow_table(tx->ft.pol);
+err_pol_ft:
+ mlx5_del_flow_rules(&tx->kspi_miss.rule);
+ mlx5_destroy_flow_group(tx->kspi_miss.group);
+err_kspi_miss:
+ mlx5_destroy_flow_table(tx->ft.sa);
+err_reqid_ft:
+ mlx5_destroy_flow_table(tx->ft.sa_kspi);
+ return err;
+}
+
+static int tx_get(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec,
+ struct mlx5e_ipsec_tx *tx)
+{
+ int err;
+
+ if (tx->ft.refcnt)
+ goto skip;
+
+ err = tx_create(mdev, tx);
+ if (err)
+ return err;
+
+skip:
+ tx->ft.refcnt++;
+ return 0;
+}
+
+static void tx_put(struct mlx5e_ipsec *ipsec, struct mlx5e_ipsec_tx *tx)
+{
+ if (--tx->ft.refcnt)
+ return;
+
+ tx_destroy(tx);
+}
+
+static struct mlx5e_ipsec_tx *tx_ft_get(struct mlx5_core_dev *mdev,
+ struct mlx5e_ipsec *ipsec)
+{
+ struct mlx5e_ipsec_tx *tx = ipsec->tx;
+ int err;
+
+ mutex_lock(&tx->ft.mutex);
+ err = tx_get(mdev, ipsec, tx);
+ mutex_unlock(&tx->ft.mutex);
+ if (err)
+ return ERR_PTR(err);
+
+ return tx;
+}
+
+static struct mlx5_flow_table *tx_ft_get_policy(struct mlx5_core_dev *mdev,
+ struct mlx5e_ipsec *ipsec,
+ u32 prio)
+{
+ struct mlx5e_ipsec_tx *tx = ipsec->tx;
+ struct mlx5_flow_table *ft;
+ int err;
+
+ mutex_lock(&tx->ft.mutex);
+ err = tx_get(mdev, ipsec, tx);
+ if (err)
+ goto err_get;
+
+ ft = tx->chains ? ipsec_chains_get_table(tx->chains, prio) : tx->ft.pol;
+ if (IS_ERR(ft)) {
+ err = PTR_ERR(ft);
+ goto err_get_ft;
+ }
+
+ mutex_unlock(&tx->ft.mutex);
+ return ft;
+
+err_get_ft:
+ tx_put(ipsec, tx);
+err_get:
+ mutex_unlock(&tx->ft.mutex);
+ return ERR_PTR(err);
+}
+
+static void tx_ft_put_policy(struct mlx5e_ipsec *ipsec, u32 prio)
+{
+ struct mlx5e_ipsec_tx *tx = ipsec->tx;
+
+ mutex_lock(&tx->ft.mutex);
+ if (tx->chains)
+ ipsec_chains_put_table(tx->chains, prio);
+
+ tx_put(ipsec, tx);
+ mutex_unlock(&tx->ft.mutex);
+}
+
+static void tx_ft_put(struct mlx5e_ipsec *ipsec)
+{
+ struct mlx5e_ipsec_tx *tx = ipsec->tx;
+
+ mutex_lock(&tx->ft.mutex);
+ tx_put(ipsec, tx);
+ mutex_unlock(&tx->ft.mutex);
+}
+
+static void setup_fte_reg_a_with_tag(struct mlx5_flow_spec *spec,
+ u16 kspi)
+{
+ /* Add IPsec indicator in metadata_reg_a. */
+ spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2;
+
+ MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
+ misc_parameters_2.metadata_reg_a);
+ MLX5_SET(fte_match_param, spec->match_value,
+ misc_parameters_2.metadata_reg_a,
+ MLX5_ETH_WQE_FT_META_IPSEC << 23 | kspi);
+}
+
+static void setup_fte_reg_a_no_tag(struct mlx5_flow_spec *spec)
+{
+ /* Add IPsec indicator in metadata_reg_a. */
+ spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2;
+
+ MLX5_SET(fte_match_param, spec->match_criteria,
+ misc_parameters_2.metadata_reg_a,
+ MLX5_ETH_WQE_FT_META_IPSEC << 23);
+ MLX5_SET(fte_match_param, spec->match_value,
+ misc_parameters_2.metadata_reg_a,
+ 0);
+}
+
+static void setup_fte_reg_c0(struct mlx5_flow_spec *spec, u32 reqid)
+{
+ /* Pass policy check before choosing this SA */
+ spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2;
+
+ MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
+ misc_parameters_2.metadata_reg_c_0);
+ MLX5_SET(fte_match_param, spec->match_value,
+ misc_parameters_2.metadata_reg_c_0, reqid);
+}
+
+static void setup_fte_upper_proto_match(struct mlx5_flow_spec *spec, struct upspec *upspec)
+{
+ switch (upspec->proto) {
+ case IPPROTO_UDP:
+ if (upspec->dport) {
+ MLX5_SET_TO_ONES(fte_match_set_lyr_2_4,
+ spec->match_criteria, udp_dport);
+ MLX5_SET(fte_match_set_lyr_2_4, spec->match_value,
+ udp_dport, upspec->dport);
+ }
+
+ if (upspec->sport) {
+ MLX5_SET_TO_ONES(fte_match_set_lyr_2_4,
+ spec->match_criteria, udp_sport);
+ MLX5_SET(fte_match_set_lyr_2_4, spec->match_value,
+ udp_dport, upspec->sport);
+ }
+ break;
+ case IPPROTO_TCP:
+ if (upspec->dport) {
+ MLX5_SET_TO_ONES(fte_match_set_lyr_2_4,
+ spec->match_criteria, tcp_dport);
+ MLX5_SET(fte_match_set_lyr_2_4, spec->match_value,
+ tcp_dport, upspec->dport);
+ }
+
+ if (upspec->sport) {
+ MLX5_SET_TO_ONES(fte_match_set_lyr_2_4,
+ spec->match_criteria, tcp_sport);
+ MLX5_SET(fte_match_set_lyr_2_4, spec->match_value,
+ tcp_dport, upspec->sport);
+ }
+ break;
+ default:
+ return;
+ }
+
+ spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
+ MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, spec->match_criteria, ip_protocol);
+ MLX5_SET(fte_match_set_lyr_2_4, spec->match_value, ip_protocol, upspec->proto);
+}
+
+static int tx_add_kspi_rule(struct mlx5e_ipsec_sa_entry *sa_entry,
+ struct mlx5e_ipsec_tx *tx,
+ struct mlx5_flow_act *flow_act,
+ struct mlx5_flow_destination *dest,
+ int num_dest)
+{
+ struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule;
+ struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
+ struct mlx5_flow_handle *rule;
+ struct mlx5_flow_spec *spec;
+ int err;
+
+ spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
+ if (!spec)
+ return -ENOMEM;
+
+ setup_fte_no_frags(spec);
+ setup_fte_reg_a_with_tag(spec, sa_entry->kspi);
+
+ rule = mlx5_add_flow_rules(tx->ft.sa_kspi, spec, flow_act, dest, num_dest);
+ if (IS_ERR(rule)) {
+ err = PTR_ERR(rule);
+ mlx5_core_err(mdev, "fail to add TX ipsec kspi rule err=%d\n", err);
+ goto err_add_kspi_flow;
+ }
+ ipsec_rule->kspi_rule = rule;
+ kvfree(spec);
+ return 0;
+
+err_add_kspi_flow:
+ kvfree(spec);
+ return err;
+}
+
+static int tx_add_reqid_ip_rules(struct mlx5e_ipsec_sa_entry *sa_entry,
+ struct mlx5e_ipsec_tx *tx,
+ struct mlx5_flow_act *flow_act,
+ struct mlx5_flow_destination *dest,
+ int num_dest)
+{
+ struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule;
+ struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs;
+ struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
+ struct mlx5_flow_handle *rule;
+ struct mlx5_flow_spec *spec;
+ int err;
+
+ spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
+ if (!spec)
+ return -ENOMEM;
+
+ flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
+
+ if(attrs->reqid) {
+ setup_fte_no_frags(spec);
+ setup_fte_reg_c0(spec, attrs->reqid);
+ rule = mlx5_add_flow_rules(tx->ft.sa, spec, flow_act, dest, num_dest);
+ if (IS_ERR(rule)) {
+ err = PTR_ERR(rule);
+ mlx5_core_err(mdev, "fail to add TX ipsec reqid rule err=%d\n", err);
+ goto err_add_reqid_rule;
+ }
+ ipsec_rule->reqid_rule = rule;
+ memset(spec, 0, sizeof(*spec));
+ }
+
+ if (attrs->family == AF_INET)
+ setup_fte_addr4(spec, &attrs->saddr.a4, &attrs->daddr.a4);
+ else
+ setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6);
+ setup_fte_no_frags(spec);
+
+ rule = mlx5_add_flow_rules(tx->ft.sa, spec, flow_act, dest, num_dest);
+ if (IS_ERR(rule)) {
+ err = PTR_ERR(rule);
+ mlx5_core_err(mdev, "fail to add TX ipsec ip rule err=%d\n", err);
+ goto err_add_ip_rule;
+ }
+ ipsec_rule->rule = rule;
+ kvfree(spec);
+ return 0;
+
+err_add_ip_rule:
+ mlx5_del_flow_rules(&ipsec_rule->reqid_rule);
+err_add_reqid_rule:
+ kvfree(spec);
+ return err;
+}
+
+static int tx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry)
+{
+ struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule;
+ struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs;
+ struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
+ struct mlx5e_ipsec *ipsec = sa_entry->ipsec;
+ struct mlx5_flow_destination dest[2] = {};
+ struct mlx5_flow_act flow_act = {};
+ struct mlx5e_ipsec_tx *tx;
+ struct mlx5_fc *counter;
+ int err;
+
+ tx = tx_ft_get(mdev, ipsec);
+ if (IS_ERR(tx))
+ return PTR_ERR(tx);
+
+ err = setup_pkt_reformat(mdev, attrs, &flow_act);
+ if (err)
+ goto err_pkt_reformat;
+
+ counter = mlx5_fc_create(mdev, false);
+ if (IS_ERR(counter)) {
+ err = PTR_ERR(counter);
+ goto err_add_cnt;
+ }
+
+ flow_act.crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_IPSEC;
+ flow_act.crypto.obj_id = sa_entry->ipsec_obj_id;
+ flow_act.flags |= FLOW_ACT_NO_APPEND;
+ flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_CRYPTO_ENCRYPT |
+ MLX5_FLOW_CONTEXT_ACTION_COUNT;
+
+ if (attrs->drop)
+ flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP;
+ else
+ flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
+
+ dest[0].ft = tx->ft.status;
+ dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
+ dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
+ dest[1].counter_id = mlx5_fc_id(counter);
+
+ err = tx_add_kspi_rule(sa_entry, tx, &flow_act, dest, 2);
+ if (err) {
+ goto err_add_kspi_rule;
+ }
+
+ err = tx_add_reqid_ip_rules(sa_entry, tx, &flow_act, dest, 2);
+ if (err) {
+ goto err_add_reqid_ip_rule;
+ }
+
+ ipsec_rule->fc = counter;
+ ipsec_rule->pkt_reformat = flow_act.pkt_reformat;
+ return 0;
+
+err_add_reqid_ip_rule:
+ mlx5_del_flow_rules(&ipsec_rule->kspi_rule);
+err_add_kspi_rule:
+ mlx5_fc_destroy(mdev, counter);
+err_add_cnt:
+ if (flow_act.pkt_reformat)
+ mlx5_packet_reformat_dealloc(mdev, flow_act.pkt_reformat);
+err_pkt_reformat:
+ tx_ft_put(ipsec);
+ return err;
+}
+
+static int tx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry)
+{
+ struct mlx5_accel_pol_xfrm_attrs *attrs = &pol_entry->attrs;
+ struct mlx5_core_dev *mdev = mlx5e_ipsec_pol2dev(pol_entry);
+ struct mlx5e_ipsec_tx *tx = pol_entry->ipsec->tx;
+ struct mlx5_flow_destination dest[2] = {};
+ struct mlx5_flow_act flow_act = {};
+ struct mlx5_flow_handle *rule;
+ struct mlx5_flow_spec *spec;
+ struct mlx5_flow_table *ft;
+ int err, dstn = 0;
+
+ ft = tx_ft_get_policy(mdev, pol_entry->ipsec, attrs->prio);
+ if (IS_ERR(ft))
+ return PTR_ERR(ft);
+
+ spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
+ if (!spec) {
+ err = -ENOMEM;
+ goto err_alloc;
+ }
+
+ if (attrs->family == AF_INET)
+ setup_fte_addr4(spec, &attrs->saddr.a4, &attrs->daddr.a4);
+ else
+ setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6);
+
+ setup_fte_no_frags(spec);
+ setup_fte_upper_proto_match(spec, &attrs->upspec);
+
+ switch (attrs->action) {
+ case IPSEC_POLICY_IPSEC:
+ flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
+ /*if (!attrs->reqid)
+ break;*/
+ err = setup_modify_header(mdev, attrs->reqid,
+ IPSEC_DIR_OUTBOUND, &flow_act);
+ if (err)
+ goto err_mod_header;
+ break;
+ case IPSEC_POLICY_DISCARD:
+ flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP |
+ MLX5_FLOW_CONTEXT_ACTION_COUNT;
+ dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
+ dest[dstn].counter_id = mlx5_fc_id(tx->fc->drop);
+ dstn++;
+ break;
+ default:
+ err = -EINVAL;
+ goto err_mod_header;
+ }
+
+ flow_act.flags |= FLOW_ACT_NO_APPEND;
+ dest[dstn].ft = tx->ft.sa;
+ dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
+ dstn++;
+ rule = mlx5_add_flow_rules(ft, spec, &flow_act, dest, dstn);
+ if (IS_ERR(rule)) {
+ err = PTR_ERR(rule);
+ mlx5_core_err(mdev, "fail to add TX ipsec rule err=%d\n", err);
+ goto err_action;
+ }
+
+ kvfree(spec);
+ pol_entry->ipsec_rule.rule = rule;
+ pol_entry->ipsec_rule.modify_hdr = flow_act.modify_hdr;
+ return 0;
+
+err_action:
+ if (flow_act.modify_hdr)
+ mlx5_modify_header_dealloc(mdev, flow_act.modify_hdr);
+err_mod_header:
+ kvfree(spec);
+err_alloc:
+ tx_ft_put_policy(pol_entry->ipsec, attrs->prio);
+ return err;
+}
+
+static int rx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry)
+{
+ struct mlx5_accel_pol_xfrm_attrs *attrs = &pol_entry->attrs;
+ struct mlx5_core_dev *mdev = mlx5e_ipsec_pol2dev(pol_entry);
+ struct mlx5e_ipsec *ipsec = pol_entry->ipsec;
+ struct mlx5_flow_destination dest[2];
+ struct mlx5_flow_act flow_act = {};
+ struct mlx5_flow_handle *rule;
+ struct mlx5_flow_spec *spec;
+ struct mlx5_flow_table *ft;
+ struct mlx5e_ipsec_rx *rx;
+ int err, dstn = 0;
+
+ rx = (attrs->family == AF_INET) ? ipsec->rx_ipv4 : ipsec->rx_ipv6;
+ ft = rx->chains ? ipsec_chains_get_table(rx->chains, attrs->prio) : rx->ft.pol;
+ if (IS_ERR(ft))
+ return PTR_ERR(ft);
+
+ spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
+ if (!spec) {
+ err = -ENOMEM;
+ goto err_alloc;
+ }
+
+ if (attrs->family == AF_INET)
+ setup_fte_addr4(spec, &attrs->saddr.a4, &attrs->daddr.a4);
+ else
+ setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6);
+
+ setup_fte_no_frags(spec);
+ setup_fte_upper_proto_match(spec, &attrs->upspec);
+
+ switch (attrs->action) {
+ case IPSEC_POLICY_IPSEC:
+ flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
+ break;
+ case IPSEC_POLICY_DISCARD:
+ flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP | MLX5_FLOW_CONTEXT_ACTION_COUNT;
+ dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
+ dest[dstn].counter_id = mlx5_fc_id(rx->fc->drop);
+ dstn++;
+ break;
+ default:
+ err = -EINVAL;
+ goto err_action;
+ }
+
+ flow_act.flags |= FLOW_ACT_NO_APPEND;
+ dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
+ dest[dstn].ft = rx->ft.sa;
+ dstn++;
+ rule = mlx5_add_flow_rules(ft, spec, &flow_act, dest, dstn);
+ if (IS_ERR(rule)) {
+ err = PTR_ERR(rule);
+ mlx5_core_err(mdev, "Fail to add RX IPsec policy rule err=%d\n", err);
+ goto err_action;
+ }
+
+ kvfree(spec);
+ pol_entry->ipsec_rule.rule = rule;
+ return 0;
+
+err_action:
+ kvfree(spec);
+err_alloc:
+ if (rx->chains)
+ ipsec_chains_put_table(rx->chains, attrs->prio);
+ return err;
+}
+
+static void ipsec_fs_destroy_counters(struct mlx5e_ipsec *ipsec)
+{
+ struct mlx5e_ipsec_rx *rx_ipv4 = ipsec->rx_ipv4;
+ struct mlx5_core_dev *mdev = ipsec->mdev;
+ struct mlx5e_ipsec_tx *tx = ipsec->tx;
+
+ mlx5_fc_destroy(mdev, rx_ipv4->fc->drop);
+ mlx5_fc_destroy(mdev, rx_ipv4->fc->cnt);
+ kfree(rx_ipv4->fc);
+ mlx5_fc_destroy(mdev, tx->fc->drop);
+ mlx5_fc_destroy(mdev, tx->fc->cnt);
+ kfree(tx->fc);
+}
+
+static int ipsec_fs_init_counters(struct mlx5e_ipsec *ipsec)
+{
+ struct mlx5e_ipsec_rx *rx_ipv4 = ipsec->rx_ipv4;
+ struct mlx5e_ipsec_rx *rx_ipv6 = ipsec->rx_ipv6;
+ struct mlx5_core_dev *mdev = ipsec->mdev;
+ struct mlx5e_ipsec_tx *tx = ipsec->tx;
+ struct mlx5e_ipsec_fc *fc;
+ struct mlx5_fc *counter;
+ int err;
+
+ fc = kzalloc(sizeof(*tx->fc), GFP_KERNEL);
+ if (!fc)
+ return -ENOMEM;
+
+ tx->fc = fc;
+ counter = mlx5_fc_create(mdev, false);
+ if (IS_ERR(counter)) {
+ err = PTR_ERR(counter);
+ goto err_tx_fc_alloc;
+ }
+
+ fc->cnt = counter;
+ counter = mlx5_fc_create(mdev, false);
+ if (IS_ERR(counter)) {
+ err = PTR_ERR(counter);
+ goto err_tx_fc_cnt;
+ }
+
+ fc->drop = counter;
+
+ fc = kzalloc(sizeof(*tx->fc), GFP_KERNEL);
+ if (!fc) {
+ err = -ENOMEM;
+ goto err_tx_fc_drop;
+ }
+
+ /* Both IPv4 and IPv6 point to same flow counters struct. */
+ rx_ipv4->fc = fc;
+ rx_ipv6->fc = fc;
+ counter = mlx5_fc_create(mdev, false);
+ if (IS_ERR(counter)) {
+ err = PTR_ERR(counter);
+ goto err_rx_fc_alloc;
+ }
+
+ fc->cnt = counter;
+ counter = mlx5_fc_create(mdev, false);
+ if (IS_ERR(counter)) {
+ err = PTR_ERR(counter);
+ goto err_rx_fc_cnt;
+ }
+
+ fc->drop = counter;
+ return 0;
+
+err_rx_fc_cnt:
+ mlx5_fc_destroy(mdev, rx_ipv4->fc->cnt);
+err_rx_fc_alloc:
+ kfree(rx_ipv4->fc);
+err_tx_fc_drop:
+ mlx5_fc_destroy(mdev, tx->fc->drop);
+err_tx_fc_cnt:
+ mlx5_fc_destroy(mdev, tx->fc->cnt);
+err_tx_fc_alloc:
+ kfree(tx->fc);
+ return err;
+}
+
+static int ipsec_status_rule(struct mlx5_core_dev *mdev,
+ struct mlx5e_ipsec_rx *rx,
+ struct mlx5_flow_destination *dest)
+{
+ u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {};
+ struct mlx5_flow_act flow_act = {};
+ struct mlx5_modify_hdr *modify_hdr;
+ struct mlx5_flow_handle *rule;
+ struct mlx5_flow_spec *spec;
+ int err;
+
+ spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
+ if (!spec)
+ return -ENOMEM;
+
+ /* Action to copy 7 bit ipsec_syndrome to regB[24:30] */
+ MLX5_SET(copy_action_in, action, action_type, MLX5_ACTION_TYPE_COPY);
+ MLX5_SET(copy_action_in, action, src_field, MLX5_ACTION_IN_FIELD_IPSEC_SYNDROME);
+ MLX5_SET(copy_action_in, action, src_offset, 0);
+ MLX5_SET(copy_action_in, action, length, 7);
+ MLX5_SET(copy_action_in, action, dst_field, MLX5_ACTION_IN_FIELD_METADATA_REG_B);
+ MLX5_SET(copy_action_in, action, dst_offset, 24);
+
+ modify_hdr = mlx5_modify_header_alloc(mdev, MLX5_FLOW_NAMESPACE_KERNEL,
+ 1, action);
+
+ if (IS_ERR(modify_hdr)) {
+ err = PTR_ERR(modify_hdr);
+ mlx5_core_err(mdev,
+ "fail to alloc ipsec copy modify_header_id err=%d\n", err);
+ goto out_spec;
+ }
+
+ /* create fte */
+ flow_act.action = MLX5_FLOW_CONTEXT_ACTION_MOD_HDR |
+ MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
+ MLX5_FLOW_CONTEXT_ACTION_COUNT;
+ flow_act.modify_hdr = modify_hdr;
+
+ rule = mlx5_add_flow_rules(rx->ft.status, spec, &flow_act, dest, 2);
+ if (IS_ERR(rule)) {
+ err = PTR_ERR(rule);
+ mlx5_core_err(mdev, "fail to add ipsec rx err copy rule err=%d\n", err);
+ goto out;
+ }
+
+ kvfree(spec);
+ rx->status.rule = rule;
+ rx->status.modify_hdr = modify_hdr;
+ return 0;
+
+out:
+ mlx5_modify_header_dealloc(mdev, modify_hdr);
+out_spec:
+ kvfree(spec);
+ return err;
+}
+
+static void ipsec_fs_rx_roce_rules_destroy(struct mlx5e_ipsec_rx_roce *rx_roce)
+{
+ if (!rx_roce->ns_rdma)
+ return;
+
+ mlx5_del_flow_rules(&rx_roce->roce_miss.rule);
+ mlx5_del_flow_rules(&rx_roce->rule);
+ mlx5_destroy_flow_group(rx_roce->roce_miss.group);
+ mlx5_destroy_flow_group(rx_roce->g);
+}
+
+static void ipsec_fs_rx_catchall_rules_destroy(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_rx *rx)
+{
+ mutex_lock(&rx->ft.mutex);
+ mlx5_del_flow_rules(&rx->sa.rule);
+ mlx5_destroy_flow_group(rx->sa.group);
+ if (rx->chains == NULL) {
+ mlx5_del_flow_rules(&rx->pol.rule);
+ mlx5_destroy_flow_group(rx->pol.group);
+ }
+ mlx5_del_flow_rules(&rx->status.rule);
+ mlx5_modify_header_dealloc(mdev, rx->status.modify_hdr);
+ ipsec_fs_rx_roce_rules_destroy(&rx->roce);
+ mutex_unlock(&rx->ft.mutex);
+}
+
+static void ipsec_fs_rx_roce_table_destroy(struct mlx5e_ipsec_rx_roce *rx_roce)
+{
+ if (!rx_roce->ns_rdma)
+ return;
+
+ mlx5_destroy_flow_table(rx_roce->ft_rdma);
+ mlx5_destroy_flow_table(rx_roce->ft);
+}
+
+static void ipsec_fs_rx_table_destroy(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_rx *rx)
+{
+ mutex_lock(&rx->ft.mutex);
+ if (rx->chains) {
+ ipsec_chains_destroy(rx->chains);
+ } else {
+ mlx5_del_flow_rules(&rx->pol.rule);
+ mlx5_destroy_flow_table(rx->ft.pol);
+ }
+ mlx5_destroy_flow_table(rx->ft.sa);
+ mlx5_destroy_flow_table(rx->ft.status);
+ ipsec_fs_rx_roce_table_destroy(&rx->roce);
+ mutex_unlock(&rx->ft.mutex);
+}
+
+static void ipsec_roce_setup_udp_dport(struct mlx5_flow_spec *spec, u16 dport)
+{
+ spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
+ MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol);
+ MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_UDP);
+ MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.udp_dport);
+ MLX5_SET(fte_match_param, spec->match_value, outer_headers.udp_dport, dport);
+}
+
+static int ipsec_roce_rx_rule_setup(struct mlx5_flow_destination *default_dst,
+ struct mlx5e_ipsec_rx_roce *roce, struct mlx5_core_dev *mdev)
+{
+ struct mlx5_flow_destination dst = {};
+ struct mlx5_flow_act flow_act = {};
+ struct mlx5_flow_handle *rule;
+ struct mlx5_flow_spec *spec;
+ int err = 0;
+
+ spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
+ if (!spec)
+ return -ENOMEM;
+
+ ipsec_roce_setup_udp_dport(spec, ROCE_V2_UDP_DPORT);
+
+ //flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;//not needed it is added in command
+ dst.type = MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE;
+ dst.ft = roce->ft_rdma;
+
+ flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
+ rule = mlx5_add_flow_rules(roce->ft, spec, &flow_act, &dst, 1);
+ if (IS_ERR(rule)) {
+ err = PTR_ERR(rule);
+ mlx5_core_err(mdev, "Fail to add RX roce ipsec rule err=%d\n",
+ err);
+ goto fail_add_rule;
+ }
+
+ roce->rule = rule;
+
+ rule = mlx5_add_flow_rules(roce->ft, NULL, &flow_act, default_dst, 1);
+ if (IS_ERR(rule)) {
+ err = PTR_ERR(rule);
+ mlx5_core_err(mdev, "Fail to add RX roce ipsec miss rule err=%d\n",
+ err);
+ goto fail_add_default_rule;
+ }
+
+ roce->roce_miss.rule = rule;
+
+ kvfree(spec);
+ return 0;
+
+fail_add_default_rule:
+ mlx5_del_flow_rules(&roce->rule);
+fail_add_rule:
+ kvfree(spec);
+ return err;
+}
+
+static int ipsec_roce_rx_rules(struct mlx5e_ipsec_rx *rx, struct mlx5_flow_destination *defdst,
+ struct mlx5_core_dev *mdev)
+{
+ int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
+ struct mlx5_flow_group *g;
+ void *outer_headers_c;
+ u32 *in;
+ int err = 0;
+ int ix = 0;
+ u8 *mc;
+
+ if (!rx->roce.ns_rdma)
+ return 0;
+
+ in = kvzalloc(inlen, GFP_KERNEL);
+ if (!in)
+ return -ENOMEM;
+
+ mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
+ outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, outer_headers);
+ MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_protocol);
+ MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, udp_dport);
+
+ MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
+ MLX5_SET_CFG(in, start_flow_index, ix);
+ ix += 1;
+ MLX5_SET_CFG(in, end_flow_index, ix - 1);
+ g = mlx5_create_flow_group(rx->roce.ft, in);
+ if (IS_ERR(g)) {
+ err = PTR_ERR(g);
+ mlx5_core_err(mdev, "Fail to create ipsec rx roce group at nic err=%d\n", err);
+ goto fail_group;
+ }
+ rx->roce.g = g;
+
+ memset(in, 0, MLX5_ST_SZ_BYTES(create_flow_group_in));
+ MLX5_SET_CFG(in, start_flow_index, ix);
+ ix += 1;
+ MLX5_SET_CFG(in, end_flow_index, ix - 1);
+ g = mlx5_create_flow_group(rx->roce.ft, in);
+ if (IS_ERR(g)) {
+ err = PTR_ERR(g);
+ mlx5_core_err(mdev, "Fail to create ipsec rx roce miss group at nic err=%d\n",
+ err);
+ goto fail_mgroup;
+ }
+ rx->roce.roce_miss.group = g;
+
+ err = ipsec_roce_rx_rule_setup(defdst, &rx->roce, mdev);
+ if (err)
+ goto fail_setup_rule;
+
+ kvfree(in);
+ return 0;
+
+fail_setup_rule:
+ mlx5_destroy_flow_group(rx->roce.roce_miss.group);
+fail_mgroup:
+ mlx5_destroy_flow_group(rx->roce.g);
+fail_group:
+ kvfree(in);
+ return err;
+}
+
+static int ipsec_fs_rx_catchall_rules(struct mlx5e_priv *priv,
+ struct mlx5e_ipsec_rx *rx,
+ struct mlx5_flow_destination *defdst)
+{
+ struct mlx5_core_dev *mdev = priv->mdev;
+ struct mlx5_flow_destination dest[2] = {};
+ int err = 0;
+
+ mutex_lock(&rx->ft.mutex);
+ /* IPsec RoCE RX rules */
+ err = ipsec_roce_rx_rules(rx, defdst, mdev);
+ if (err)
+ goto out;
+
+ /* IPsec Rx IP Status table rule */
+ dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
+ if (rx->roce.ft)
+ dest[0].ft = rx->roce.ft;
+ else
+ dest[0].ft = priv->fts.vlan.t;
+
+ dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
+ dest[1].counter_id = mlx5_fc_id(rx->fc->cnt);
+ err = ipsec_status_rule(mdev, rx, dest);
+ if (err)
+ goto err_roce_rules_destroy;
+
+ if (!rx->chains) {
+ /* IPsec Rx IP policy default miss rule */
+ err = ipsec_miss_create(mdev, rx->ft.pol, &rx->pol, defdst);
+ if (err)
+ goto err_status_rule_destroy;
+ }
+
+ /* FIXME: This is workaround to current design
+ * which installs SA on firt packet. So we need to forward this
+ * packet to the stack. It doesn't work with RoCE and eswitch traffic,
+ */
+ err = ipsec_miss_create(mdev, rx->ft.sa, &rx->sa, defdst);
+ if (err)
+ goto err_status_sa_rule_destroy;
+
+ mutex_unlock(&rx->ft.mutex);
+ return 0;
+
+err_status_sa_rule_destroy:
+ if (!rx->chains) {
+ mlx5_del_flow_rules(&rx->pol.rule);
+ mlx5_destroy_flow_group(rx->pol.group);
+ }
+err_status_rule_destroy:
+ mlx5_del_flow_rules(&rx->status.rule);
+ mlx5_modify_header_dealloc(mdev, rx->status.modify_hdr);
+err_roce_rules_destroy:
+ ipsec_fs_rx_roce_rules_destroy(&rx->roce);
+out:
+ mutex_unlock(&rx->ft.mutex);
+ return err;
+}
+
+static int ipsec_fs_rx_roce_tables_create(struct mlx5e_ipsec_rx *rx,
+ int rx_init_level, int rdma_init_level)
+{
+ struct mlx5_flow_table_attr ft_attr = {};
+ struct mlx5_flow_table *ft;
+ int err = 0;
+
+ if (!rx->roce.ns_rdma)
+ return 0;
+
+ ft_attr.max_fte = 2;
+ ft_attr.level = rx_init_level;
+ ft = mlx5_create_flow_table(rx->ns, &ft_attr);
+ if (IS_ERR(ft)) {
+ err = PTR_ERR(ft);
+ return err;
+ }
+ rx->roce.ft = ft;
+
+ ft_attr.max_fte = 0;
+ ft_attr.level = rdma_init_level;
+ ft = mlx5_create_flow_table(rx->roce.ns_rdma, &ft_attr);
+ if (IS_ERR(ft)) {
+ err = PTR_ERR(ft);
+ goto out;
+ }
+ rx->roce.ft_rdma = ft;
+
+ return 0;
+out:
+ mlx5_destroy_flow_table(rx->roce.ft);
+ rx->roce.ft = NULL;
+ return err;
+}
+
+static int ipsec_fs_rx_table_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_rx *rx,
+ int rx_init_level, int rdma_init_level)
+{
+ struct mlx5_flow_namespace *ns = rx->ns;
+ struct mlx5_flow_table *ft;
+ int err = 0;
+
+ mutex_lock(&rx->ft.mutex);
+
+ /* IPsec Rx IP SA table create */
+ ft = ipsec_rx_ft_create(ns, rx_init_level + 1, 0, 1);
+ if (IS_ERR(ft)) {
+ err = PTR_ERR(ft);
+ goto out;
+ }
+ rx->ft.sa = ft;
+
+ /* IPsec Rx IP Status table create */
+ ft = ipsec_rx_ft_create(ns, rx_init_level + 2, 0, 1);
+ if (IS_ERR(ft)) {
+ err = PTR_ERR(ft);
+ goto err_sa_table_destroy;
+ }
+ rx->ft.status = ft;
+
+ if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PRIO) {
+ rx->chains = ipsec_chains_create(mdev, rx->ft.sa,
+ MLX5_FLOW_NAMESPACE_KERNEL, 0,
+ rx_init_level, &rx->ft.pol);
+ if (IS_ERR(rx->chains)) {
+ err = PTR_ERR(rx->chains);
+ goto err_status_table_destroy;
+ }
+ } else {
+ ft = ipsec_rx_ft_create(ns, rx_init_level, 0, 1);
+ if (IS_ERR(ft)) {
+ err = PTR_ERR(ft);
+ goto err_status_table_destroy;
+ }
+ rx->ft.pol = ft;
+ }
+
+ /* IPsec RoCE RX tables create*/
+ err = ipsec_fs_rx_roce_tables_create(rx, rx_init_level + 3,
+ rdma_init_level);
+ if (err)
+ goto err_pol_table_destroy;
+
+ goto out;
+
+err_pol_table_destroy:
+ mlx5_destroy_flow_table(rx->ft.pol);
+err_status_table_destroy:
+ mlx5_destroy_flow_table(rx->ft.status);
+err_sa_table_destroy:
+ mlx5_destroy_flow_table(rx->ft.sa);
+out:
+ mutex_unlock(&rx->ft.mutex);
+ return err;
+}
+
+#define NIC_RDMA_BOTH_DIRS_CAPS (MLX5_FT_NIC_RX_2_NIC_RX_RDMA | MLX5_FT_NIC_TX_RDMA_2_NIC_TX)
+
+static void mlx5e_accel_ipsec_fs_init_roce(struct mlx5e_ipsec *ipsec)
+{
+ struct mlx5_core_dev *mdev = ipsec->mdev;
+ struct mlx5_flow_namespace *ns;
+
+ if ((MLX5_CAP_GEN_2(ipsec->mdev, flow_table_type_2_type) &
+ NIC_RDMA_BOTH_DIRS_CAPS) != NIC_RDMA_BOTH_DIRS_CAPS) {
+ mlx5_core_dbg(mdev, "Failed to init roce ns, capabilities not supported\n");
+ return;
+ }
+
+ ns = mlx5_get_flow_namespace(ipsec->mdev, MLX5_FLOW_NAMESPACE_RDMA_RX_IPSEC);
+ if (!ns) {
+ mlx5_core_err(mdev, "Failed to init roce rx ns\n");
+ return;
+ }
+
+ ipsec->rx_ipv4->roce.ns_rdma = ns;
+ ipsec->rx_ipv6->roce.ns_rdma = ns;
+
+ ns = mlx5_get_flow_namespace(ipsec->mdev, MLX5_FLOW_NAMESPACE_RDMA_TX_IPSEC);
+ if (!ns) {
+ ipsec->rx_ipv4->roce.ns_rdma = NULL;
+ ipsec->rx_ipv6->roce.ns_rdma = NULL;
+ mlx5_core_err(mdev, "Failed to init roce tx ns\n");
+ return;
+ }
+
+ ipsec->tx->roce.ns = ns;
+}
+
+int mlx5e_accel_ipsec_fs_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry)
+{
+ if (sa_entry->attrs.dir == IPSEC_DIR_OUTBOUND)
+ return tx_add_rule(sa_entry);
+
+ return rx_add_rule(sa_entry);
+}
+
+void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_ipsec_sa_entry *sa_entry)
+{
+ struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule;
+ struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
+
+ mlx5_del_flow_rules(&ipsec_rule->rule);
+ mlx5_del_flow_rules(&ipsec_rule->kspi_rule);
+ if (ipsec_rule->reqid_rule)
+ mlx5_del_flow_rules(&ipsec_rule->reqid_rule);
+ mlx5_fc_destroy(mdev, ipsec_rule->fc);
+ mlx5_packet_reformat_dealloc(mdev, ipsec_rule->pkt_reformat);
+ if (sa_entry->attrs.dir == IPSEC_DIR_OUTBOUND) {
+ tx_ft_put(sa_entry->ipsec);
+ return;
+ }
+
+ if (ipsec_rule->modify_hdr)
+ mlx5_modify_header_dealloc(mdev, ipsec_rule->modify_hdr);
+}
+
+int mlx5e_accel_ipsec_fs_add_pol(struct mlx5e_ipsec_pol_entry *pol_entry)
+{
+ if (pol_entry->attrs.dir == IPSEC_DIR_OUTBOUND)
+ return tx_add_policy(pol_entry);
+
+ return rx_add_policy(pol_entry);
+}
+
+void mlx5e_accel_ipsec_fs_del_pol(struct mlx5e_ipsec_pol_entry *pol_entry)
+{
+ struct mlx5e_ipsec_rule *ipsec_rule = &pol_entry->ipsec_rule;
+ struct mlx5_core_dev *mdev = mlx5e_ipsec_pol2dev(pol_entry);
+
+ mlx5_del_flow_rules(&ipsec_rule->rule);
+
+ if (pol_entry->attrs.dir == IPSEC_DIR_INBOUND) {
+ struct mlx5e_ipsec_rx *rx;
+
+ rx = (pol_entry->attrs.family == AF_INET)
+ ? pol_entry->ipsec->rx_ipv4
+ : pol_entry->ipsec->rx_ipv6;
+ if (rx->chains)
+ ipsec_chains_put_table(rx->chains,
+ pol_entry->attrs.prio);
+ return;
+ }
+
+ if (ipsec_rule->modify_hdr)
+ mlx5_modify_header_dealloc(mdev, ipsec_rule->modify_hdr);
+
+ tx_ft_put_policy(pol_entry->ipsec, pol_entry->attrs.prio);
+}
+
+void mlx5e_accel_ipsec_fs_rx_catchall_rules_destroy(struct mlx5e_priv *priv)
+{
+ /* Check if IPsec supported */
+ if (!priv->ipsec)
+ return;
+
+ ipsec_fs_rx_catchall_rules_destroy(priv->mdev, priv->ipsec->rx_ipv4);
+ ipsec_fs_rx_catchall_rules_destroy(priv->mdev, priv->ipsec->rx_ipv6);
+}
+
+int mlx5e_accel_ipsec_fs_rx_catchall_rules(struct mlx5e_priv *priv)
+{
+ struct mlx5e_ipsec *ipsec = priv->ipsec;
+ struct mlx5_flow_destination dest = {};
+ int err = 0;
+
+ /* Check if IPsec supported */
+ if (!ipsec)
+ return 0;
+
+ dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
+ dest.ft = priv->fts.vlan.t;
+ err = ipsec_fs_rx_catchall_rules(priv, ipsec->rx_ipv6, &dest);
+ if (err)
+ goto out;
+
+ err = ipsec_fs_rx_catchall_rules(priv, ipsec->rx_ipv4, &dest);
+ if (err)
+ ipsec_fs_rx_catchall_rules_destroy(priv->mdev, priv->ipsec->rx_ipv6);
+out:
+ return err;
+}
+
+void mlx5e_accel_ipsec_fs_rx_tables_destroy(struct mlx5e_priv *priv)
+{
+ struct mlx5_core_dev *mdev = priv->mdev;
+ struct mlx5e_ipsec *ipsec = priv->ipsec;
+
+ /* Check if IPsec supported */
+ if (!ipsec)
+ return;
+
+ ipsec_fs_rx_table_destroy(mdev, ipsec->rx_ipv6);
+ ipsec_fs_rx_table_destroy(mdev, ipsec->rx_ipv4);
+}
+
+int mlx5e_accel_ipsec_fs_rx_tables_create(struct mlx5e_priv *priv)
+{
+ struct mlx5e_ipsec *ipsec = priv->ipsec;
+ int err = 0;
+
+ /* Check if IPsec supported */
+ if (!ipsec)
+ return 0;
+
+ err = ipsec_fs_rx_table_create(ipsec->mdev, ipsec->rx_ipv4, 0, 0);
+ if (err)
+ goto out;
+
+ err = ipsec_fs_rx_table_create(ipsec->mdev, ipsec->rx_ipv6, 4, 1);
+ if (err) {
+ ipsec_fs_rx_table_destroy(priv->mdev, ipsec->rx_ipv4);
+ goto out;
+ }
+
+ priv->fts.ipsec_ft = priv->ipsec->rx_ipv4->ft.pol;
+out:
+ return err;
+}
+
+void mlx5e_accel_ipsec_fs_cleanup(struct mlx5e_ipsec *ipsec)
+{
+ WARN_ON(ipsec->tx->ft.refcnt);
+ mutex_destroy(&ipsec->rx_ipv6->ft.mutex);
+ mutex_destroy(&ipsec->rx_ipv4->ft.mutex);
+ mutex_destroy(&ipsec->tx->ft.mutex);
+ ipsec_fs_destroy_counters(ipsec);
+ kfree(ipsec->rx_ipv6);
+ kfree(ipsec->rx_ipv4);
+ kfree(ipsec->tx);
+}
+
+int mlx5e_accel_ipsec_fs_init(struct mlx5e_ipsec *ipsec)
+{
+ struct mlx5_flow_namespace *tns, *rns;
+ int err = -ENOMEM;
+
+ tns = mlx5_get_flow_namespace(ipsec->mdev, MLX5_FLOW_NAMESPACE_EGRESS_IPSEC);
+ if (!tns)
+ return -EOPNOTSUPP;
+
+ rns = mlx5_get_flow_namespace(ipsec->mdev, MLX5_FLOW_NAMESPACE_KERNEL);
+ if (!rns)
+ return -EOPNOTSUPP;
+
+ ipsec->tx = kzalloc(sizeof(*ipsec->tx), GFP_KERNEL);
+ if (!ipsec->tx)
+ return -ENOMEM;
+
+ ipsec->rx_ipv4 = kzalloc(sizeof(*ipsec->rx_ipv4), GFP_KERNEL);
+ if (!ipsec->rx_ipv4)
+ goto err_tx;
+
+ ipsec->rx_ipv6 = kzalloc(sizeof(*ipsec->rx_ipv6), GFP_KERNEL);
+ if (!ipsec->rx_ipv6)
+ goto err_rx_ipv4;
+
+ err = ipsec_fs_init_counters(ipsec);
+ if (err)
+ goto err_rx_ipv6;
+
+ ipsec->tx->ns = tns;
+ mutex_init(&ipsec->tx->ft.mutex);
+ ipsec->rx_ipv4->ns = rns;
+ ipsec->rx_ipv6->ns = rns;
+ mutex_init(&ipsec->rx_ipv4->ft.mutex);
+ mutex_init(&ipsec->rx_ipv6->ft.mutex);
+
+ mlx5e_accel_ipsec_fs_init_roce(ipsec);
+
+ return 0;
+
+err_rx_ipv6:
+ kfree(ipsec->rx_ipv6);
+err_rx_ipv4:
+ kfree(ipsec->rx_ipv4);
+err_tx:
+ kfree(ipsec->tx);
+ return err;
+}
+
+void mlx5e_accel_ipsec_fs_modify(struct mlx5e_ipsec_sa_entry *sa_entry)
+{
+ struct mlx5e_ipsec_sa_entry sa_entry_shadow = {};
+ int err;
+
+ memcpy(&sa_entry_shadow, sa_entry, sizeof(*sa_entry));
+ memset(&sa_entry_shadow.ipsec_rule, 0x00, sizeof(sa_entry->ipsec_rule));
+
+ err = mlx5e_accel_ipsec_fs_add_rule(&sa_entry_shadow);
+ if (err)
+ return;
+ mlx5e_accel_ipsec_fs_del_rule(sa_entry);
+ memcpy(sa_entry, &sa_entry_shadow, sizeof(*sa_entry));
+}
diff --git a/sys/dev/mlx5/mlx5_accel/mlx5_ipsec_offload.c b/sys/dev/mlx5/mlx5_accel/mlx5_ipsec_offload.c
new file mode 100644
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_accel/mlx5_ipsec_offload.c
@@ -0,0 +1,480 @@
+/*-
+ * Copyright (c) 2023 NVIDIA corporation & affiliates.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/types.h>
+#include <netinet/in.h>
+#include <sys/socket.h>
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <net/pfkeyv2.h>
+#include <netipsec/ipsec.h>
+#include <dev/mlx5/mlx5_en/en.h>
+#include <dev/mlx5/crypto.h>
+#include <dev/mlx5/mlx5_accel/ipsec.h>
+
+u32 mlx5_ipsec_device_caps(struct mlx5_core_dev *mdev)
+{
+ u32 caps = 0;
+
+ if (!MLX5_CAP_GEN(mdev, ipsec_offload))
+ return 0;
+
+ if (!MLX5_CAP_GEN(mdev, log_max_dek))
+ return 0;
+
+ if (!(MLX5_CAP_GEN_64(mdev, general_obj_types) &
+ MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_IPSEC))
+ return 0;
+
+ if (!MLX5_CAP_FLOWTABLE_NIC_TX(mdev, ipsec_encrypt) ||
+ !MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ipsec_decrypt))
+ return 0;
+
+ if (!MLX5_CAP_IPSEC(mdev, ipsec_crypto_esp_aes_gcm_128_encrypt) ||
+ !MLX5_CAP_IPSEC(mdev, ipsec_crypto_esp_aes_gcm_128_decrypt))
+ return 0;
+
+ if (MLX5_CAP_IPSEC(mdev, ipsec_full_offload)) {
+ if (MLX5_CAP_FLOWTABLE_NIC_TX(mdev,
+ reformat_add_esp_trasport) &&
+ MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
+ reformat_del_esp_trasport) &&
+ MLX5_CAP_FLOWTABLE_NIC_RX(mdev, decap))
+ caps |= MLX5_IPSEC_CAP_PACKET_OFFLOAD;
+
+ if (MLX5_CAP_FLOWTABLE_NIC_TX(mdev, ignore_flow_level) &&
+ MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ignore_flow_level))
+ caps |= MLX5_IPSEC_CAP_PRIO;
+
+ if (MLX5_CAP_FLOWTABLE_NIC_TX(mdev, reformat_add_esp_transport_over_udp) &&
+ MLX5_CAP_FLOWTABLE_NIC_RX(mdev, reformat_del_esp_transport_over_udp))
+ caps |= MLX5_IPSEC_CAP_ESPINUDP;
+ }
+
+ if (!caps)
+ return 0;
+
+ if (MLX5_CAP_IPSEC(mdev, ipsec_esn))
+ caps |= MLX5_IPSEC_CAP_ESN;
+
+ return caps;
+}
+EXPORT_SYMBOL_GPL(mlx5_ipsec_device_caps);
+
+static void mlx5e_ipsec_packet_setup(void *obj, u32 pdn,
+ struct mlx5_accel_esp_xfrm_attrs *attrs)
+{
+ void *aso_ctx;
+
+ aso_ctx = MLX5_ADDR_OF(ipsec_obj, obj, ipsec_aso);
+ /* ASO context */
+ MLX5_SET(ipsec_obj, obj, ipsec_aso_access_pd, pdn);
+ MLX5_SET(ipsec_obj, obj, full_offload, 1);
+ MLX5_SET(ipsec_aso, aso_ctx, valid, 1);
+ /* MLX5_IPSEC_ASO_REG_C_4_5 is type C register that is used
+ * in flow steering to perform matching against. Please be
+ * aware that this register was chosen arbitrary and can't
+ * be used in other places as long as IPsec packet offload
+ * active.
+ */
+ MLX5_SET(ipsec_obj, obj, aso_return_reg, MLX5_IPSEC_ASO_REG_C_4_5);
+ if (attrs->replay_esn.trigger) {
+ MLX5_SET(ipsec_aso, aso_ctx, esn_event_arm, 1);
+
+ if (attrs->dir == IPSEC_DIR_INBOUND) {
+ MLX5_SET(ipsec_aso, aso_ctx, window_sz,
+ attrs->replay_esn.replay_window);
+ MLX5_SET(ipsec_aso, aso_ctx, mode,
+ MLX5_IPSEC_ASO_REPLAY_PROTECTION);
+ }
+ MLX5_SET(ipsec_aso, aso_ctx, mode_parameter,
+ attrs->replay_esn.esn);
+ }
+
+ switch (attrs->dir) {
+ case IPSEC_DIR_OUTBOUND:
+ MLX5_SET(ipsec_aso, aso_ctx, mode, MLX5_IPSEC_ASO_INC_SN);
+ break;
+ default:
+ break;
+ }
+}
+
+static int mlx5_create_ipsec_obj(struct mlx5e_ipsec_sa_entry *sa_entry)
+{
+ struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs;
+ struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
+ struct aes_gcm_keymat *aes_gcm = &attrs->aes_gcm;
+ u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
+ u32 in[MLX5_ST_SZ_DW(create_ipsec_obj_in)] = {};
+ void *obj, *salt_p, *salt_iv_p;
+ int err;
+
+ obj = MLX5_ADDR_OF(create_ipsec_obj_in, in, ipsec_object);
+
+ /* salt and seq_iv */
+ salt_p = MLX5_ADDR_OF(ipsec_obj, obj, salt);
+ memcpy(salt_p, &aes_gcm->salt, sizeof(aes_gcm->salt));
+
+ MLX5_SET(ipsec_obj, obj, icv_length, MLX5_IPSEC_OBJECT_ICV_LEN_16B);
+ salt_iv_p = MLX5_ADDR_OF(ipsec_obj, obj, implicit_iv);
+ memcpy(salt_iv_p, &aes_gcm->seq_iv, sizeof(aes_gcm->seq_iv));
+
+ /* esn */
+ if (attrs->replay_esn.trigger) {
+ MLX5_SET(ipsec_obj, obj, esn_en, 1);
+ MLX5_SET(ipsec_obj, obj, esn_msb, attrs->replay_esn.esn_msb);
+ MLX5_SET(ipsec_obj, obj, esn_overlap, attrs->replay_esn.overlap);
+ }
+
+ /* enc./dec. key */
+ MLX5_SET(ipsec_obj, obj, dekn, sa_entry->enc_key_id);
+
+ /* general object fields set */
+ MLX5_SET(general_obj_in_cmd_hdr, in, opcode,
+ MLX5_CMD_OP_CREATE_GENERAL_OBJ);
+ MLX5_SET(general_obj_in_cmd_hdr, in, obj_type,
+ MLX5_GENERAL_OBJECT_TYPES_IPSEC);
+
+ mlx5e_ipsec_packet_setup(obj, sa_entry->ipsec->pdn, attrs);
+
+ err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
+ if (!err)
+ sa_entry->ipsec_obj_id =
+ MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
+
+ return err;
+}
+
+static void mlx5_destroy_ipsec_obj(struct mlx5e_ipsec_sa_entry *sa_entry)
+{
+ struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
+ u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {};
+ u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
+
+ MLX5_SET(general_obj_in_cmd_hdr, in, opcode,
+ MLX5_CMD_OP_DESTROY_GENERAL_OBJ);
+ MLX5_SET(general_obj_in_cmd_hdr, in, obj_type,
+ MLX5_GENERAL_OBJECT_TYPES_IPSEC);
+ MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, sa_entry->ipsec_obj_id);
+
+ mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
+}
+
+int mlx5_ipsec_create_sa_ctx(struct mlx5e_ipsec_sa_entry *sa_entry)
+{
+ struct aes_gcm_keymat *aes_gcm = &sa_entry->attrs.aes_gcm;
+ struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
+ int err;
+
+ /* key */
+ err = mlx5_encryption_key_create(mdev, sa_entry->ipsec->pdn,
+ MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_IPSEC,
+ aes_gcm->aes_key,
+ aes_gcm->key_len,
+ &sa_entry->enc_key_id);
+ if (err) {
+ mlx5_core_dbg(mdev, "Failed to create encryption key (err = %d)\n", err);
+ return err;
+ }
+
+ err = mlx5_create_ipsec_obj(sa_entry);
+ if (err) {
+ mlx5_core_dbg(mdev, "Failed to create IPsec object (err = %d)\n", err);
+ goto err_enc_key;
+ }
+
+ return 0;
+
+err_enc_key:
+ mlx5_encryption_key_destroy(mdev, sa_entry->enc_key_id);
+ return err;
+}
+
+void mlx5_ipsec_free_sa_ctx(struct mlx5e_ipsec_sa_entry *sa_entry)
+{
+ struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
+
+ mlx5_destroy_ipsec_obj(sa_entry);
+ mlx5_encryption_key_destroy(mdev, sa_entry->enc_key_id);
+}
+
+static void mlx5e_ipsec_aso_copy(struct mlx5_wqe_aso_ctrl_seg *ctrl,
+ struct mlx5_wqe_aso_ctrl_seg *data)
+{
+ if (!data)
+ return;
+
+ ctrl->data_mask_mode = data->data_mask_mode;
+ ctrl->condition_1_0_operand = data->condition_1_0_operand;
+ ctrl->condition_1_0_offset = data->condition_1_0_offset;
+ ctrl->data_offset_condition_operand = data->data_offset_condition_operand;
+ ctrl->condition_0_data = data->condition_0_data;
+ ctrl->condition_0_mask = data->condition_0_mask;
+ ctrl->condition_1_data = data->condition_1_data;
+ ctrl->condition_1_mask = data->condition_1_mask;
+ ctrl->bitwise_data = data->bitwise_data;
+ ctrl->data_mask = data->data_mask;
+}
+
+static int mlx5e_ipsec_aso_query(struct mlx5e_ipsec_sa_entry *sa_entry,
+ struct mlx5_wqe_aso_ctrl_seg *data)
+{
+ struct mlx5e_ipsec *ipsec = sa_entry->ipsec;
+ struct mlx5e_ipsec_aso *aso = ipsec->aso;
+ struct mlx5_wqe_aso_ctrl_seg *ctrl;
+ struct mlx5_aso_wqe *wqe;
+ unsigned long expires;
+ u8 ds_cnt;
+ int ret;
+
+ spin_lock_bh(&aso->lock);
+ memset(aso->ctx, 0, sizeof(aso->ctx));
+ wqe = mlx5_aso_get_wqe(aso->aso);
+ ds_cnt = DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_DS);
+ mlx5_aso_build_wqe(aso->aso, ds_cnt, wqe, sa_entry->ipsec_obj_id,
+ MLX5_ACCESS_ASO_OPC_MOD_IPSEC);
+
+ ctrl = &wqe->aso_ctrl;
+ ctrl->va_l = cpu_to_be32(lower_32_bits(aso->dma_addr) | ASO_CTRL_READ_EN);
+ ctrl->va_h = cpu_to_be32(upper_32_bits(aso->dma_addr));
+ ctrl->l_key = cpu_to_be32(ipsec->mkey);
+ mlx5e_ipsec_aso_copy(ctrl, data);
+
+ mlx5_aso_post_wqe(aso->aso, false, &wqe->ctrl);
+ expires = jiffies + msecs_to_jiffies(10);
+ do {
+ ret = mlx5_aso_poll_cq(aso->aso, false);
+ if (ret)
+ /* We are in atomic context */
+ udelay(10);
+ } while (ret && time_is_after_jiffies(expires));
+ spin_unlock_bh(&aso->lock);
+
+ return ret;
+}
+
+#define MLX5E_IPSEC_ESN_SCOPE_MID 0x80000000L
+
+static int mlx5_modify_ipsec_obj(struct mlx5e_ipsec_sa_entry *sa_entry,
+ const struct mlx5_accel_esp_xfrm_attrs *attrs)
+{
+ struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
+ u32 in[MLX5_ST_SZ_DW(modify_ipsec_obj_in)] = {};
+ u32 out[MLX5_ST_SZ_DW(query_ipsec_obj_out)];
+ u64 modify_field_select = 0;
+ u64 general_obj_types;
+ void *obj;
+ int err;
+
+ general_obj_types = MLX5_CAP_GEN_64(mdev, general_obj_types);
+ if (!(general_obj_types & MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_IPSEC))
+ return -EINVAL;
+
+ /* general object fields set */
+ MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_QUERY_GENERAL_OBJ);
+ MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_IPSEC);
+ MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, sa_entry->ipsec_obj_id);
+ err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
+ if (err) {
+ mlx5_core_err(mdev, "Query IPsec object failed (Object id %d), err = %d\n",
+ sa_entry->ipsec_obj_id, err);
+ return err;
+ }
+
+ obj = MLX5_ADDR_OF(query_ipsec_obj_out, out, ipsec_object);
+ modify_field_select = MLX5_GET64(ipsec_obj, obj, modify_field_select);
+
+ /* esn */
+ if (!(modify_field_select & MLX5_MODIFY_IPSEC_BITMASK_ESN_OVERLAP) ||
+ !(modify_field_select & MLX5_MODIFY_IPSEC_BITMASK_ESN_MSB))
+ return -EOPNOTSUPP;
+
+ obj = MLX5_ADDR_OF(modify_ipsec_obj_in, in, ipsec_object);
+ MLX5_SET64(ipsec_obj, obj, modify_field_select,
+ MLX5_MODIFY_IPSEC_BITMASK_ESN_OVERLAP |
+ MLX5_MODIFY_IPSEC_BITMASK_ESN_MSB);
+ MLX5_SET(ipsec_obj, obj, esn_msb, attrs->replay_esn.esn_msb);
+ MLX5_SET(ipsec_obj, obj, esn_overlap, attrs->replay_esn.overlap);
+
+ /* general object fields set */
+ MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_MODIFY_GENERAL_OBJ);
+
+ return mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
+}
+
+static void mlx5_accel_esp_modify_xfrm(struct mlx5e_ipsec_sa_entry *sa_entry,
+ const struct mlx5_accel_esp_xfrm_attrs *attrs)
+{
+ int err;
+
+ err = mlx5_modify_ipsec_obj(sa_entry, attrs);
+ if (err)
+ return;
+
+ memcpy(&sa_entry->attrs, attrs, sizeof(sa_entry->attrs));
+}
+
+static void mlx5e_ipsec_aso_update(struct mlx5e_ipsec_sa_entry *sa_entry,
+ struct mlx5_wqe_aso_ctrl_seg *data)
+{
+ data->data_mask_mode = MLX5_ASO_DATA_MASK_MODE_BITWISE_64BIT << 6;
+ data->condition_1_0_operand = MLX5_ASO_ALWAYS_TRUE | MLX5_ASO_ALWAYS_TRUE << 4;
+
+ mlx5e_ipsec_aso_query(sa_entry, data);
+}
+
+#define MLX5_IPSEC_ASO_REMOVE_FLOW_PKT_CNT_OFFSET 0
+
+static void mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry,
+ u32 mode_param)
+{
+ struct mlx5_accel_esp_xfrm_attrs attrs = {};
+ struct mlx5_wqe_aso_ctrl_seg data = {};
+
+ if (mode_param < MLX5E_IPSEC_ESN_SCOPE_MID) {
+ sa_entry->esn_state.esn_msb++;
+ sa_entry->esn_state.overlap = 0;
+ } else {
+ sa_entry->esn_state.overlap = 1;
+ }
+
+ mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &attrs, sa_entry->attrs.dir);
+
+ mlx5_accel_esp_modify_xfrm(sa_entry, &attrs);
+
+ data.data_offset_condition_operand = MLX5_IPSEC_ASO_REMOVE_FLOW_PKT_CNT_OFFSET;
+ data.bitwise_data = cpu_to_be64(BIT_ULL(54));
+ data.data_mask = data.bitwise_data;
+
+ mlx5e_ipsec_aso_update(sa_entry, &data);
+}
+
+static void mlx5e_ipsec_handle_event(struct work_struct *_work)
+{
+ struct mlx5e_ipsec_work *work =
+ container_of(_work, struct mlx5e_ipsec_work, work);
+ struct mlx5e_ipsec_sa_entry *sa_entry = work->data;
+ struct mlx5_accel_esp_xfrm_attrs *attrs;
+ struct mlx5e_ipsec_aso *aso;
+ int ret;
+
+ aso = sa_entry->ipsec->aso;
+ attrs = &sa_entry->attrs;
+
+ /* TODO: Kostia, this event should be locked/protected
+ * from concurent SA delete.
+ */
+ ret = mlx5e_ipsec_aso_query(sa_entry, NULL);
+ if (ret)
+ goto unlock;
+
+ if (attrs->replay_esn.trigger &&
+ !MLX5_GET(ipsec_aso, aso->ctx, esn_event_arm)) {
+ u32 mode_param = MLX5_GET(ipsec_aso, aso->ctx, mode_parameter);
+
+ mlx5e_ipsec_update_esn_state(sa_entry, mode_param);
+ }
+
+unlock:
+ kfree(work);
+}
+
+void mlx5_object_change_event(struct mlx5_core_dev *dev, struct mlx5_eqe *eqe)
+{
+ struct mlx5e_ipsec_sa_entry *sa_entry;
+ struct mlx5_eqe_obj_change *object;
+ struct mlx5e_ipsec_work *work;
+ u16 type;
+
+ object = &eqe->data.obj_change;
+ type = be16_to_cpu(object->obj_type);
+
+ if (type != MLX5_GENERAL_OBJECT_TYPES_IPSEC)
+ return;
+
+ sa_entry = xa_load(&dev->ipsec_sadb, be32_to_cpu(object->obj_id));
+ if (!sa_entry)
+ return;
+
+ work = kmalloc(sizeof(*work), GFP_ATOMIC);
+ if (!work)
+ return;
+
+ INIT_WORK(&work->work, mlx5e_ipsec_handle_event);
+ work->data = sa_entry;
+
+ queue_work(sa_entry->ipsec->wq, &work->work);
+}
+
+int mlx5e_ipsec_aso_init(struct mlx5e_ipsec *ipsec)
+{
+ struct mlx5_core_dev *mdev = ipsec->mdev;
+ struct mlx5e_ipsec_aso *aso;
+ struct device *pdev;
+ int err;
+
+ aso = kzalloc(sizeof(*ipsec->aso), GFP_KERNEL);
+ if (!aso)
+ return -ENOMEM;
+
+ pdev = &mdev->pdev->dev;
+ aso->dma_addr = dma_map_single(pdev, aso->ctx, sizeof(aso->ctx), DMA_BIDIRECTIONAL);
+ err = dma_mapping_error(pdev, aso->dma_addr);
+ if (err)
+ goto err_dma;
+
+ aso->aso = mlx5_aso_create(mdev, ipsec->pdn);
+ if (IS_ERR(aso->aso)) {
+ err = PTR_ERR(aso->aso);
+ goto err_aso_create;
+ }
+
+ spin_lock_init(&aso->lock);
+ ipsec->aso = aso;
+ return 0;
+
+err_aso_create:
+ dma_unmap_single(pdev, aso->dma_addr, sizeof(aso->ctx), DMA_BIDIRECTIONAL);
+err_dma:
+ kfree(aso);
+ return err;
+}
+
+void mlx5e_ipsec_aso_cleanup(struct mlx5e_ipsec *ipsec)
+{
+ struct mlx5_core_dev *mdev = ipsec->mdev;
+ struct mlx5e_ipsec_aso *aso;
+ struct device *pdev;
+
+ aso = ipsec->aso;
+ pdev = &mdev->pdev->dev;
+
+ mlx5_aso_destroy(aso->aso);
+ dma_unmap_single(pdev, aso->dma_addr, sizeof(aso->ctx), DMA_BIDIRECTIONAL);
+ kfree(aso);
+}
diff --git a/sys/dev/mlx5/mlx5_accel/mlx5_ipsec_rxtx.c b/sys/dev/mlx5/mlx5_accel/mlx5_ipsec_rxtx.c
new file mode 100644
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_accel/mlx5_ipsec_rxtx.c
@@ -0,0 +1,77 @@
+/*-
+ * Copyright (c) 2023 NVIDIA corporation & affiliates.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/mbuf.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <netipsec/keydb.h>
+#include <netipsec/ipsec_accel.h>
+#include <dev/mlx5/qp.h>
+#include <dev/mlx5/mlx5_en/en.h>
+#include <dev/mlx5/mlx5_accel/ipsec.h>
+
+#define MLX5_IPSEC_METADATA_HANDLE(ipsec_metadata) (ipsec_metadata & 0xFFFFFF)
+
+int mlx5_accel_ipsec_rx_tag_add(if_t ifp, struct mbuf *mb)
+{
+ struct mlx5e_priv *priv;
+ struct ipsec_accel_in_tag *tag;
+ struct m_tag *mtag;
+
+ priv = if_getsoftc(ifp);
+ if (priv->ipsec == NULL)
+ return (0);
+
+ mtag = m_tag_get(PACKET_TAG_IPSEC_ACCEL_IN, sizeof(*tag), M_NOWAIT);
+ if (mtag == NULL)
+ return -ENOMEM;
+
+ m_tag_prepend(mb, mtag);
+ return 0;
+}
+
+int mlx5e_accel_ipsec_handle_rx_cqe(struct mbuf *mb, struct mlx5_cqe64 *cqe)
+{
+ struct ipsec_accel_in_tag *tag;
+ u32 drv_spi;
+
+ drv_spi = MLX5_IPSEC_METADATA_HANDLE(be32_to_cpu(cqe->ft_metadata));
+ tag = (struct ipsec_accel_in_tag *) m_tag_find(mb, PACKET_TAG_IPSEC_ACCEL_IN, NULL);
+ WARN_ON(tag == NULL);
+ if (tag)
+ tag->drv_spi = drv_spi;
+
+ return 0;
+}
+
+void
+mlx5e_accel_ipsec_handle_tx_wqe(struct mbuf *mb, struct mlx5e_tx_wqe *wqe,
+ struct ipsec_accel_out_tag *tag)
+{
+ wqe->eth.flow_table_metadata = cpu_to_be32(
+ mlx5e_accel_ipsec_get_metadata(tag->drv_spi));
+}
diff --git a/sys/dev/mlx5/mlx5_attic/mlx5_flow_table.c b/sys/dev/mlx5/mlx5_attic/mlx5_flow_table.c
--- a/sys/dev/mlx5/mlx5_attic/mlx5_flow_table.c
+++ b/sys/dev/mlx5/mlx5_attic/mlx5_flow_table.c
@@ -235,9 +235,12 @@
misc_parameters);
void *mc_inner = MLX5_ADDR_OF(fte_match_param, match_criteria,
inner_headers);
+ void *mc_misc2 = MLX5_ADDR_OF(fte_match_param, match_criteria,
+ misc_parameters_2);
int mc_outer_sz = MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4);
int mc_misc_sz = MLX5_ST_SZ_BYTES(fte_match_set_misc);
int mc_inner_sz = MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4);
+ int mc_misc2_sz = MLX5_ST_SZ_BYTES(fte_match_set_misc2);
int i;
for (i = 0; i < ft->num_groups; i++) {
@@ -251,6 +254,9 @@
void *gmc_inner = MLX5_ADDR_OF(fte_match_param,
g->match_criteria,
inner_headers);
+ void *gmc_misc2 = MLX5_ADDR_OF(fte_match_param,
+ g->match_criteria,
+ misc_parameters_2);
if (g->match_criteria_enable != match_criteria_enable)
continue;
@@ -267,6 +273,9 @@
if (memcmp(mc_inner, gmc_inner, mc_inner_sz))
continue;
+ if (match_criteria_enable & MLX5_MATCH_MISC_PARAMETERS_2)
+ if (memcmp(mc_misc2, gmc_misc2, mc_misc2_sz))
+ continue;
*group_ix = i;
return 0;
}
diff --git a/sys/dev/mlx5/mlx5_attic/mlx5_fs_debugfs.c b/sys/dev/mlx5/mlx5_attic/mlx5_fs_debugfs.c
--- a/sys/dev/mlx5/mlx5_attic/mlx5_fs_debugfs.c
+++ b/sys/dev/mlx5/mlx5_attic/mlx5_fs_debugfs.c
@@ -858,6 +858,10 @@
case MLX5_FLOW_DESTINATION_TYPE_VPORT:
return simple_read_from_buffer(buf, count, pos, "vport\n",
sizeof("vport\n") - 1);
+ case MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE:
+ return simple_read_from_buffer(buf, count, pos,
+ "table type\n",
+ sizeof("table type\n") - 1);
}
return 0;
}
@@ -1281,8 +1285,8 @@
if (!dst->debugfs.type)
return -ENOMEM;
- if (dst->dest_attr.type ==
- MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE) {
+ if (dst->dest_attr.type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE ||
+ dst->dest_attr.type == MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE) {
char *path;
if (IS_ERR_OR_NULL(dst->dest_attr.ft->base.debugfs.dir))
@@ -1299,9 +1303,14 @@
*/
snprintf(path, 1024, "../../../../../../../%s",
dst->dest_attr.ft->base.debugfs.dir->d_name.name);
- dst->debugfs.ft = debugfs_create_symlink("flow_table",
- base->debugfs.dir,
- path);
+ if (dst->dest_attr.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE)
+ dst->debugfs.ft = debugfs_create_symlink("flow_table",
+ base->debugfs.dir,
+ path);
+ else
+ dst->debugfs.ft = debugfs_create_symlink("table_type",
+ base->debugfs.dir,
+ path);
if (!dst->debugfs.ft) {
kfree(path);
return -ENOMEM;
diff --git a/sys/dev/mlx5/mlx5_core/eswitch.h b/sys/dev/mlx5/mlx5_core/eswitch.h
--- a/sys/dev/mlx5/mlx5_core/eswitch.h
+++ b/sys/dev/mlx5/mlx5_core/eswitch.h
@@ -31,6 +31,8 @@
#include <linux/if_ether.h>
#include <dev/mlx5/device.h>
+#define MLX5_ESWITCH_MANAGER(mdev) MLX5_CAP_GEN(mdev, eswitch_flow_table)
+
#define MLX5_MAX_UC_PER_VPORT(dev) \
(1 << MLX5_CAP_GEN(dev, log_max_current_uc_list))
@@ -85,15 +87,15 @@
struct vport_ingress {
struct mlx5_flow_table *acl;
struct mlx5_flow_group *drop_grp;
- struct mlx5_flow_rule *drop_rule;
+ struct mlx5_flow_handle *drop_rule;
};
struct vport_egress {
struct mlx5_flow_table *acl;
struct mlx5_flow_group *allowed_vlans_grp;
struct mlx5_flow_group *drop_grp;
- struct mlx5_flow_rule *allowed_vlan;
- struct mlx5_flow_rule *drop_rule;
+ struct mlx5_flow_handle *allowed_vlan;
+ struct mlx5_flow_handle *drop_rule;
};
struct mlx5_vport {
diff --git a/sys/dev/mlx5/mlx5_core/fs_chains.h b/sys/dev/mlx5/mlx5_core/fs_chains.h
new file mode 100644
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_core/fs_chains.h
@@ -0,0 +1,71 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
+/* Copyright (c) 2020 Mellanox Technologies. */
+
+#ifndef __ML5_ESW_CHAINS_H__
+#define __ML5_ESW_CHAINS_H__
+
+#include <dev/mlx5/fs.h>
+
+struct mlx5_fs_chains;
+
+enum mlx5_chains_flags {
+ MLX5_CHAINS_AND_PRIOS_SUPPORTED = BIT(0),
+ MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED = BIT(1),
+ MLX5_CHAINS_FT_TUNNEL_SUPPORTED = BIT(2),
+};
+
+struct mlx5_chains_attr {
+ enum mlx5_flow_namespace_type ns;
+ int fs_base_prio;
+ int fs_base_level;
+ u32 flags;
+ u32 max_grp_num;
+ struct mlx5_flow_table *default_ft;
+};
+
+bool
+mlx5_chains_prios_supported(struct mlx5_fs_chains *chains);
+bool mlx5_chains_ignore_flow_level_supported(struct mlx5_fs_chains *chains);
+bool
+mlx5_chains_backwards_supported(struct mlx5_fs_chains *chains);
+u32
+mlx5_chains_get_prio_range(struct mlx5_fs_chains *chains);
+u32
+mlx5_chains_get_chain_range(struct mlx5_fs_chains *chains);
+u32
+mlx5_chains_get_nf_ft_chain(struct mlx5_fs_chains *chains);
+
+struct mlx5_flow_table *
+mlx5_chains_get_table(struct mlx5_fs_chains *chains, u32 chain, u32 prio,
+ u32 level);
+void
+mlx5_chains_put_table(struct mlx5_fs_chains *chains, u32 chain, u32 prio,
+ u32 level);
+
+struct mlx5_flow_table *
+mlx5_chains_get_tc_end_ft(struct mlx5_fs_chains *chains);
+
+struct mlx5_flow_table *
+mlx5_chains_create_global_table(struct mlx5_fs_chains *chains);
+void
+mlx5_chains_destroy_global_table(struct mlx5_fs_chains *chains,
+ struct mlx5_flow_table *ft);
+
+int
+mlx5_chains_get_chain_mapping(struct mlx5_fs_chains *chains, u32 chain,
+ u32 *chain_mapping);
+int
+mlx5_chains_put_chain_mapping(struct mlx5_fs_chains *chains,
+ u32 chain_mapping);
+
+struct mlx5_fs_chains *
+mlx5_chains_create(struct mlx5_core_dev *dev, struct mlx5_chains_attr *attr);
+void mlx5_chains_destroy(struct mlx5_fs_chains *chains);
+
+void
+mlx5_chains_set_end_ft(struct mlx5_fs_chains *chains,
+ struct mlx5_flow_table *ft);
+void
+mlx5_chains_print_info(struct mlx5_fs_chains *chains);
+
+#endif /* __ML5_ESW_CHAINS_H__ */
diff --git a/sys/dev/mlx5/mlx5_core/fs_cmd.h b/sys/dev/mlx5/mlx5_core/fs_cmd.h
new file mode 100644
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_core/fs_cmd.h
@@ -0,0 +1,120 @@
+/*
+ * Copyright (c) 2015, Mellanox Technologies. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef _MLX5_FS_CMD_
+#define _MLX5_FS_CMD_
+
+#include "fs_core.h"
+
+struct mlx5_flow_cmds {
+ int (*create_flow_table)(struct mlx5_flow_root_namespace *ns,
+ struct mlx5_flow_table *ft,
+ struct mlx5_flow_table_attr *ft_attr,
+ struct mlx5_flow_table *next_ft);
+ int (*destroy_flow_table)(struct mlx5_flow_root_namespace *ns,
+ struct mlx5_flow_table *ft);
+
+ int (*modify_flow_table)(struct mlx5_flow_root_namespace *ns,
+ struct mlx5_flow_table *ft,
+ struct mlx5_flow_table *next_ft);
+
+ int (*create_flow_group)(struct mlx5_flow_root_namespace *ns,
+ struct mlx5_flow_table *ft,
+ u32 *in,
+ struct mlx5_flow_group *fg);
+
+ int (*destroy_flow_group)(struct mlx5_flow_root_namespace *ns,
+ struct mlx5_flow_table *ft,
+ struct mlx5_flow_group *fg);
+
+ int (*create_fte)(struct mlx5_flow_root_namespace *ns,
+ struct mlx5_flow_table *ft,
+ struct mlx5_flow_group *fg,
+ struct fs_fte *fte);
+
+ int (*update_fte)(struct mlx5_flow_root_namespace *ns,
+ struct mlx5_flow_table *ft,
+ struct mlx5_flow_group *fg,
+ int modify_mask,
+ struct fs_fte *fte);
+
+ int (*delete_fte)(struct mlx5_flow_root_namespace *ns,
+ struct mlx5_flow_table *ft,
+ struct fs_fte *fte);
+
+ int (*update_root_ft)(struct mlx5_flow_root_namespace *ns,
+ struct mlx5_flow_table *ft,
+ u32 underlay_qpn,
+ bool disconnect);
+
+ int (*packet_reformat_alloc)(struct mlx5_flow_root_namespace *ns,
+ struct mlx5_pkt_reformat_params *params,
+ enum mlx5_flow_namespace_type namespace,
+ struct mlx5_pkt_reformat *pkt_reformat);
+
+ void (*packet_reformat_dealloc)(struct mlx5_flow_root_namespace *ns,
+ struct mlx5_pkt_reformat *pkt_reformat);
+
+ int (*modify_header_alloc)(struct mlx5_flow_root_namespace *ns,
+ u8 namespace, u8 num_actions,
+ void *modify_actions,
+ struct mlx5_modify_hdr *modify_hdr);
+
+ void (*modify_header_dealloc)(struct mlx5_flow_root_namespace *ns,
+ struct mlx5_modify_hdr *modify_hdr);
+
+ int (*set_peer)(struct mlx5_flow_root_namespace *ns,
+ struct mlx5_flow_root_namespace *peer_ns);
+
+ int (*create_ns)(struct mlx5_flow_root_namespace *ns);
+ int (*destroy_ns)(struct mlx5_flow_root_namespace *ns);
+
+ u32 (*get_capabilities)(struct mlx5_flow_root_namespace *ns,
+ enum fs_flow_table_type ft_type);
+};
+
+int mlx5_cmd_fc_alloc(struct mlx5_core_dev *dev, u32 *id);
+int mlx5_cmd_fc_bulk_alloc(struct mlx5_core_dev *dev,
+ enum mlx5_fc_bulk_alloc_bitmask alloc_bitmask,
+ u32 *id);
+int mlx5_cmd_fc_free(struct mlx5_core_dev *dev, u32 id);
+int mlx5_cmd_fc_query(struct mlx5_core_dev *dev, u32 id,
+ u64 *packets, u64 *bytes);
+
+int mlx5_cmd_fc_get_bulk_query_out_len(int bulk_len);
+int mlx5_cmd_fc_bulk_query(struct mlx5_core_dev *dev, u32 base_id, int bulk_len,
+ u32 *out);
+
+const struct mlx5_flow_cmds *mlx5_fs_cmd_get_default(enum fs_flow_table_type type);
+const struct mlx5_flow_cmds *mlx5_fs_cmd_get_fw_cmds(void);
+
+#endif
diff --git a/sys/dev/mlx5/mlx5_core/fs_core.h b/sys/dev/mlx5/mlx5_core/fs_core.h
--- a/sys/dev/mlx5/mlx5_core/fs_core.h
+++ b/sys/dev/mlx5/mlx5_core/fs_core.h
@@ -1,333 +1,359 @@
-/*-
- * Copyright (c) 2013-2017, Mellanox Technologies, Ltd. All rights reserved.
+/*
+ * Copyright (c) 2015, Mellanox Technologies. All rights reserved.
*
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
*
- * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
*
- * $FreeBSD$
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
*/
#ifndef _MLX5_FS_CORE_
#define _MLX5_FS_CORE_
-#include <asm/atomic.h>
-#include <linux/completion.h>
-#include <linux/mutex.h>
#include <dev/mlx5/fs.h>
-enum fs_type {
+#define FDB_TC_MAX_CHAIN 3
+#define FDB_FT_CHAIN (FDB_TC_MAX_CHAIN + 1)
+#define FDB_TC_SLOW_PATH_CHAIN (FDB_FT_CHAIN + 1)
+
+/* The index of the last real chain (FT) + 1 as chain zero is valid as well */
+#define FDB_NUM_CHAINS (FDB_FT_CHAIN + 1)
+
+#define FDB_TC_MAX_PRIO 16
+#define FDB_TC_LEVELS_PER_PRIO 2
+
+struct mlx5_flow_definer {
+ enum mlx5_flow_namespace_type ns_type;
+ u32 id;
+};
+
+struct mlx5_modify_hdr {
+ enum mlx5_flow_namespace_type ns_type;
+ union {
+ u32 id;
+ };
+};
+
+struct mlx5_pkt_reformat {
+ enum mlx5_flow_namespace_type ns_type;
+ int reformat_type; /* from mlx5_ifc */
+ union {
+ u32 id;
+ };
+};
+
+/* FS_TYPE_PRIO_CHAINS is a PRIO that will have namespaces only,
+ * and those are in parallel to one another when going over them to connect
+ * a new flow table. Meaning the last flow table in a TYPE_PRIO prio in one
+ * parallel namespace will not automatically connect to the first flow table
+ * found in any prio in any next namespace, but skip the entire containing
+ * TYPE_PRIO_CHAINS prio.
+ *
+ * This is used to implement tc chains, each chain of prios is a different
+ * namespace inside a containing TYPE_PRIO_CHAINS prio.
+ */
+
+enum fs_node_type {
FS_TYPE_NAMESPACE,
FS_TYPE_PRIO,
+ FS_TYPE_PRIO_CHAINS,
FS_TYPE_FLOW_TABLE,
FS_TYPE_FLOW_GROUP,
FS_TYPE_FLOW_ENTRY,
FS_TYPE_FLOW_DEST
};
-enum fs_ft_type {
+/**********************************************************************************************************/
+
+
+#define fs_ft_type fs_flow_table_type
+/************************************************************************************************************/
+enum fs_flow_table_type {
FS_FT_NIC_RX = 0x0,
+ FS_FT_NIC_TX = 0x1,
FS_FT_ESW_EGRESS_ACL = 0x2,
FS_FT_ESW_INGRESS_ACL = 0x3,
FS_FT_FDB = 0X4,
- FS_FT_SNIFFER_RX = 0x5,
- FS_FT_SNIFFER_TX = 0x6
+ FS_FT_SNIFFER_RX = 0X5,
+ FS_FT_SNIFFER_TX = 0X6,
+ FS_FT_RDMA_RX = 0X7,
+ FS_FT_RDMA_TX = 0X8,
+ FS_FT_PORT_SEL = 0X9,
+ FS_FT_MAX_TYPE = FS_FT_PORT_SEL,
+};
+
+enum fs_flow_table_op_mod {
+ FS_FT_OP_MOD_NORMAL,
+ FS_FT_OP_MOD_LAG_DEMUX,
};
enum fs_fte_status {
FS_FTE_STATUS_EXISTING = 1UL << 0,
};
-/* Should always be the first variable in the struct */
-struct fs_base {
- struct list_head list;
- struct fs_base *parent;
- enum fs_type type;
- struct kref refcount;
+enum mlx5_flow_steering_mode {
+ MLX5_FLOW_STEERING_MODE_DMFS,
+ MLX5_FLOW_STEERING_MODE_SMFS
+};
+
+enum mlx5_flow_steering_capabilty {
+ MLX5_FLOW_STEERING_CAP_VLAN_PUSH_ON_RX = 1UL << 0,
+ MLX5_FLOW_STEERING_CAP_VLAN_POP_ON_TX = 1UL << 1,
+ MLX5_FLOW_STEERING_CAP_MATCH_RANGES = 1UL << 2,
+};
+
+struct mlx5_flow_steering {
+ struct mlx5_core_dev *dev;
+ enum mlx5_flow_steering_mode mode;
+ struct kmem_cache *fgs_cache;
+ struct kmem_cache *ftes_cache;
+ struct mlx5_flow_root_namespace *root_ns;
+ struct mlx5_flow_root_namespace *fdb_root_ns;
+ struct mlx5_flow_namespace **fdb_sub_ns;
+ struct mlx5_flow_root_namespace **esw_egress_root_ns;
+ struct mlx5_flow_root_namespace **esw_ingress_root_ns;
+ struct mlx5_flow_root_namespace *sniffer_tx_root_ns;
+ struct mlx5_flow_root_namespace *sniffer_rx_root_ns;
+ struct mlx5_flow_root_namespace *rdma_rx_root_ns;
+ struct mlx5_flow_root_namespace *rdma_tx_root_ns;
+ struct mlx5_flow_root_namespace *egress_root_ns;
+ struct mlx5_flow_root_namespace *port_sel_root_ns;
+ int esw_egress_acl_vports;
+ int esw_ingress_acl_vports;
+};
+
+struct fs_node {
+ struct list_head list;
+ struct list_head children;
+ enum fs_node_type type;
+ struct fs_node *parent;
+ struct fs_node *root;
/* lock the node for writing and traversing */
- struct mutex lock;
- struct completion complete;
- atomic_t users_refcount;
- const char *name;
+ struct rw_semaphore lock;
+ refcount_t refcount;
+ bool active;
+ void (*del_hw_func)(struct fs_node *);
+ void (*del_sw_func)(struct fs_node *);
+ atomic_t version;
};
struct mlx5_flow_rule {
- struct fs_base base;
+ struct fs_node node;
+ struct mlx5_flow_table *ft;
struct mlx5_flow_destination dest_attr;
- struct list_head clients_data;
- /*protect clients lits*/
- struct mutex clients_lock;
-};
-
-struct fs_fte {
- struct fs_base base;
- u32 val[MLX5_ST_SZ_DW(fte_match_param)];
- uint32_t dests_size;
- struct list_head dests;
- uint32_t index; /* index in ft */
- struct mlx5_flow_act flow_act;
- u32 sw_action; /* enum mlx5_rule_fwd_action */
- enum fs_fte_status status;
+ /* next_ft should be accessed under chain_lock and only of
+ * destination type is FWD_NEXT_fT.
+ */
+ struct list_head next_ft;
+ u32 sw_action;
};
-struct fs_star_rule {
- struct mlx5_flow_group *fg;
- struct fs_fte *fte;
+struct mlx5_flow_handle {
+ int num_rules;
+ struct mlx5_flow_rule *rule[];
};
+/* Type of children is mlx5_flow_group */
struct mlx5_flow_table {
- struct fs_base base;
- /* sorted list by start_index */
- struct list_head fgs;
+ struct fs_node node;
+ u32 id;
+ u16 vport;
+ unsigned int max_fte;
+ unsigned int level;
+ enum fs_flow_table_type type;
+ enum fs_flow_table_op_mod op_mod;
struct {
bool active;
- unsigned int max_types;
+ unsigned int required_groups;
unsigned int group_size;
- unsigned int num_types;
+ unsigned int num_groups;
unsigned int max_fte;
} autogroup;
- unsigned int max_fte;
- unsigned int level;
- uint32_t id;
- u16 vport;
- enum fs_ft_type type;
- struct fs_star_rule star_rule;
- unsigned int shared_refcount;
+ /* Protect fwd_rules */
+ struct mutex lock;
+ /* FWD rules that point on this flow table */
+ struct list_head fwd_rules;
+ u32 flags;
+ struct xarray fgs_xa;
+ enum mlx5_flow_table_miss_action def_miss_action;
+ struct mlx5_flow_namespace *ns;
};
-enum fs_prio_flags {
- MLX5_CORE_FS_PRIO_SHARED = 1
+struct mlx5_ft_underlay_qp {
+ struct list_head list;
+ u32 qpn;
};
+#define MLX5_FTE_MATCH_PARAM_RESERVED reserved_at_e00
+/* Calculate the fte_match_param length and without the reserved length.
+ * Make sure the reserved field is the last.
+ */
+#define MLX5_ST_SZ_DW_MATCH_PARAM \
+ ((MLX5_BYTE_OFF(fte_match_param, MLX5_FTE_MATCH_PARAM_RESERVED) / sizeof(u32)) + \
+ BUILD_BUG_ON_ZERO(MLX5_ST_SZ_BYTES(fte_match_param) != \
+ MLX5_FLD_SZ_BYTES(fte_match_param, \
+ MLX5_FTE_MATCH_PARAM_RESERVED) +\
+ MLX5_BYTE_OFF(fte_match_param, \
+ MLX5_FTE_MATCH_PARAM_RESERVED)))
+
+/* Type of children is mlx5_flow_rule */
+struct fs_fte {
+ struct fs_node node;
+ u32 val[MLX5_ST_SZ_DW_MATCH_PARAM];
+ u32 dests_size;
+ u32 fwd_dests;
+ u32 index;
+ struct mlx5_flow_context flow_context;
+ struct mlx5_flow_act action;
+ enum fs_fte_status status;
+ struct mlx5_fc *counter;
+ int modify_mask;
+};
+
+/* Type of children is mlx5_flow_table/namespace */
struct fs_prio {
- struct fs_base base;
- struct list_head objs; /* each object is a namespace or ft */
- unsigned int max_ft;
- unsigned int num_ft;
- unsigned int max_ns;
+ struct fs_node node;
+ unsigned int num_levels;
+ unsigned int start_level;
unsigned int prio;
- /*When create shared flow table, this lock should be taken*/
- struct mutex shared_lock;
- u8 flags;
+ unsigned int num_ft;
};
+/* Type of children is fs_prio */
struct mlx5_flow_namespace {
/* parent == NULL => root ns */
- struct fs_base base;
- /* sorted by priority number */
- struct list_head prios; /* list of fs_prios */
- struct list_head list_notifiers;
- struct rw_semaphore notifiers_rw_sem;
- struct rw_semaphore dests_rw_sem;
+ struct fs_node node;
+ enum mlx5_flow_table_miss_action def_miss_action;
+};
+
+struct mlx5_flow_group_mask {
+ u8 match_criteria_enable;
+ u32 match_criteria[MLX5_ST_SZ_DW_MATCH_PARAM];
+};
+
+/* Type of children is fs_fte */
+struct mlx5_flow_group {
+ struct fs_node node;
+ struct mlx5_flow_group_mask mask;
+ u32 start_index;
+ u32 max_ftes;
+ struct ida fte_allocator;
+ u32 id;
+ struct xarray ftes_xa;
};
struct mlx5_flow_root_namespace {
struct mlx5_flow_namespace ns;
- struct mlx5_flow_table *ft_level_0;
- enum fs_ft_type table_type;
+ enum mlx5_flow_steering_mode mode;
+ enum fs_flow_table_type table_type;
struct mlx5_core_dev *dev;
struct mlx5_flow_table *root_ft;
- /* When chaining flow-tables, this lock should be taken */
- struct mutex fs_chain_lock;
+ /* Should be held when chaining flow tables */
+ struct mutex chain_lock;
+ struct list_head underlay_qpns;
+ const struct mlx5_flow_cmds *cmds;
};
-struct mlx5_flow_group {
- struct fs_base base;
- struct list_head ftes;
- struct mlx5_core_fs_mask mask;
- uint32_t start_index;
- uint32_t max_ftes;
- uint32_t num_ftes;
- uint32_t id;
-};
+int mlx5_init_fc_stats(struct mlx5_core_dev *dev);
+void mlx5_cleanup_fc_stats(struct mlx5_core_dev *dev);
+void mlx5_fc_queue_stats_work(struct mlx5_core_dev *dev,
+ struct delayed_work *dwork,
+ unsigned long delay);
+void mlx5_fc_update_sampling_interval(struct mlx5_core_dev *dev,
+ unsigned long interval);
-struct mlx5_flow_handler {
- struct list_head list;
- rule_event_fn add_dst_cb;
- rule_event_fn del_dst_cb;
- void *client_context;
- struct mlx5_flow_namespace *ns;
-};
+const struct mlx5_flow_cmds *mlx5_fs_cmd_get_fw_cmds(void);
-struct fs_client_priv_data {
- struct mlx5_flow_handler *fs_handler;
- struct list_head list;
- void *client_dst_data;
-};
+int mlx5_flow_namespace_set_peer(struct mlx5_flow_root_namespace *ns,
+ struct mlx5_flow_root_namespace *peer_ns);
-struct mlx5_modify_hdr {
- enum mlx5_flow_namespace_type ns_type;
- u32 id;
-};
+int mlx5_flow_namespace_set_mode(struct mlx5_flow_namespace *ns,
+ enum mlx5_flow_steering_mode mode);
-struct mlx5_pkt_reformat {
- enum mlx5_flow_namespace_type ns_type;
- int reformat_type; /* from mlx5_ifc */
- u32 id;
-};
+int mlx5_fs_core_alloc(struct mlx5_core_dev *dev);
+void mlx5_fs_core_free(struct mlx5_core_dev *dev);
+int mlx5_fs_core_init(struct mlx5_core_dev *dev);
+void mlx5_fs_core_cleanup(struct mlx5_core_dev *dev);
-void _fs_remove_node(struct kref *kref);
-#define fs_get_obj(v, _base) {v = container_of((_base), typeof(*v), base); }
-#define fs_get_parent(v, child) {v = (child)->base.parent ? \
- container_of((child)->base.parent, \
- typeof(*v), base) : NULL; }
+int mlx5_fs_egress_acls_init(struct mlx5_core_dev *dev, int total_vports);
+void mlx5_fs_egress_acls_cleanup(struct mlx5_core_dev *dev);
+int mlx5_fs_ingress_acls_init(struct mlx5_core_dev *dev, int total_vports);
+void mlx5_fs_ingress_acls_cleanup(struct mlx5_core_dev *dev);
-#define fs_list_for_each_entry(pos, cond, root) \
- list_for_each_entry(pos, root, base.list) \
- if (!(cond)) {} else
+u32 mlx5_fs_get_capabilities(struct mlx5_core_dev *dev, enum mlx5_flow_namespace_type type);
-#define fs_list_for_each_entry_continue(pos, cond, root) \
- list_for_each_entry_continue(pos, root, base.list) \
- if (!(cond)) {} else
+struct mlx5_flow_root_namespace *find_root(struct fs_node *node);
-#define fs_list_for_each_entry_reverse(pos, cond, root) \
- list_for_each_entry_reverse(pos, root, base.list) \
- if (!(cond)) {} else
+#define fs_get_obj(v, _node) {v = container_of((_node), typeof(*v), node); }
-#define fs_list_for_each_entry_continue_reverse(pos, cond, root) \
- list_for_each_entry_continue_reverse(pos, root, base.list) \
- if (!(cond)) {} else
+#define fs_list_for_each_entry(pos, root) \
+ list_for_each_entry(pos, root, node.list)
-#define fs_for_each_ft(pos, prio) \
- fs_list_for_each_entry(pos, (pos)->base.type == FS_TYPE_FLOW_TABLE, \
- &(prio)->objs)
+#define fs_list_for_each_entry_safe(pos, tmp, root) \
+ list_for_each_entry_safe(pos, tmp, root, node.list)
-#define fs_for_each_ft_reverse(pos, prio) \
- fs_list_for_each_entry_reverse(pos, \
- (pos)->base.type == FS_TYPE_FLOW_TABLE, \
- &(prio)->objs)
+#define fs_for_each_ns_or_ft_reverse(pos, prio) \
+ list_for_each_entry_reverse(pos, &(prio)->node.children, list)
-#define fs_for_each_ns(pos, prio) \
- fs_list_for_each_entry(pos, \
- (pos)->base.type == FS_TYPE_NAMESPACE, \
- &(prio)->objs)
-
-#define fs_for_each_ns_or_ft_reverse(pos, prio) \
- list_for_each_entry_reverse(pos, &(prio)->objs, list) \
- if (!((pos)->type == FS_TYPE_NAMESPACE || \
- (pos)->type == FS_TYPE_FLOW_TABLE)) {} else
-
-#define fs_for_each_ns_or_ft(pos, prio) \
- list_for_each_entry(pos, &(prio)->objs, list) \
- if (!((pos)->type == FS_TYPE_NAMESPACE || \
- (pos)->type == FS_TYPE_FLOW_TABLE)) {} else
-
-#define fs_for_each_ns_or_ft_continue_reverse(pos, prio) \
- list_for_each_entry_continue_reverse(pos, &(prio)->objs, list) \
- if (!((pos)->type == FS_TYPE_NAMESPACE || \
- (pos)->type == FS_TYPE_FLOW_TABLE)) {} else
-
-#define fs_for_each_ns_or_ft_continue(pos, prio) \
- list_for_each_entry_continue(pos, &(prio)->objs, list) \
- if (!((pos)->type == FS_TYPE_NAMESPACE || \
- (pos)->type == FS_TYPE_FLOW_TABLE)) {} else
+#define fs_for_each_ns_or_ft(pos, prio) \
+ list_for_each_entry(pos, (&(prio)->node.children), list)
#define fs_for_each_prio(pos, ns) \
- fs_list_for_each_entry(pos, (pos)->base.type == FS_TYPE_PRIO, \
- &(ns)->prios)
+ fs_list_for_each_entry(pos, &(ns)->node.children)
-#define fs_for_each_prio_reverse(pos, ns) \
- fs_list_for_each_entry_reverse(pos, (pos)->base.type == FS_TYPE_PRIO, \
- &(ns)->prios)
+#define fs_for_each_ns(pos, prio) \
+ fs_list_for_each_entry(pos, &(prio)->node.children)
-#define fs_for_each_prio_continue(pos, ns) \
- fs_list_for_each_entry_continue(pos, (pos)->base.type == FS_TYPE_PRIO, \
- &(ns)->prios)
+#define fs_for_each_ft(pos, prio) \
+ fs_list_for_each_entry(pos, &(prio)->node.children)
-#define fs_for_each_prio_continue_reverse(pos, ns) \
- fs_list_for_each_entry_continue_reverse(pos, \
- (pos)->base.type == FS_TYPE_PRIO, \
- &(ns)->prios)
+#define fs_for_each_ft_safe(pos, tmp, prio) \
+ fs_list_for_each_entry_safe(pos, tmp, &(prio)->node.children)
#define fs_for_each_fg(pos, ft) \
- fs_list_for_each_entry(pos, (pos)->base.type == FS_TYPE_FLOW_GROUP, \
- &(ft)->fgs)
+ fs_list_for_each_entry(pos, &(ft)->node.children)
#define fs_for_each_fte(pos, fg) \
- fs_list_for_each_entry(pos, (pos)->base.type == FS_TYPE_FLOW_ENTRY, \
- &(fg)->ftes)
+ fs_list_for_each_entry(pos, &(fg)->node.children)
+
#define fs_for_each_dst(pos, fte) \
- fs_list_for_each_entry(pos, (pos)->base.type == FS_TYPE_FLOW_DEST, \
- &(fte)->dests)
-
-int mlx5_cmd_fs_create_ft(struct mlx5_core_dev *dev,
- u16 vport, enum fs_ft_type type, unsigned int level,
- unsigned int log_size, const char *name, unsigned int *table_id);
-
-int mlx5_cmd_fs_destroy_ft(struct mlx5_core_dev *dev,
- u16 vport,
- enum fs_ft_type type, unsigned int table_id);
-
-int mlx5_cmd_fs_create_fg(struct mlx5_core_dev *dev,
- u32 *in,
- u16 vport,
- enum fs_ft_type type, unsigned int table_id,
- unsigned int *group_id);
-
-int mlx5_cmd_fs_destroy_fg(struct mlx5_core_dev *dev,
- u16 vport,
- enum fs_ft_type type, unsigned int table_id,
- unsigned int group_id);
-
-
-int mlx5_cmd_fs_set_fte(struct mlx5_core_dev *dev,
- u16 vport,
- enum fs_fte_status *fte_status,
- u32 *match_val,
- enum fs_ft_type type, unsigned int table_id,
- unsigned int index, unsigned int group_id,
- struct mlx5_flow_act *flow_act,
- u32 sw_action, int dest_size,
- struct list_head *dests); /* mlx5_flow_desination */
-
-int mlx5_cmd_fs_delete_fte(struct mlx5_core_dev *dev,
- u16 vport,
- enum fs_fte_status *fte_status,
- enum fs_ft_type type, unsigned int table_id,
- unsigned int index);
-
-int mlx5_cmd_update_root_ft(struct mlx5_core_dev *dev,
- enum fs_ft_type type,
- unsigned int id);
-
-int mlx5_init_fs(struct mlx5_core_dev *dev);
-void mlx5_cleanup_fs(struct mlx5_core_dev *dev);
-void mlx5_fc_update_sampling_interval(struct mlx5_core_dev *dev,
- unsigned long interval);
+ fs_list_for_each_entry(pos, &(fte)->node.children)
+
+#define MLX5_CAP_FLOWTABLE_TYPE(mdev, cap, type) ( \
+ (type == FS_FT_NIC_RX) ? MLX5_CAP_FLOWTABLE_NIC_RX(mdev, cap) : \
+ (type == FS_FT_NIC_TX) ? MLX5_CAP_FLOWTABLE_NIC_TX(mdev, cap) : \
+ (type == FS_FT_ESW_EGRESS_ACL) ? MLX5_CAP_ESW_EGRESS_ACL(mdev, cap) : \
+ (type == FS_FT_ESW_INGRESS_ACL) ? MLX5_CAP_ESW_INGRESS_ACL(mdev, cap) : \
+ (type == FS_FT_FDB) ? MLX5_CAP_ESW_FLOWTABLE_FDB(mdev, cap) : \
+ (type == FS_FT_SNIFFER_RX) ? MLX5_CAP_FLOWTABLE_SNIFFER_RX(mdev, cap) : \
+ (type == FS_FT_SNIFFER_TX) ? MLX5_CAP_FLOWTABLE_SNIFFER_TX(mdev, cap) : \
+ (type == FS_FT_RDMA_RX) ? MLX5_CAP_FLOWTABLE_RDMA_RX(mdev, cap) : \
+ (type == FS_FT_RDMA_TX) ? MLX5_CAP_FLOWTABLE_RDMA_TX(mdev, cap) : \
+ (type == FS_FT_PORT_SEL) ? MLX5_CAP_FLOWTABLE_PORT_SELECTION(mdev, cap) : \
+ (BUILD_BUG_ON_ZERO(FS_FT_PORT_SEL != FS_FT_MAX_TYPE))\
+ )
-int mlx5_cmd_modify_header_alloc(struct mlx5_core_dev *dev,
- enum mlx5_flow_namespace_type namespace,
- u8 num_actions,
- void *modify_actions,
- struct mlx5_modify_hdr *modify_hdr);
-void mlx5_cmd_modify_header_dealloc(struct mlx5_core_dev *dev,
- struct mlx5_modify_hdr *modify_hdr);
-int mlx5_cmd_packet_reformat_alloc(struct mlx5_core_dev *dev,
- struct mlx5_pkt_reformat_params *params,
- enum mlx5_flow_namespace_type namespace,
- struct mlx5_pkt_reformat *pkt_reformat);
-void mlx5_cmd_packet_reformat_dealloc(struct mlx5_core_dev *dev,
- struct mlx5_pkt_reformat *pkt_reformat);
-int mlx5_init_fc_stats(struct mlx5_core_dev *dev);
-void mlx5_cleanup_fc_stats(struct mlx5_core_dev *dev);
-void mlx5_fc_queue_stats_work(struct mlx5_core_dev *dev,
- struct delayed_work *dwork,
- unsigned long delay);
#endif
diff --git a/sys/dev/mlx5/mlx5_core/fs_ft_pool.h b/sys/dev/mlx5/mlx5_core/fs_ft_pool.h
new file mode 100644
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_core/fs_ft_pool.h
@@ -0,0 +1,23 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
+/* Copyright (c) 2021 Mellanox Technologies. */
+
+#ifndef __MLX5_FS_FT_POOL_H__
+#define __MLX5_FS_FT_POOL_H__
+
+#include <linux/module.h>
+#include <dev/mlx5/driver.h>
+#include <dev/mlx5/mlx5_core/fs_core.h>
+#include <linux/compiler.h>
+
+#define POOL_NEXT_SIZE 0
+
+int mlx5_ft_pool_init(struct mlx5_core_dev *dev);
+void mlx5_ft_pool_destroy(struct mlx5_core_dev *dev);
+
+int
+mlx5_ft_pool_get_avail_sz(struct mlx5_core_dev *dev, enum fs_flow_table_type table_type,
+ int desired_size);
+void
+mlx5_ft_pool_put_sz(struct mlx5_core_dev *dev, int sz);
+
+#endif /* __MLX5_FS_FT_POOL_H__ */
diff --git a/sys/dev/mlx5/mlx5_core/fs_tcp.h b/sys/dev/mlx5/mlx5_core/fs_tcp.h
--- a/sys/dev/mlx5/mlx5_core/fs_tcp.h
+++ b/sys/dev/mlx5/mlx5_core/fs_tcp.h
@@ -27,15 +27,15 @@
#define __MLX5E_ACCEL_FS_TCP_H__
struct inpcb;
-struct mlx5_flow_rule;
+struct mlx5_flow_handle;
struct mlx5e_priv;
int mlx5e_accel_fs_tcp_create(struct mlx5e_priv *);
void mlx5e_accel_fs_tcp_destroy(struct mlx5e_priv *);
-struct mlx5_flow_rule *
+struct mlx5_flow_handle *
mlx5e_accel_fs_add_inpcb(struct mlx5e_priv *,
struct inpcb *, uint32_t tirn, uint32_t flow_tag, uint16_t vlan_id);
#define MLX5E_ACCEL_FS_ADD_INPCB_NO_VLAN 0xFFFF
-void mlx5e_accel_fs_del_inpcb(struct mlx5_flow_rule *);
+void mlx5e_accel_fs_del_inpcb(struct mlx5_flow_handle *);
#endif /* __MLX5E_ACCEL_FS_TCP_H__ */
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_cmd.c b/sys/dev/mlx5/mlx5_core/mlx5_cmd.c
--- a/sys/dev/mlx5/mlx5_core/mlx5_cmd.c
+++ b/sys/dev/mlx5/mlx5_core/mlx5_cmd.c
@@ -615,6 +615,9 @@
MLX5_COMMAND_STR_CASE(MODIFY_GENERAL_OBJ);
MLX5_COMMAND_STR_CASE(QUERY_GENERAL_OBJ);
MLX5_COMMAND_STR_CASE(DESTROY_GENERAL_OBJ);
+ MLX5_COMMAND_STR_CASE(ALLOC_FLOW_COUNTER);
+ MLX5_COMMAND_STR_CASE(DEALLOC_FLOW_COUNTER);
+ MLX5_COMMAND_STR_CASE(QUERY_FLOW_COUNTER);
default: return "unknown command opcode";
}
}
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_core.h b/sys/dev/mlx5/mlx5_core/mlx5_core.h
--- a/sys/dev/mlx5/mlx5_core/mlx5_core.h
+++ b/sys/dev/mlx5/mlx5_core/mlx5_core.h
@@ -165,4 +165,14 @@
u8 mlx5_get_nic_state(struct mlx5_core_dev *dev);
void mlx5_set_nic_state(struct mlx5_core_dev *dev, u8 state);
+/************************************************ TESTTEST********************************************/
+static inline int mlx5_init_fs(struct mlx5_core_dev *dev)
+{
+ return 0;
+}
+
+static inline int mlx5_cleanup_fs(struct mlx5_core_dev *dev)
+{
+ return 0;
+}
#endif /* __MLX5_CORE_H__ */
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_crypto.c b/sys/dev/mlx5/mlx5_core/mlx5_crypto.c
new file mode 100644
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_core/mlx5_crypto.c
@@ -0,0 +1,95 @@
+/*-
+ * Copyright (c) 2019-2021, Mellanox Technologies, Ltd. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include "opt_rss.h"
+#include "opt_ratelimit.h"
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+
+#include <dev/mlx5/driver.h>
+#include <dev/mlx5/crypto.h>
+
+int mlx5_encryption_key_create(struct mlx5_core_dev *mdev, u32 pdn, u32 key_type,
+ const void *p_key, u32 key_len, u32 *p_obj_id)
+{
+ u32 in[MLX5_ST_SZ_DW(create_encryption_key_in)] = {};
+ u32 out[MLX5_ST_SZ_DW(create_encryption_key_out)] = {};
+ u64 general_obj_types;
+ int err;
+
+ general_obj_types = MLX5_CAP_GEN_64(mdev, general_obj_types);
+ if (!(general_obj_types & MLX5_HCA_CAP_GENERAL_OBJ_TYPES_ENCRYPTION_KEY))
+ return -EINVAL;
+
+ switch (key_len) {
+ case 128 / 8:
+ memcpy(MLX5_ADDR_OF(create_encryption_key_in, in,
+ encryption_key_object.key[4]), p_key, 128 / 8);
+ MLX5_SET(create_encryption_key_in, in, encryption_key_object.pd, pdn);
+ MLX5_SET(create_encryption_key_in, in, encryption_key_object.key_size,
+ MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_KEY_SIZE_128);
+ MLX5_SET(create_encryption_key_in, in, encryption_key_object.key_type,
+ key_type);
+ break;
+ case 256 / 8:
+ memcpy(MLX5_ADDR_OF(create_encryption_key_in, in,
+ encryption_key_object.key[0]), p_key, 256 / 8);
+ MLX5_SET(create_encryption_key_in, in, encryption_key_object.pd, pdn);
+ MLX5_SET(create_encryption_key_in, in, encryption_key_object.key_size,
+ MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_KEY_SIZE_256);
+ MLX5_SET(create_encryption_key_in, in, encryption_key_object.key_type,
+ key_type);
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ MLX5_SET(create_encryption_key_in, in, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJ);
+ MLX5_SET(create_encryption_key_in, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY);
+
+ err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
+ if (err == 0)
+ *p_obj_id = MLX5_GET(create_encryption_key_out, out, obj_id);
+
+ /* avoid leaking key on the stack */
+ explicit_bzero(in, sizeof(in));
+
+ return err;
+}
+
+int mlx5_encryption_key_destroy(struct mlx5_core_dev *mdev, u32 oid)
+{
+ u32 in[MLX5_ST_SZ_DW(destroy_encryption_key_in)] = {};
+ u32 out[MLX5_ST_SZ_DW(destroy_encryption_key_out)] = {};
+
+ MLX5_SET(destroy_encryption_key_in, in, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJ);
+ MLX5_SET(destroy_encryption_key_in, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY);
+ MLX5_SET(destroy_encryption_key_in, in, obj_id, oid);
+
+ return mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
+}
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_eq.c b/sys/dev/mlx5/mlx5_core/mlx5_eq.c
--- a/sys/dev/mlx5/mlx5_core/mlx5_eq.c
+++ b/sys/dev/mlx5/mlx5_core/mlx5_eq.c
@@ -33,6 +33,7 @@
#include <dev/mlx5/mlx5_ifc.h>
#include <dev/mlx5/mlx5_core/mlx5_core.h>
#include <dev/mlx5/mlx5_core/eswitch.h>
+#include <dev/mlx5/mlx5_accel/ipsec.h>
#ifdef RSS
#include <net/rss_config.h>
@@ -165,6 +166,8 @@
return "MLX5_EVENT_TYPE_CODING_DCBX_CHANGE_EVENT";
case MLX5_EVENT_TYPE_CODING_GENERAL_NOTIFICATION_EVENT:
return "MLX5_EVENT_TYPE_CODING_GENERAL_NOTIFICATION_EVENT";
+ case MLX5_EVENT_TYPE_OBJECT_CHANGE:
+ return "MLX5_EVENT_TYPE_OBJECT_CHANGE";
default:
return "Unrecognized event";
}
@@ -366,6 +369,10 @@
mlx5_temp_warning_event(dev, eqe);
break;
+ case MLX5_EVENT_TYPE_OBJECT_CHANGE:
+ mlx5_object_change_event(dev, eqe);
+ break;
+
default:
mlx5_core_warn(dev, "Unhandled event 0x%x on EQ 0x%x\n",
eqe->type, eq->eqn);
@@ -567,6 +574,10 @@
MLX5_EVENT_TYPE_CODING_GENERAL_NOTIFICATION_EVENT);
}
+ if (mlx5_ipsec_device_caps(dev) & MLX5_IPSEC_CAP_PACKET_OFFLOAD)
+ async_event_mask |=
+ (1ull << MLX5_EVENT_TYPE_OBJECT_CHANGE);
+
err = mlx5_create_map_eq(dev, &table->cmd_eq, MLX5_EQ_VEC_CMD,
MLX5_NUM_CMD_EQE, 1ull << MLX5_EVENT_TYPE_CMD);
if (err) {
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_eswitch.c b/sys/dev/mlx5/mlx5_core/mlx5_eswitch.c
--- a/sys/dev/mlx5/mlx5_core/mlx5_eswitch.c
+++ b/sys/dev/mlx5/mlx5_core/mlx5_eswitch.c
@@ -65,7 +65,7 @@
/* E-Switch MC FDB table hash node */
struct esw_mc_addr { /* SRIOV only */
struct l2addr_node node;
- struct mlx5_flow_rule *uplink_rule; /* Forward to uplink rule */
+ struct mlx5_flow_handle *uplink_rule; /* Forward to uplink rule */
u32 refcnt;
};
@@ -74,7 +74,7 @@
struct l2addr_node node;
u8 action;
u32 vport;
- struct mlx5_flow_rule *flow_rule; /* SRIOV only */
+ struct mlx5_flow_handle *flow_rule; /* SRIOV only */
};
enum {
@@ -216,59 +216,54 @@
}
/* E-Switch FDB */
-static struct mlx5_flow_rule *
+static struct mlx5_flow_handle *
esw_fdb_set_vport_rule(struct mlx5_eswitch *esw, u8 mac[ETH_ALEN], u32 vport)
{
- int match_header = MLX5_MATCH_OUTER_HEADERS;
struct mlx5_flow_destination dest;
- struct mlx5_flow_rule *flow_rule = NULL;
+ struct mlx5_flow_handle *flow_rule = NULL;
struct mlx5_flow_act flow_act = {};
- u32 *match_v;
- u32 *match_c;
+ struct mlx5_flow_spec *spec;
u8 *dmac_v;
u8 *dmac_c;
- match_v = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), GFP_KERNEL);
- match_c = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), GFP_KERNEL);
- if (!match_v || !match_c) {
- printf("mlx5_core: WARN: ""FDB: Failed to alloc match parameters\n");
+ spec = kzalloc(sizeof(*spec), GFP_KERNEL);
+ if (!spec) {
+ printf("mlx5_core: WARN: ""FDB: Failed to alloc flow spec\n");
goto out;
}
- dmac_v = MLX5_ADDR_OF(fte_match_param, match_v,
+ dmac_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
outer_headers.dmac_47_16);
- dmac_c = MLX5_ADDR_OF(fte_match_param, match_c,
+ dmac_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
outer_headers.dmac_47_16);
+ spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
ether_addr_copy(dmac_v, mac);
/* Match criteria mask */
memset(dmac_c, 0xff, 6);
dest.type = MLX5_FLOW_CONTEXT_DEST_TYPE_VPORT;
- dest.vport_num = vport;
+ dest.vport.num = vport;
esw_debug(esw->dev,
"\tFDB add rule dmac_v(%pM) dmac_c(%pM) -> vport(%d)\n",
dmac_v, dmac_c, vport);
+ flow_act.action = MLX5_FLOW_RULE_FWD_ACTION_DEST;
flow_rule =
- mlx5_add_flow_rule(esw->fdb_table.fdb,
- match_header,
- match_c,
- match_v,
- MLX5_FLOW_RULE_FWD_ACTION_DEST,
- &flow_act, &dest);
+ mlx5_add_flow_rules(esw->fdb_table.fdb, spec,
+ &flow_act, &dest, 1);
if (IS_ERR_OR_NULL(flow_rule)) {
printf("mlx5_core: WARN: ""FDB: Failed to add flow rule: dmac_v(%pM) dmac_c(%pM) -> vport(%d), err(%ld)\n", dmac_v, dmac_c, vport, PTR_ERR(flow_rule));
flow_rule = NULL;
}
out:
- kfree(match_v);
- kfree(match_c);
+ kfree(spec);
return flow_rule;
}
static int esw_create_fdb_table(struct mlx5_eswitch *esw)
{
int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
+ struct mlx5_flow_table_attr ft_attr = {};
struct mlx5_core_dev *dev = esw->dev;
struct mlx5_flow_namespace *root_ns;
struct mlx5_flow_table *fdb;
@@ -296,7 +291,9 @@
/* (-2) Since MaorG said so .. */
table_size = BIT(MLX5_CAP_ESW_FLOWTABLE_FDB(dev, log_max_ft_size)) - 2;
- fdb = mlx5_create_flow_table(root_ns, 0, "FDB", table_size);
+ ft_attr.prio = 0;
+ ft_attr.max_fte = table_size;
+ fdb = mlx5_create_flow_table(root_ns, &ft_attr);
if (IS_ERR_OR_NULL(fdb)) {
err = PTR_ERR(fdb);
esw_warn(dev, "Failed to create FDB Table err %d\n", err);
@@ -398,7 +395,7 @@
mlx5_mpfs_del_mac(esw->dev, esw_uc->table_index);
- mlx5_del_flow_rule(&vaddr->flow_rule);
+ mlx5_del_flow_rules(&vaddr->flow_rule);
l2addr_hash_del(esw_uc);
return 0;
@@ -457,12 +454,12 @@
vport, mac, vaddr->flow_rule, esw_mc->refcnt,
esw_mc->uplink_rule);
- mlx5_del_flow_rule(&vaddr->flow_rule);
+ mlx5_del_flow_rules(&vaddr->flow_rule);
if (--esw_mc->refcnt)
return 0;
- mlx5_del_flow_rule(&esw_mc->uplink_rule);
+ mlx5_del_flow_rules(&esw_mc->uplink_rule);
l2addr_hash_del(esw_mc);
return 0;
@@ -603,13 +600,13 @@
struct mlx5_vport *vport)
{
int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
+ struct mlx5_flow_table_attr ft_attr = {};
struct mlx5_flow_group *vlan_grp = NULL;
struct mlx5_flow_group *drop_grp = NULL;
struct mlx5_core_dev *dev = esw->dev;
struct mlx5_flow_namespace *root_ns;
struct mlx5_flow_table *acl;
void *match_criteria;
- char table_name[32];
u32 *flow_group_in;
int table_size = 2;
int err = 0;
@@ -630,8 +627,8 @@
if (!flow_group_in)
return;
- snprintf(table_name, 32, "egress_%d", vport->vport);
- acl = mlx5_create_vport_flow_table(root_ns, vport->vport, 0, table_name, table_size);
+ ft_attr.max_fte = table_size;
+ acl = mlx5_create_vport_flow_table(root_ns, &ft_attr, vport->vport);
if (IS_ERR_OR_NULL(acl)) {
err = PTR_ERR(acl);
esw_warn(dev, "Failed to create E-Switch vport[%d] egress flow Table, err(%d)\n",
@@ -679,8 +676,8 @@
static void esw_vport_cleanup_egress_rules(struct mlx5_eswitch *esw,
struct mlx5_vport *vport)
{
- mlx5_del_flow_rule(&vport->egress.allowed_vlan);
- mlx5_del_flow_rule(&vport->egress.drop_rule);
+ mlx5_del_flow_rules(&vport->egress.allowed_vlan);
+ mlx5_del_flow_rules(&vport->egress.drop_rule);
}
static void esw_vport_disable_egress_acl(struct mlx5_eswitch *esw,
@@ -704,12 +701,12 @@
struct mlx5_vport *vport)
{
int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
+ struct mlx5_flow_table_attr ft_attr = {};
struct mlx5_core_dev *dev = esw->dev;
struct mlx5_flow_namespace *root_ns;
struct mlx5_flow_table *acl;
struct mlx5_flow_group *g;
void *match_criteria;
- char table_name[32];
u32 *flow_group_in;
int table_size = 1;
int err = 0;
@@ -730,8 +727,8 @@
if (!flow_group_in)
return;
- snprintf(table_name, 32, "ingress_%d", vport->vport);
- acl = mlx5_create_vport_flow_table(root_ns, vport->vport, 0, table_name, table_size);
+ ft_attr.max_fte = table_size;
+ acl = mlx5_create_vport_flow_table(root_ns, &ft_attr, vport->vport);
if (IS_ERR_OR_NULL(acl)) {
err = PTR_ERR(acl);
esw_warn(dev, "Failed to create E-Switch vport[%d] ingress flow Table, err(%d)\n",
@@ -764,7 +761,7 @@
static void esw_vport_cleanup_ingress_rules(struct mlx5_eswitch *esw,
struct mlx5_vport *vport)
{
- mlx5_del_flow_rule(&vport->ingress.drop_rule);
+ mlx5_del_flow_rules(&vport->ingress.drop_rule);
}
static void esw_vport_disable_ingress_acl(struct mlx5_eswitch *esw,
@@ -786,9 +783,7 @@
struct mlx5_vport *vport)
{
struct mlx5_flow_act flow_act = {};
- struct mlx5_flow_destination dest;
- u32 *match_v;
- u32 *match_c;
+ struct mlx5_flow_spec *spec;
int err = 0;
if (IS_ERR_OR_NULL(vport->ingress.acl)) {
@@ -807,35 +802,28 @@
"vport[%d] configure ingress rules, vlan(%d) qos(%d)\n",
vport->vport, vport->vlan, vport->qos);
- match_v = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), GFP_KERNEL);
- match_c = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), GFP_KERNEL);
- if (!match_v || !match_c) {
+ spec = kzalloc(sizeof(*spec), GFP_KERNEL);
+ if (!spec) {
err = -ENOMEM;
esw_warn(esw->dev, "vport[%d] configure ingress rules failed, err(%d)\n",
vport->vport, err);
goto out;
}
- MLX5_SET_TO_ONES(fte_match_param, match_c, outer_headers.cvlan_tag);
- MLX5_SET_TO_ONES(fte_match_param, match_v, outer_headers.cvlan_tag);
-
- dest.type = MLX5_FLOW_CONTEXT_DEST_TYPE_VPORT;
- dest.vport_num = vport->vport;
+ MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.cvlan_tag);
+ MLX5_SET_TO_ONES(fte_match_param, spec->match_value, outer_headers.cvlan_tag);
+ flow_act.action = MLX5_FLOW_RULE_FWD_ACTION_DROP;
+ spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
vport->ingress.drop_rule =
- mlx5_add_flow_rule(vport->ingress.acl,
- MLX5_MATCH_OUTER_HEADERS,
- match_c,
- match_v,
- MLX5_FLOW_RULE_FWD_ACTION_DROP,
- &flow_act, &dest);
+ mlx5_add_flow_rules(vport->ingress.acl, spec,
+ &flow_act, NULL, 0);
if (IS_ERR_OR_NULL(vport->ingress.drop_rule)) {
err = PTR_ERR(vport->ingress.drop_rule);
printf("mlx5_core: WARN: ""vport[%d] configure ingress rules, err(%d)\n", vport->vport, err);
vport->ingress.drop_rule = NULL;
}
out:
- kfree(match_v);
- kfree(match_c);
+ kfree(spec);
return err;
}
@@ -843,9 +831,7 @@
struct mlx5_vport *vport)
{
struct mlx5_flow_act flow_act = {};
- struct mlx5_flow_destination dest;
- u32 *match_v;
- u32 *match_c;
+ struct mlx5_flow_spec *spec;
int err = 0;
if (IS_ERR_OR_NULL(vport->egress.acl)) {
@@ -863,9 +849,8 @@
"vport[%d] configure egress rules, vlan(%d) qos(%d)\n",
vport->vport, vport->vlan, vport->qos);
- match_v = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), GFP_KERNEL);
- match_c = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), GFP_KERNEL);
- if (!match_v || !match_c) {
+ spec = kzalloc(sizeof(*spec), GFP_KERNEL);
+ if (!spec) {
err = -ENOMEM;
esw_warn(esw->dev, "vport[%d] configure egress rules failed, err(%d)\n",
vport->vport, err);
@@ -873,21 +858,17 @@
}
/* Allowed vlan rule */
- MLX5_SET_TO_ONES(fte_match_param, match_c, outer_headers.cvlan_tag);
- MLX5_SET_TO_ONES(fte_match_param, match_v, outer_headers.cvlan_tag);
- MLX5_SET_TO_ONES(fte_match_param, match_c, outer_headers.first_vid);
- MLX5_SET(fte_match_param, match_v, outer_headers.first_vid, vport->vlan);
+ MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.cvlan_tag);
+ MLX5_SET_TO_ONES(fte_match_param, spec->match_value, outer_headers.cvlan_tag);
+ MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.first_vid);
+ MLX5_SET(fte_match_param, spec->match_value, outer_headers.first_vid, vport->vlan);
- dest.type = MLX5_FLOW_CONTEXT_DEST_TYPE_VPORT;
- dest.vport_num = vport->vport;
+ spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
+ flow_act.action = MLX5_FLOW_RULE_FWD_ACTION_ALLOW;
vport->egress.allowed_vlan =
- mlx5_add_flow_rule(vport->egress.acl,
- MLX5_MATCH_OUTER_HEADERS,
- match_c,
- match_v,
- MLX5_FLOW_RULE_FWD_ACTION_ALLOW,
- &flow_act, &dest);
+ mlx5_add_flow_rules(vport->egress.acl, spec,
+ &flow_act, NULL, 0);
if (IS_ERR_OR_NULL(vport->egress.allowed_vlan)) {
err = PTR_ERR(vport->egress.allowed_vlan);
printf("mlx5_core: WARN: ""vport[%d] configure egress allowed vlan rule failed, err(%d)\n", vport->vport, err);
@@ -895,24 +876,17 @@
goto out;
}
- /* Drop others rule (star rule) */
- memset(match_c, 0, MLX5_ST_SZ_BYTES(fte_match_param));
- memset(match_v, 0, MLX5_ST_SZ_BYTES(fte_match_param));
+ flow_act.action = MLX5_FLOW_RULE_FWD_ACTION_DROP;
vport->egress.drop_rule =
- mlx5_add_flow_rule(vport->egress.acl,
- 0,
- match_c,
- match_v,
- MLX5_FLOW_RULE_FWD_ACTION_DROP,
- &flow_act, &dest);
+ mlx5_add_flow_rules(vport->egress.acl, NULL,
+ &flow_act, NULL, 0);
if (IS_ERR_OR_NULL(vport->egress.drop_rule)) {
err = PTR_ERR(vport->egress.drop_rule);
printf("mlx5_core: WARN: ""vport[%d] configure egress drop rule failed, err(%d)\n", vport->vport, err);
vport->egress.drop_rule = NULL;
}
out:
- kfree(match_v);
- kfree(match_c);
+ kfree(spec);
return err;
}
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fc_cmd.h b/sys/dev/mlx5/mlx5_core/mlx5_fc_cmd.h
deleted file mode 100644
--- a/sys/dev/mlx5/mlx5_core/mlx5_fc_cmd.h
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Copyright (c) 2023, NVIDIA Technologies. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#ifndef _MLX5_FC_CMD_
-#define _MLX5_FC_CMD_
-
-#include "fs_core.h"
-
-int mlx5_cmd_fc_alloc(struct mlx5_core_dev *dev, u32 *id);
-int mlx5_cmd_fc_bulk_alloc(struct mlx5_core_dev *dev,
- enum mlx5_fc_bulk_alloc_bitmask alloc_bitmask,
- u32 *id);
-int mlx5_cmd_fc_free(struct mlx5_core_dev *dev, u32 id);
-int mlx5_cmd_fc_query(struct mlx5_core_dev *dev, u32 id,
- u64 *packets, u64 *bytes);
-
-int mlx5_cmd_fc_bulk_query(struct mlx5_core_dev *dev, u32 base_id, int bulk_len,
- u32 *out);
-static inline int mlx5_cmd_fc_get_bulk_query_out_len(int bulk_len)
-{
- return MLX5_ST_SZ_BYTES(query_flow_counter_out) +
- MLX5_ST_SZ_BYTES(traffic_counter) * bulk_len;
-}
-
-#endif
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fc_cmd.c b/sys/dev/mlx5/mlx5_core/mlx5_fc_cmd.c
deleted file mode 100644
--- a/sys/dev/mlx5/mlx5_core/mlx5_fc_cmd.c
+++ /dev/null
@@ -1,102 +0,0 @@
-/*-
- * Copyright (c) 2022 NVIDIA corporation & affiliates.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * $FreeBSD$
- */
-
-#include <dev/mlx5/driver.h>
-#include <dev/mlx5/device.h>
-#include <dev/mlx5/mlx5_ifc.h>
-#include <dev/mlx5/mlx5_core/mlx5_fc_cmd.h>
-#include <dev/mlx5/mlx5_core/mlx5_core.h>
-
-int mlx5_cmd_fc_bulk_alloc(struct mlx5_core_dev *dev,
- enum mlx5_fc_bulk_alloc_bitmask alloc_bitmask,
- u32 *id)
-{
- u32 out[MLX5_ST_SZ_DW(alloc_flow_counter_out)] = {};
- u32 in[MLX5_ST_SZ_DW(alloc_flow_counter_in)] = {};
- int err;
-
- MLX5_SET(alloc_flow_counter_in, in, opcode,
- MLX5_CMD_OP_ALLOC_FLOW_COUNTER);
- MLX5_SET(alloc_flow_counter_in, in, flow_counter_bulk, alloc_bitmask);
-
- err = mlx5_cmd_exec_inout(dev, alloc_flow_counter, in, out);
- if (!err)
- *id = MLX5_GET(alloc_flow_counter_out, out, flow_counter_id);
- return err;
-}
-
-int mlx5_cmd_fc_alloc(struct mlx5_core_dev *dev, u32 *id)
-{
- return mlx5_cmd_fc_bulk_alloc(dev, 0, id);
-}
-
-int mlx5_cmd_fc_free(struct mlx5_core_dev *dev, u32 id)
-{
- u32 in[MLX5_ST_SZ_DW(dealloc_flow_counter_in)] = {};
-
- MLX5_SET(dealloc_flow_counter_in, in, opcode,
- MLX5_CMD_OP_DEALLOC_FLOW_COUNTER);
- MLX5_SET(dealloc_flow_counter_in, in, flow_counter_id, id);
- return mlx5_cmd_exec_in(dev, dealloc_flow_counter, in);
-}
-
-int mlx5_cmd_fc_query(struct mlx5_core_dev *dev, u32 id,
- u64 *packets, u64 *bytes)
-{
- u32 out[MLX5_ST_SZ_BYTES(query_flow_counter_out) +
- MLX5_ST_SZ_BYTES(traffic_counter)] = {};
- u32 in[MLX5_ST_SZ_DW(query_flow_counter_in)] = {};
- void *stats;
- int err = 0;
-
- MLX5_SET(query_flow_counter_in, in, opcode,
- MLX5_CMD_OP_QUERY_FLOW_COUNTER);
- MLX5_SET(query_flow_counter_in, in, op_mod, 0);
- MLX5_SET(query_flow_counter_in, in, flow_counter_id, id);
- err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
- if (err)
- return err;
-
- stats = MLX5_ADDR_OF(query_flow_counter_out, out, flow_statistics);
- *packets = MLX5_GET64(traffic_counter, stats, packets);
- *bytes = MLX5_GET64(traffic_counter, stats, octets);
- return 0;
-}
-
-int mlx5_cmd_fc_bulk_query(struct mlx5_core_dev *dev, u32 base_id, int bulk_len,
- u32 *out)
-{
- int outlen = mlx5_cmd_fc_get_bulk_query_out_len(bulk_len);
- u32 in[MLX5_ST_SZ_DW(query_flow_counter_in)] = {};
-
- MLX5_SET(query_flow_counter_in, in, opcode,
- MLX5_CMD_OP_QUERY_FLOW_COUNTER);
- MLX5_SET(query_flow_counter_in, in, flow_counter_id, base_id);
- MLX5_SET(query_flow_counter_in, in, num_of_counters, bulk_len);
- return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen);
-}
-
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fs_chains.c b/sys/dev/mlx5/mlx5_core/mlx5_fs_chains.c
new file mode 100644
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_core/mlx5_fs_chains.c
@@ -0,0 +1,664 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+// Copyright (c) 2020 Mellanox Technologies.
+
+#include <dev/mlx5/driver.h>
+#include <dev/mlx5/mlx5_ifc.h>
+#include <dev/mlx5/fs.h>
+
+#include "mlx5_core.h"
+#include "fs_chains.h"
+#include "fs_ft_pool.h"
+#include "fs_core.h"
+
+#define chains_lock(chains) ((chains)->lock)
+#define chains_xa(chains) ((chains)->chains_xa)
+#define prios_xa(chains) ((chains)->prios_xa)
+#define chains_default_ft(chains) ((chains)->chains_default_ft)
+#define chains_end_ft(chains) ((chains)->chains_end_ft)
+#define FT_TBL_SZ (64 * 1024)
+
+struct mlx5_fs_chains {
+ struct mlx5_core_dev *dev;
+
+ struct xarray chains_xa;
+ struct xarray prios_xa;
+ /* Protects above chains_ht and prios_ht */
+ struct mutex lock;
+
+ struct mlx5_flow_table *chains_default_ft;
+ struct mlx5_flow_table *chains_end_ft;
+
+ enum mlx5_flow_namespace_type ns;
+ u32 group_num;
+ u32 flags;
+ int fs_base_prio;
+ int fs_base_level;
+};
+
+struct fs_chain {
+ u32 chain;
+
+ int ref;
+ int id;
+ uint32_t xa_idx;
+
+ struct mlx5_fs_chains *chains;
+ struct list_head prios_list;
+ struct mlx5_flow_handle *restore_rule;
+ struct mlx5_modify_hdr *miss_modify_hdr;
+};
+
+struct prio_key {
+ u32 chain;
+ u32 prio;
+ u32 level;
+};
+
+struct prio {
+ struct list_head list;
+
+ struct prio_key key;
+ uint32_t xa_idx;
+
+ int ref;
+
+ struct fs_chain *chain;
+ struct mlx5_flow_table *ft;
+ struct mlx5_flow_table *next_ft;
+ struct mlx5_flow_group *miss_group;
+ struct mlx5_flow_handle *miss_rule;
+};
+
+/*
+static const struct rhashtable_params chain_params = {
+ .head_offset = offsetof(struct fs_chain, node),
+ .key_offset = offsetof(struct fs_chain, chain),
+ .key_len = sizeof_field(struct fs_chain, chain),
+ .automatic_shrinking = true,
+};
+
+static const struct rhashtable_params prio_params = {
+ .head_offset = offsetof(struct prio, node),
+ .key_offset = offsetof(struct prio, key),
+ .key_len = sizeof_field(struct prio, key),
+ .automatic_shrinking = true,
+};
+*/
+
+bool mlx5_chains_prios_supported(struct mlx5_fs_chains *chains)
+{
+ return chains->flags & MLX5_CHAINS_AND_PRIOS_SUPPORTED;
+}
+
+bool mlx5_chains_ignore_flow_level_supported(struct mlx5_fs_chains *chains)
+{
+ return chains->flags & MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED;
+}
+
+bool mlx5_chains_backwards_supported(struct mlx5_fs_chains *chains)
+{
+ return mlx5_chains_prios_supported(chains) &&
+ mlx5_chains_ignore_flow_level_supported(chains);
+}
+
+u32 mlx5_chains_get_chain_range(struct mlx5_fs_chains *chains)
+{
+ if (!mlx5_chains_prios_supported(chains))
+ return 1;
+
+ if (mlx5_chains_ignore_flow_level_supported(chains))
+ return UINT_MAX - 1;
+
+ /* We should get here only for eswitch case */
+ return FDB_TC_MAX_CHAIN;
+}
+
+u32 mlx5_chains_get_nf_ft_chain(struct mlx5_fs_chains *chains)
+{
+ return mlx5_chains_get_chain_range(chains) + 1;
+}
+
+u32 mlx5_chains_get_prio_range(struct mlx5_fs_chains *chains)
+{
+ if (mlx5_chains_ignore_flow_level_supported(chains))
+ return UINT_MAX;
+
+ if (!chains->dev->priv.eswitch)
+ return 1;
+
+ /* We should get here only for eswitch case */
+ return FDB_TC_MAX_PRIO;
+}
+
+static unsigned int mlx5_chains_get_level_range(struct mlx5_fs_chains *chains)
+{
+ if (mlx5_chains_ignore_flow_level_supported(chains))
+ return UINT_MAX;
+
+ /* Same value for FDB and NIC RX tables */
+ return FDB_TC_LEVELS_PER_PRIO;
+}
+
+void
+mlx5_chains_set_end_ft(struct mlx5_fs_chains *chains,
+ struct mlx5_flow_table *ft)
+{
+ chains_end_ft(chains) = ft;
+}
+
+static struct mlx5_flow_table *
+mlx5_chains_create_table(struct mlx5_fs_chains *chains,
+ u32 chain, u32 prio, u32 level)
+{
+ struct mlx5_flow_table_attr ft_attr = {};
+ struct mlx5_flow_namespace *ns;
+ struct mlx5_flow_table *ft;
+ int sz;
+
+ if (chains->flags & MLX5_CHAINS_FT_TUNNEL_SUPPORTED)
+ ft_attr.flags |= (MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT |
+ MLX5_FLOW_TABLE_TUNNEL_EN_DECAP);
+
+ sz = (chain == mlx5_chains_get_nf_ft_chain(chains)) ? FT_TBL_SZ : POOL_NEXT_SIZE;
+ ft_attr.max_fte = sz;
+
+ /* We use chains_default_ft(chains) as the table's next_ft till
+ * ignore_flow_level is allowed on FT creation and not just for FTEs.
+ * Instead caller should add an explicit miss rule if needed.
+ */
+ ft_attr.next_ft = chains_default_ft(chains);
+
+ /* The root table(chain 0, prio 1, level 0) is required to be
+ * connected to the previous fs_core managed prio.
+ * We always create it, as a managed table, in order to align with
+ * fs_core logic.
+ */
+ if (!mlx5_chains_ignore_flow_level_supported(chains) ||
+ (chain == 0 && prio == 1 && level == 0)) {
+ ft_attr.level = chains->fs_base_level;
+ ft_attr.prio = chains->fs_base_prio;
+ ns = (chains->ns == MLX5_FLOW_NAMESPACE_FDB) ?
+ mlx5_get_fdb_sub_ns(chains->dev, chain) :
+ mlx5_get_flow_namespace(chains->dev, chains->ns);
+ } else {
+ ft_attr.flags |= MLX5_FLOW_TABLE_UNMANAGED;
+ ft_attr.prio = chains->fs_base_prio;
+ /* Firmware doesn't allow us to create another level 0 table,
+ * so we create all unmanaged tables as level 1 (base + 1).
+ *
+ * To connect them, we use explicit miss rules with
+ * ignore_flow_level. Caller is responsible to create
+ * these rules (if needed).
+ */
+ ft_attr.level = chains->fs_base_level + 1;
+ ns = mlx5_get_flow_namespace(chains->dev, chains->ns);
+ }
+
+ ft_attr.autogroup.num_reserved_entries = 2;
+ ft_attr.autogroup.max_num_groups = chains->group_num;
+ ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr);
+ if (IS_ERR(ft)) {
+ mlx5_core_warn(chains->dev, "Failed to create chains table err %d (chain: %d, prio: %d, level: %d, size: %d)\n",
+ (int)PTR_ERR(ft), chain, prio, level, sz);
+ return ft;
+ }
+
+ return ft;
+}
+
+static struct fs_chain *
+mlx5_chains_create_chain(struct mlx5_fs_chains *chains, u32 chain)
+{
+ struct fs_chain *chain_s = NULL;
+ int err;
+
+ chain_s = kvzalloc(sizeof(*chain_s), GFP_KERNEL);
+ if (!chain_s)
+ return ERR_PTR(-ENOMEM);
+
+ chain_s->chains = chains;
+ chain_s->chain = chain;
+ INIT_LIST_HEAD(&chain_s->prios_list);
+
+ err = xa_alloc(&chains_xa(chains), &chain_s->xa_idx, chain_s,
+ xa_limit_32b, GFP_KERNEL);
+ if (err)
+ goto err_insert;
+
+ return chain_s;
+
+err_insert:
+ kvfree(chain_s);
+ return ERR_PTR(err);
+}
+
+static void
+mlx5_chains_destroy_chain(struct fs_chain *chain)
+{
+ struct mlx5_fs_chains *chains = chain->chains;
+
+ xa_erase(&chains_xa(chains), chain->xa_idx);
+ kvfree(chain);
+}
+
+static struct fs_chain *
+mlx5_chains_get_chain(struct mlx5_fs_chains *chains, u32 chain)
+{
+ struct fs_chain *chain_s = NULL;
+ unsigned long idx;
+
+ xa_for_each(&chains_xa(chains), idx, chain_s) {
+ if (chain_s->chain == chain)
+ break;
+ }
+
+ if (!chain_s) {
+ chain_s = mlx5_chains_create_chain(chains, chain);
+ if (IS_ERR(chain_s))
+ return chain_s;
+ }
+
+ chain_s->ref++;
+
+ return chain_s;
+}
+
+static struct mlx5_flow_handle *
+mlx5_chains_add_miss_rule(struct fs_chain *chain,
+ struct mlx5_flow_table *ft,
+ struct mlx5_flow_table *next_ft)
+{
+ struct mlx5_flow_destination dest = {};
+ struct mlx5_flow_act act = {};
+
+ act.flags = FLOW_ACT_NO_APPEND;
+ if (mlx5_chains_ignore_flow_level_supported(chain->chains))
+ act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
+
+ act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
+ dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
+ dest.ft = next_ft;
+
+ return mlx5_add_flow_rules(ft, NULL, &act, &dest, 1);
+}
+
+static int
+mlx5_chains_update_prio_prevs(struct prio *prio,
+ struct mlx5_flow_table *next_ft)
+{
+ struct mlx5_flow_handle *miss_rules[FDB_TC_LEVELS_PER_PRIO + 1] = {};
+ struct fs_chain *chain = prio->chain;
+ struct prio *pos;
+ int n = 0, err;
+
+ if (prio->key.level)
+ return 0;
+
+ /* Iterate in reverse order until reaching the level 0 rule of
+ * the previous priority, adding all the miss rules first, so we can
+ * revert them if any of them fails.
+ */
+ pos = prio;
+ list_for_each_entry_continue_reverse(pos,
+ &chain->prios_list,
+ list) {
+ miss_rules[n] = mlx5_chains_add_miss_rule(chain,
+ pos->ft,
+ next_ft);
+ if (IS_ERR(miss_rules[n])) {
+ err = PTR_ERR(miss_rules[n]);
+ goto err_prev_rule;
+ }
+
+ n++;
+ if (!pos->key.level)
+ break;
+ }
+
+ /* Success, delete old miss rules, and update the pointers. */
+ n = 0;
+ pos = prio;
+ list_for_each_entry_continue_reverse(pos,
+ &chain->prios_list,
+ list) {
+ mlx5_del_flow_rules(&pos->miss_rule);
+
+ pos->miss_rule = miss_rules[n];
+ pos->next_ft = next_ft;
+
+ n++;
+ if (!pos->key.level)
+ break;
+ }
+
+ return 0;
+
+err_prev_rule:
+ while (--n >= 0)
+ mlx5_del_flow_rules(&miss_rules[n]);
+
+ return err;
+}
+
+static void
+mlx5_chains_put_chain(struct fs_chain *chain)
+{
+ if (--chain->ref == 0)
+ mlx5_chains_destroy_chain(chain);
+}
+
+static struct prio *
+mlx5_chains_create_prio(struct mlx5_fs_chains *chains,
+ u32 chain, u32 prio, u32 level)
+{
+ int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
+ struct mlx5_flow_handle *miss_rule;
+ struct mlx5_flow_group *miss_group;
+ struct mlx5_flow_table *next_ft;
+ struct mlx5_flow_table *ft;
+ struct fs_chain *chain_s;
+ struct list_head *pos;
+ struct prio *prio_s;
+ u32 *flow_group_in;
+ int err;
+
+ chain_s = mlx5_chains_get_chain(chains, chain);
+ if (IS_ERR(chain_s))
+ return ERR_CAST(chain_s);
+
+ prio_s = kvzalloc(sizeof(*prio_s), GFP_KERNEL);
+ flow_group_in = kvzalloc(inlen, GFP_KERNEL);
+ if (!prio_s || !flow_group_in) {
+ err = -ENOMEM;
+ goto err_alloc;
+ }
+
+ /* Chain's prio list is sorted by prio and level.
+ * And all levels of some prio point to the next prio's level 0.
+ * Example list (prio, level):
+ * (3,0)->(3,1)->(5,0)->(5,1)->(6,1)->(7,0)
+ * In hardware, we will we have the following pointers:
+ * (3,0) -> (5,0) -> (7,0) -> Slow path
+ * (3,1) -> (5,0)
+ * (5,1) -> (7,0)
+ * (6,1) -> (7,0)
+ */
+
+ /* Default miss for each chain: */
+ next_ft = (chain == mlx5_chains_get_nf_ft_chain(chains)) ?
+ chains_default_ft(chains) :
+ chains_end_ft(chains);
+ list_for_each(pos, &chain_s->prios_list) {
+ struct prio *p = list_entry(pos, struct prio, list);
+
+ /* exit on first pos that is larger */
+ if (prio < p->key.prio || (prio == p->key.prio &&
+ level < p->key.level)) {
+ /* Get next level 0 table */
+ next_ft = p->key.level == 0 ? p->ft : p->next_ft;
+ break;
+ }
+ }
+
+ ft = mlx5_chains_create_table(chains, chain, prio, level);
+ if (IS_ERR(ft)) {
+ err = PTR_ERR(ft);
+ goto err_create;
+ }
+
+ MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index,
+ ft->max_fte - 2);
+ MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
+ ft->max_fte - 1);
+ miss_group = mlx5_create_flow_group(ft, flow_group_in);
+ if (IS_ERR(miss_group)) {
+ err = PTR_ERR(miss_group);
+ goto err_group;
+ }
+
+ /* Add miss rule to next_ft */
+ miss_rule = mlx5_chains_add_miss_rule(chain_s, ft, next_ft);
+ if (IS_ERR(miss_rule)) {
+ err = PTR_ERR(miss_rule);
+ goto err_miss_rule;
+ }
+
+ prio_s->miss_group = miss_group;
+ prio_s->miss_rule = miss_rule;
+ prio_s->next_ft = next_ft;
+ prio_s->chain = chain_s;
+ prio_s->key.chain = chain;
+ prio_s->key.prio = prio;
+ prio_s->key.level = level;
+ prio_s->ft = ft;
+
+ err = xa_alloc(&prios_xa(chains), &prio_s->xa_idx, prio_s,
+ xa_limit_32b, GFP_KERNEL);
+ if (err)
+ goto err_insert;
+
+ list_add(&prio_s->list, pos->prev);
+
+ /* Table is ready, connect it */
+ err = mlx5_chains_update_prio_prevs(prio_s, ft);
+ if (err)
+ goto err_update;
+
+ kvfree(flow_group_in);
+ return prio_s;
+
+err_update:
+ list_del(&prio_s->list);
+ xa_erase(&prios_xa(chains), prio_s->xa_idx);
+err_insert:
+ mlx5_del_flow_rules(&miss_rule);
+err_miss_rule:
+ mlx5_destroy_flow_group(miss_group);
+err_group:
+ mlx5_destroy_flow_table(ft);
+err_create:
+err_alloc:
+ kvfree(prio_s);
+ kvfree(flow_group_in);
+ mlx5_chains_put_chain(chain_s);
+ return ERR_PTR(err);
+}
+
+static void
+mlx5_chains_destroy_prio(struct mlx5_fs_chains *chains,
+ struct prio *prio)
+{
+ struct fs_chain *chain = prio->chain;
+
+ WARN_ON(mlx5_chains_update_prio_prevs(prio,
+ prio->next_ft));
+
+ list_del(&prio->list);
+ xa_erase(&prios_xa(chains), prio->xa_idx);
+ mlx5_del_flow_rules(&prio->miss_rule);
+ mlx5_destroy_flow_group(prio->miss_group);
+ mlx5_destroy_flow_table(prio->ft);
+ mlx5_chains_put_chain(chain);
+ kvfree(prio);
+}
+
+struct mlx5_flow_table *
+mlx5_chains_get_table(struct mlx5_fs_chains *chains, u32 chain, u32 prio,
+ u32 level)
+{
+ struct mlx5_flow_table *prev_fts;
+ struct prio *prio_s;
+ unsigned long idx;
+ int l = 0;
+
+ if ((chain > mlx5_chains_get_chain_range(chains) &&
+ chain != mlx5_chains_get_nf_ft_chain(chains)) ||
+ prio > mlx5_chains_get_prio_range(chains) ||
+ level > mlx5_chains_get_level_range(chains))
+ return ERR_PTR(-EOPNOTSUPP);
+
+ /* create earlier levels for correct fs_core lookup when
+ * connecting tables.
+ */
+ for (l = 0; l < level; l++) {
+ prev_fts = mlx5_chains_get_table(chains, chain, prio, l);
+ if (IS_ERR(prev_fts)) {
+ prio_s = ERR_CAST(prev_fts);
+ goto err_get_prevs;
+ }
+ }
+
+ mutex_lock(&chains_lock(chains));
+ xa_for_each(&prios_xa(chains), idx, prio_s) {
+ if (chain == prio_s->key.chain &&
+ prio == prio_s->key.prio &&
+ level == prio_s->key.level)
+ break;
+ }
+ if (!prio_s) {
+ prio_s = mlx5_chains_create_prio(chains, chain,
+ prio, level);
+ if (IS_ERR(prio_s))
+ goto err_create_prio;
+ }
+
+ ++prio_s->ref;
+ mutex_unlock(&chains_lock(chains));
+
+ return prio_s->ft;
+
+err_create_prio:
+ mutex_unlock(&chains_lock(chains));
+err_get_prevs:
+ while (--l >= 0)
+ mlx5_chains_put_table(chains, chain, prio, l);
+ return ERR_CAST(prio_s);
+}
+
+void
+mlx5_chains_put_table(struct mlx5_fs_chains *chains, u32 chain, u32 prio,
+ u32 level)
+{
+ struct prio *prio_s;
+ unsigned long idx;
+
+ mutex_lock(&chains_lock(chains));
+
+ xa_for_each(&prios_xa(chains), idx, prio_s) {
+ if (chain == prio_s->key.chain &&
+ prio == prio_s->key.prio &&
+ level == prio_s->key.level)
+ break;
+ }
+ if (!prio_s)
+ goto err_get_prio;
+
+ if (--prio_s->ref == 0)
+ mlx5_chains_destroy_prio(chains, prio_s);
+ mutex_unlock(&chains_lock(chains));
+
+ while (level-- > 0)
+ mlx5_chains_put_table(chains, chain, prio, level);
+
+ return;
+
+err_get_prio:
+ mutex_unlock(&chains_lock(chains));
+ WARN_ONCE(1,
+ "Couldn't find table: (chain: %d prio: %d level: %d)",
+ chain, prio, level);
+}
+
+struct mlx5_flow_table *
+mlx5_chains_get_tc_end_ft(struct mlx5_fs_chains *chains)
+{
+ return chains_end_ft(chains);
+}
+
+struct mlx5_flow_table *
+mlx5_chains_create_global_table(struct mlx5_fs_chains *chains)
+{
+ u32 chain, prio, level;
+ int err;
+
+ if (!mlx5_chains_ignore_flow_level_supported(chains)) {
+ err = -EOPNOTSUPP;
+
+ mlx5_core_warn(chains->dev,
+ "Couldn't create global flow table, ignore_flow_level not supported.");
+ goto err_ignore;
+ }
+
+ chain = mlx5_chains_get_chain_range(chains),
+ prio = mlx5_chains_get_prio_range(chains);
+ level = mlx5_chains_get_level_range(chains);
+
+ return mlx5_chains_create_table(chains, chain, prio, level);
+
+err_ignore:
+ return ERR_PTR(err);
+}
+
+void
+mlx5_chains_destroy_global_table(struct mlx5_fs_chains *chains,
+ struct mlx5_flow_table *ft)
+{
+ mlx5_destroy_flow_table(ft);
+}
+
+static struct mlx5_fs_chains *
+mlx5_chains_init(struct mlx5_core_dev *dev, struct mlx5_chains_attr *attr)
+{
+ struct mlx5_fs_chains *chains;
+
+ chains = kzalloc(sizeof(*chains), GFP_KERNEL);
+ if (!chains)
+ return ERR_PTR(-ENOMEM);
+
+ chains->dev = dev;
+ chains->flags = attr->flags;
+ chains->ns = attr->ns;
+ chains->group_num = attr->max_grp_num;
+ chains->fs_base_prio = attr->fs_base_prio;
+ chains->fs_base_level = attr->fs_base_level;
+ chains_default_ft(chains) = chains_end_ft(chains) = attr->default_ft;
+
+ xa_init(&chains_xa(chains));
+ xa_init(&prios_xa(chains));
+
+ mutex_init(&chains_lock(chains));
+
+ return chains;
+}
+
+static void
+mlx5_chains_cleanup(struct mlx5_fs_chains *chains)
+{
+ mutex_destroy(&chains_lock(chains));
+ xa_destroy(&prios_xa(chains));
+ xa_destroy(&chains_xa(chains));
+
+ kfree(chains);
+}
+
+struct mlx5_fs_chains *
+mlx5_chains_create(struct mlx5_core_dev *dev, struct mlx5_chains_attr *attr)
+{
+ struct mlx5_fs_chains *chains;
+
+ chains = mlx5_chains_init(dev, attr);
+
+ return chains;
+}
+
+void
+mlx5_chains_destroy(struct mlx5_fs_chains *chains)
+{
+ mlx5_chains_cleanup(chains);
+}
+
+void
+mlx5_chains_print_info(struct mlx5_fs_chains *chains)
+{
+ mlx5_core_dbg(chains->dev, "Flow table chains groups(%d)\n", chains->group_num);
+}
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fs_cmd.c b/sys/dev/mlx5/mlx5_core/mlx5_fs_cmd.c
--- a/sys/dev/mlx5/mlx5_core/mlx5_fs_cmd.c
+++ b/sys/dev/mlx5/mlx5_core/mlx5_fs_cmd.c
@@ -1,469 +1,993 @@
-/*-
- * Copyright (c) 2013-2017, Mellanox Technologies, Ltd. All rights reserved.
+/*
+ * Copyright (c) 2015, Mellanox Technologies. All rights reserved.
*
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
*
- * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
*
- * $FreeBSD$
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
*/
-#include "opt_rss.h"
-
-#include <linux/types.h>
-#include <linux/module.h>
-#include <dev/mlx5/mlx5_ifc.h>
+#include <dev/mlx5/driver.h>
#include <dev/mlx5/device.h>
-#include <dev/mlx5/fs.h>
+#include <dev/mlx5/mlx5_ifc.h>
-#include <dev/mlx5/mlx5_core/fs_core.h>
-#include <dev/mlx5/mlx5_core/mlx5_core.h>
+#include "fs_core.h"
+#include "fs_cmd.h"
+#include "fs_ft_pool.h"
+#include "mlx5_core.h"
+#include "eswitch.h"
-int mlx5_cmd_update_root_ft(struct mlx5_core_dev *dev,
- enum fs_ft_type type,
- unsigned int id)
+static int mlx5_cmd_stub_update_root_ft(struct mlx5_flow_root_namespace *ns,
+ struct mlx5_flow_table *ft,
+ u32 underlay_qpn,
+ bool disconnect)
{
- u32 in[MLX5_ST_SZ_DW(set_flow_table_root_in)] = {0};
- u32 out[MLX5_ST_SZ_DW(set_flow_table_root_out)] = {0};
+ return 0;
+}
- if (!dev)
- return -EINVAL;
+static int mlx5_cmd_stub_create_flow_table(struct mlx5_flow_root_namespace *ns,
+ struct mlx5_flow_table *ft,
+ struct mlx5_flow_table_attr *ft_attr,
+ struct mlx5_flow_table *next_ft)
+{
+ int max_fte = ft_attr->max_fte;
+
+ ft->max_fte = max_fte ? roundup_pow_of_two(max_fte) : 1;
+
+ return 0;
+}
+
+static int mlx5_cmd_stub_destroy_flow_table(struct mlx5_flow_root_namespace *ns,
+ struct mlx5_flow_table *ft)
+{
+ return 0;
+}
+
+static int mlx5_cmd_stub_modify_flow_table(struct mlx5_flow_root_namespace *ns,
+ struct mlx5_flow_table *ft,
+ struct mlx5_flow_table *next_ft)
+{
+ return 0;
+}
+
+static int mlx5_cmd_stub_create_flow_group(struct mlx5_flow_root_namespace *ns,
+ struct mlx5_flow_table *ft,
+ u32 *in,
+ struct mlx5_flow_group *fg)
+{
+ return 0;
+}
+
+static int mlx5_cmd_stub_destroy_flow_group(struct mlx5_flow_root_namespace *ns,
+ struct mlx5_flow_table *ft,
+ struct mlx5_flow_group *fg)
+{
+ return 0;
+}
+
+static int mlx5_cmd_stub_create_fte(struct mlx5_flow_root_namespace *ns,
+ struct mlx5_flow_table *ft,
+ struct mlx5_flow_group *group,
+ struct fs_fte *fte)
+{
+ return 0;
+}
+
+static int mlx5_cmd_stub_update_fte(struct mlx5_flow_root_namespace *ns,
+ struct mlx5_flow_table *ft,
+ struct mlx5_flow_group *group,
+ int modify_mask,
+ struct fs_fte *fte)
+{
+ return -EOPNOTSUPP;
+}
+
+static int mlx5_cmd_stub_delete_fte(struct mlx5_flow_root_namespace *ns,
+ struct mlx5_flow_table *ft,
+ struct fs_fte *fte)
+{
+ return 0;
+}
+
+static int mlx5_cmd_stub_packet_reformat_alloc(struct mlx5_flow_root_namespace *ns,
+ struct mlx5_pkt_reformat_params *params,
+ enum mlx5_flow_namespace_type namespace,
+ struct mlx5_pkt_reformat *pkt_reformat)
+{
+ return 0;
+}
+
+static void mlx5_cmd_stub_packet_reformat_dealloc(struct mlx5_flow_root_namespace *ns,
+ struct mlx5_pkt_reformat *pkt_reformat)
+{
+}
+
+static int mlx5_cmd_stub_modify_header_alloc(struct mlx5_flow_root_namespace *ns,
+ u8 namespace, u8 num_actions,
+ void *modify_actions,
+ struct mlx5_modify_hdr *modify_hdr)
+{
+ return 0;
+}
+
+static void mlx5_cmd_stub_modify_header_dealloc(struct mlx5_flow_root_namespace *ns,
+ struct mlx5_modify_hdr *modify_hdr)
+{
+}
+
+static int mlx5_cmd_stub_set_peer(struct mlx5_flow_root_namespace *ns,
+ struct mlx5_flow_root_namespace *peer_ns)
+{
+ return 0;
+}
+
+static int mlx5_cmd_stub_create_ns(struct mlx5_flow_root_namespace *ns)
+{
+ return 0;
+}
+
+static int mlx5_cmd_stub_destroy_ns(struct mlx5_flow_root_namespace *ns)
+{
+ return 0;
+}
+
+static u32 mlx5_cmd_stub_get_capabilities(struct mlx5_flow_root_namespace *ns,
+ enum fs_flow_table_type ft_type)
+{
+ return 0;
+}
+
+static int mlx5_cmd_update_root_ft(struct mlx5_flow_root_namespace *ns,
+ struct mlx5_flow_table *ft, u32 underlay_qpn,
+ bool disconnect)
+{
+ u32 in[MLX5_ST_SZ_DW(set_flow_table_root_in)] = {};
+ struct mlx5_core_dev *dev = ns->dev;
+
+ if ((MLX5_CAP_GEN(dev, port_type) == MLX5_CAP_PORT_TYPE_IB) &&
+ underlay_qpn == 0)
+ return 0;
MLX5_SET(set_flow_table_root_in, in, opcode,
MLX5_CMD_OP_SET_FLOW_TABLE_ROOT);
- MLX5_SET(set_flow_table_root_in, in, table_type, type);
- MLX5_SET(set_flow_table_root_in, in, table_id, id);
+ MLX5_SET(set_flow_table_root_in, in, table_type, ft->type);
+
+ if (disconnect)
+ MLX5_SET(set_flow_table_root_in, in, op_mod, 1);
+ else
+ MLX5_SET(set_flow_table_root_in, in, table_id, ft->id);
- return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+ MLX5_SET(set_flow_table_root_in, in, underlay_qpn, underlay_qpn);
+ MLX5_SET(set_flow_table_root_in, in, vport_number, ft->vport);
+ MLX5_SET(set_flow_table_root_in, in, other_vport,
+ !!(ft->flags & MLX5_FLOW_TABLE_OTHER_VPORT));
+
+ return mlx5_cmd_exec_in(dev, set_flow_table_root, in);
}
-int mlx5_cmd_fs_create_ft(struct mlx5_core_dev *dev,
- u16 vport, enum fs_ft_type type, unsigned int level,
- unsigned int log_size, const char *name, unsigned int *table_id)
+static int mlx5_cmd_create_flow_table(struct mlx5_flow_root_namespace *ns,
+ struct mlx5_flow_table *ft,
+ struct mlx5_flow_table_attr *ft_attr,
+ struct mlx5_flow_table *next_ft)
{
- u32 in[MLX5_ST_SZ_DW(create_flow_table_in)] = {0};
- u32 out[MLX5_ST_SZ_DW(create_flow_table_out)] = {0};
+ int en_encap = !!(ft->flags & MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT);
+ int en_decap = !!(ft->flags & MLX5_FLOW_TABLE_TUNNEL_EN_DECAP);
+ int term = !!(ft->flags & MLX5_FLOW_TABLE_TERMINATION);
+ u32 out[MLX5_ST_SZ_DW(create_flow_table_out)] = {};
+ u32 in[MLX5_ST_SZ_DW(create_flow_table_in)] = {};
+ struct mlx5_core_dev *dev = ns->dev;
+ unsigned int size;
int err;
- if (!dev)
- return -EINVAL;
+ size = mlx5_ft_pool_get_avail_sz(dev, ft->type, ft_attr->max_fte);
+ if (!size)
+ return -ENOSPC;
MLX5_SET(create_flow_table_in, in, opcode,
MLX5_CMD_OP_CREATE_FLOW_TABLE);
- MLX5_SET(create_flow_table_in, in, table_type, type);
- MLX5_SET(create_flow_table_in, in, flow_table_context.level, level);
- MLX5_SET(create_flow_table_in, in, flow_table_context.log_size,
- log_size);
- if (strstr(name, FS_REFORMAT_KEYWORD) != NULL)
- MLX5_SET(create_flow_table_in, in,
- flow_table_context.reformat_en, 1);
- if (vport) {
- MLX5_SET(create_flow_table_in, in, vport_number, vport);
- MLX5_SET(create_flow_table_in, in, other_vport, 1);
+ MLX5_SET(create_flow_table_in, in, uid, ft_attr->uid);
+ MLX5_SET(create_flow_table_in, in, table_type, ft->type);
+ MLX5_SET(create_flow_table_in, in, flow_table_context.level, ft->level);
+ MLX5_SET(create_flow_table_in, in, flow_table_context.log_size, size ? ilog2(size) : 0);
+ MLX5_SET(create_flow_table_in, in, vport_number, ft->vport);
+ MLX5_SET(create_flow_table_in, in, other_vport,
+ !!(ft->flags & MLX5_FLOW_TABLE_OTHER_VPORT));
+
+ MLX5_SET(create_flow_table_in, in, flow_table_context.decap_en,
+ en_decap);
+ MLX5_SET(create_flow_table_in, in, flow_table_context.reformat_en,
+ en_encap);
+ MLX5_SET(create_flow_table_in, in, flow_table_context.termination_table,
+ term);
+
+ switch (ft->op_mod) {
+ case FS_FT_OP_MOD_NORMAL:
+ if (next_ft) {
+ MLX5_SET(create_flow_table_in, in,
+ flow_table_context.table_miss_action,
+ MLX5_FLOW_TABLE_MISS_ACTION_FWD);
+ MLX5_SET(create_flow_table_in, in,
+ flow_table_context.table_miss_id, next_ft->id);
+ } else {
+ MLX5_SET(create_flow_table_in, in,
+ flow_table_context.table_miss_action,
+ ft->def_miss_action);
+ }
+ break;
+
+ case FS_FT_OP_MOD_LAG_DEMUX:
+ MLX5_SET(create_flow_table_in, in, op_mod, 0x1);
+ if (next_ft)
+ MLX5_SET(create_flow_table_in, in,
+ flow_table_context.lag_master_next_table_id,
+ next_ft->id);
+ break;
}
- err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
- if (!err)
- *table_id = MLX5_GET(create_flow_table_out, out, table_id);
+ err = mlx5_cmd_exec_inout(dev, create_flow_table, in, out);
+ if (!err) {
+ ft->id = MLX5_GET(create_flow_table_out, out,
+ table_id);
+ ft->max_fte = size;
+ } else {
+ mlx5_ft_pool_put_sz(ns->dev, size);
+ }
return err;
}
-int mlx5_cmd_fs_destroy_ft(struct mlx5_core_dev *dev,
- u16 vport,
- enum fs_ft_type type, unsigned int table_id)
+static int mlx5_cmd_destroy_flow_table(struct mlx5_flow_root_namespace *ns,
+ struct mlx5_flow_table *ft)
{
- u32 in[MLX5_ST_SZ_DW(destroy_flow_table_in)] = {0};
- u32 out[MLX5_ST_SZ_DW(destroy_flow_table_out)] = {0};
-
- if (!dev)
- return -EINVAL;
+ u32 in[MLX5_ST_SZ_DW(destroy_flow_table_in)] = {};
+ struct mlx5_core_dev *dev = ns->dev;
+ int err;
MLX5_SET(destroy_flow_table_in, in, opcode,
MLX5_CMD_OP_DESTROY_FLOW_TABLE);
- MLX5_SET(destroy_flow_table_in, in, table_type, type);
- MLX5_SET(destroy_flow_table_in, in, table_id, table_id);
- if (vport) {
- MLX5_SET(destroy_flow_table_in, in, vport_number, vport);
- MLX5_SET(destroy_flow_table_in, in, other_vport, 1);
+ MLX5_SET(destroy_flow_table_in, in, table_type, ft->type);
+ MLX5_SET(destroy_flow_table_in, in, table_id, ft->id);
+ MLX5_SET(destroy_flow_table_in, in, vport_number, ft->vport);
+ MLX5_SET(destroy_flow_table_in, in, other_vport,
+ !!(ft->flags & MLX5_FLOW_TABLE_OTHER_VPORT));
+
+ err = mlx5_cmd_exec_in(dev, destroy_flow_table, in);
+ if (!err)
+ mlx5_ft_pool_put_sz(ns->dev, ft->max_fte);
+
+ return err;
+}
+
+static int mlx5_cmd_modify_flow_table(struct mlx5_flow_root_namespace *ns,
+ struct mlx5_flow_table *ft,
+ struct mlx5_flow_table *next_ft)
+{
+ u32 in[MLX5_ST_SZ_DW(modify_flow_table_in)] = {};
+ struct mlx5_core_dev *dev = ns->dev;
+
+ MLX5_SET(modify_flow_table_in, in, opcode,
+ MLX5_CMD_OP_MODIFY_FLOW_TABLE);
+ MLX5_SET(modify_flow_table_in, in, table_type, ft->type);
+ MLX5_SET(modify_flow_table_in, in, table_id, ft->id);
+
+ if (ft->op_mod == FS_FT_OP_MOD_LAG_DEMUX) {
+ MLX5_SET(modify_flow_table_in, in, modify_field_select,
+ MLX5_MODIFY_FLOW_TABLE_LAG_NEXT_TABLE_ID);
+ if (next_ft) {
+ MLX5_SET(modify_flow_table_in, in,
+ flow_table_context.lag_master_next_table_id, next_ft->id);
+ } else {
+ MLX5_SET(modify_flow_table_in, in,
+ flow_table_context.lag_master_next_table_id, 0);
+ }
+ } else {
+ MLX5_SET(modify_flow_table_in, in, vport_number, ft->vport);
+ MLX5_SET(modify_flow_table_in, in, other_vport,
+ !!(ft->flags & MLX5_FLOW_TABLE_OTHER_VPORT));
+ MLX5_SET(modify_flow_table_in, in, modify_field_select,
+ MLX5_MODIFY_FLOW_TABLE_MISS_TABLE_ID);
+ if (next_ft) {
+ MLX5_SET(modify_flow_table_in, in,
+ flow_table_context.table_miss_action,
+ MLX5_FLOW_TABLE_MISS_ACTION_FWD);
+ MLX5_SET(modify_flow_table_in, in,
+ flow_table_context.table_miss_id,
+ next_ft->id);
+ } else {
+ MLX5_SET(modify_flow_table_in, in,
+ flow_table_context.table_miss_action,
+ ft->def_miss_action);
+ }
}
- return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+ return mlx5_cmd_exec_in(dev, modify_flow_table, in);
}
-int mlx5_cmd_fs_create_fg(struct mlx5_core_dev *dev,
- u32 *in,
- u16 vport,
- enum fs_ft_type type, unsigned int table_id,
- unsigned int *group_id)
+static int mlx5_cmd_create_flow_group(struct mlx5_flow_root_namespace *ns,
+ struct mlx5_flow_table *ft,
+ u32 *in,
+ struct mlx5_flow_group *fg)
{
- u32 out[MLX5_ST_SZ_DW(create_flow_group_out)] = {0};
+ u32 out[MLX5_ST_SZ_DW(create_flow_group_out)] = {};
+ struct mlx5_core_dev *dev = ns->dev;
int err;
- int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
- if (!dev)
- return -EINVAL;
MLX5_SET(create_flow_group_in, in, opcode,
MLX5_CMD_OP_CREATE_FLOW_GROUP);
- MLX5_SET(create_flow_group_in, in, table_type, type);
- MLX5_SET(create_flow_group_in, in, table_id, table_id);
- if (vport) {
- MLX5_SET(create_flow_group_in, in, vport_number, vport);
- MLX5_SET(create_flow_group_in, in, other_vport, 1);
- }
-
- err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
+ MLX5_SET(create_flow_group_in, in, table_type, ft->type);
+ MLX5_SET(create_flow_group_in, in, table_id, ft->id);
+ MLX5_SET(create_flow_group_in, in, vport_number, ft->vport);
+ MLX5_SET(create_flow_group_in, in, other_vport,
+ !!(ft->flags & MLX5_FLOW_TABLE_OTHER_VPORT));
+ err = mlx5_cmd_exec_inout(dev, create_flow_group, in, out);
if (!err)
- *group_id = MLX5_GET(create_flow_group_out, out, group_id);
-
+ fg->id = MLX5_GET(create_flow_group_out, out,
+ group_id);
return err;
}
-int mlx5_cmd_fs_destroy_fg(struct mlx5_core_dev *dev,
- u16 vport,
- enum fs_ft_type type, unsigned int table_id,
- unsigned int group_id)
+static int mlx5_cmd_destroy_flow_group(struct mlx5_flow_root_namespace *ns,
+ struct mlx5_flow_table *ft,
+ struct mlx5_flow_group *fg)
{
- u32 in[MLX5_ST_SZ_DW(destroy_flow_group_in)] = {0};
- u32 out[MLX5_ST_SZ_DW(destroy_flow_group_out)] = {0};
-
- if (!dev)
- return -EINVAL;
+ u32 in[MLX5_ST_SZ_DW(destroy_flow_group_in)] = {};
+ struct mlx5_core_dev *dev = ns->dev;
MLX5_SET(destroy_flow_group_in, in, opcode,
MLX5_CMD_OP_DESTROY_FLOW_GROUP);
- MLX5_SET(destroy_flow_group_in, in, table_type, type);
- MLX5_SET(destroy_flow_group_in, in, table_id, table_id);
- MLX5_SET(destroy_flow_group_in, in, group_id, group_id);
- if (vport) {
- MLX5_SET(destroy_flow_group_in, in, vport_number, vport);
- MLX5_SET(destroy_flow_group_in, in, other_vport, 1);
- }
-
- return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+ MLX5_SET(destroy_flow_group_in, in, table_type, ft->type);
+ MLX5_SET(destroy_flow_group_in, in, table_id, ft->id);
+ MLX5_SET(destroy_flow_group_in, in, group_id, fg->id);
+ MLX5_SET(destroy_flow_group_in, in, vport_number, ft->vport);
+ MLX5_SET(destroy_flow_group_in, in, other_vport,
+ !!(ft->flags & MLX5_FLOW_TABLE_OTHER_VPORT));
+ return mlx5_cmd_exec_in(dev, destroy_flow_group, in);
}
-int mlx5_cmd_fs_set_fte(struct mlx5_core_dev *dev,
- u16 vport,
- enum fs_fte_status *fte_status,
- u32 *match_val,
- enum fs_ft_type type, unsigned int table_id,
- unsigned int index, unsigned int group_id,
- struct mlx5_flow_act *flow_act,
- u32 sw_action, int dest_size,
- struct list_head *dests) /* mlx5_flow_desination */
+static int mlx5_set_extended_dest(struct mlx5_core_dev *dev,
+ struct fs_fte *fte, bool *extended_dest)
{
- u32 out[MLX5_ST_SZ_DW(set_fte_out)] = {0};
- u32 *in;
- unsigned int inlen;
+ int fw_log_max_fdb_encap_uplink =
+ MLX5_CAP_ESW(dev, log_max_fdb_encap_uplink);
+ int num_fwd_destinations = 0;
struct mlx5_flow_rule *dst;
- void *in_flow_context;
- void *in_match_value;
- void *in_dests;
- int err;
- int opmod = 0;
- int modify_mask = 0;
- int atomic_mod_cap;
- u32 prm_action = 0;
- int count_list = 0;
+ int num_encap = 0;
- if (sw_action != MLX5_FLOW_RULE_FWD_ACTION_DEST)
- dest_size = 0;
+ *extended_dest = false;
+ if (!(fte->action.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST))
+ return 0;
- if (sw_action & MLX5_FLOW_RULE_FWD_ACTION_ALLOW)
- prm_action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW;
+ list_for_each_entry(dst, &fte->node.children, node.list) {
+ if (dst->dest_attr.type == MLX5_FLOW_DESTINATION_TYPE_COUNTER ||
+ dst->dest_attr.type == MLX5_FLOW_DESTINATION_TYPE_NONE)
+ continue;
+ if ((dst->dest_attr.type == MLX5_FLOW_DESTINATION_TYPE_VPORT ||
+ dst->dest_attr.type == MLX5_FLOW_DESTINATION_TYPE_UPLINK) &&
+ dst->dest_attr.vport.flags & MLX5_FLOW_DEST_VPORT_REFORMAT_ID)
+ num_encap++;
+ num_fwd_destinations++;
+ }
+ if (num_fwd_destinations > 1 && num_encap > 0)
+ *extended_dest = true;
- if (sw_action & MLX5_FLOW_RULE_FWD_ACTION_DROP)
- prm_action |= MLX5_FLOW_CONTEXT_ACTION_DROP;
+ if (*extended_dest && !fw_log_max_fdb_encap_uplink) {
+ mlx5_core_warn(dev, "FW does not support extended destination");
+ return -EOPNOTSUPP;
+ }
+ if (num_encap > (1 << fw_log_max_fdb_encap_uplink)) {
+ mlx5_core_warn(dev, "FW does not support more than %d encaps",
+ 1 << fw_log_max_fdb_encap_uplink);
+ return -EOPNOTSUPP;
+ }
- if (sw_action & MLX5_FLOW_RULE_FWD_ACTION_DEST)
- prm_action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
+ return 0;
+}
- if (flow_act->actions & MLX5_FLOW_ACT_ACTIONS_COUNT) {
- prm_action |= MLX5_FLOW_CONTEXT_ACTION_COUNT;
- count_list = 1;
- }
+static void
+mlx5_cmd_set_fte_flow_meter(struct fs_fte *fte, void *in_flow_context)
+{
+ void *exe_aso_ctrl;
+ void *execute_aso;
+
+ execute_aso = MLX5_ADDR_OF(flow_context, in_flow_context,
+ execute_aso[0]);
+ MLX5_SET(execute_aso, execute_aso, valid, 1);
+ MLX5_SET(execute_aso, execute_aso, aso_object_id,
+ fte->action.exe_aso.object_id);
+
+ exe_aso_ctrl = MLX5_ADDR_OF(execute_aso, execute_aso, exe_aso_ctrl);
+ MLX5_SET(exe_aso_ctrl_flow_meter, exe_aso_ctrl, return_reg_id,
+ fte->action.exe_aso.return_reg_id);
+ MLX5_SET(exe_aso_ctrl_flow_meter, exe_aso_ctrl, aso_type,
+ fte->action.exe_aso.type);
+ MLX5_SET(exe_aso_ctrl_flow_meter, exe_aso_ctrl, init_color,
+ fte->action.exe_aso.flow_meter.init_color);
+ MLX5_SET(exe_aso_ctrl_flow_meter, exe_aso_ctrl, meter_id,
+ fte->action.exe_aso.flow_meter.meter_idx);
+}
- inlen = MLX5_ST_SZ_BYTES(set_fte_in) +
- (dest_size + count_list) * MLX5_ST_SZ_BYTES(dest_format_struct);
+static int mlx5_cmd_set_fte(struct mlx5_core_dev *dev,
+ int opmod, int modify_mask,
+ struct mlx5_flow_table *ft,
+ unsigned group_id,
+ struct fs_fte *fte)
+{
+ u32 out[MLX5_ST_SZ_DW(set_fte_out)] = {0};
+ bool extended_dest = false;
+ struct mlx5_flow_rule *dst;
+ void *in_flow_context, *vlan;
+ void *in_match_value;
+ unsigned int inlen;
+ int dst_cnt_size;
+ void *in_dests;
+ u32 *in;
+ int err;
- if (!dev)
- return -EINVAL;
+ if (mlx5_set_extended_dest(dev, fte, &extended_dest))
+ return -EOPNOTSUPP;
- if (*fte_status & FS_FTE_STATUS_EXISTING) {
- atomic_mod_cap = MLX5_CAP_FLOWTABLE(dev,
- flow_table_properties_nic_receive.
- flow_modify_en);
- if (!atomic_mod_cap)
- return -ENOTSUPP;
- opmod = 1;
- modify_mask = 1 <<
- MLX5_SET_FTE_MODIFY_ENABLE_MASK_DESTINATION_LIST;
- }
+ if (!extended_dest)
+ dst_cnt_size = MLX5_ST_SZ_BYTES(dest_format_struct);
+ else
+ dst_cnt_size = MLX5_ST_SZ_BYTES(extended_dest_format);
- in = mlx5_vzalloc(inlen);
- if (!in) {
- mlx5_core_warn(dev, "failed to allocate inbox\n");
+ inlen = MLX5_ST_SZ_BYTES(set_fte_in) + fte->dests_size * dst_cnt_size;
+ in = kvzalloc(inlen, GFP_KERNEL);
+ if (!in)
return -ENOMEM;
- }
MLX5_SET(set_fte_in, in, opcode, MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY);
MLX5_SET(set_fte_in, in, op_mod, opmod);
MLX5_SET(set_fte_in, in, modify_enable_mask, modify_mask);
- MLX5_SET(set_fte_in, in, table_type, type);
- MLX5_SET(set_fte_in, in, table_id, table_id);
- MLX5_SET(set_fte_in, in, flow_index, index);
- if (vport) {
- MLX5_SET(set_fte_in, in, vport_number, vport);
- MLX5_SET(set_fte_in, in, other_vport, 1);
- }
+ MLX5_SET(set_fte_in, in, table_type, ft->type);
+ MLX5_SET(set_fte_in, in, table_id, ft->id);
+ MLX5_SET(set_fte_in, in, flow_index, fte->index);
+ MLX5_SET(set_fte_in, in, ignore_flow_level,
+ !!(fte->action.flags & FLOW_ACT_IGNORE_FLOW_LEVEL));
+
+ MLX5_SET(set_fte_in, in, vport_number, ft->vport);
+ MLX5_SET(set_fte_in, in, other_vport,
+ !!(ft->flags & MLX5_FLOW_TABLE_OTHER_VPORT));
in_flow_context = MLX5_ADDR_OF(set_fte_in, in, flow_context);
MLX5_SET(flow_context, in_flow_context, group_id, group_id);
- if (flow_act->actions & MLX5_FLOW_ACT_ACTIONS_FLOW_TAG)
- MLX5_SET(flow_context, in_flow_context, flow_tag, flow_act->flow_tag);
- if (flow_act->actions & MLX5_FLOW_ACT_ACTIONS_MODIFY_HDR) {
- MLX5_SET(flow_context, in_flow_context, modify_header_id,
- flow_act->modify_hdr->id);
- prm_action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
- }
- if (flow_act->actions & MLX5_FLOW_ACT_ACTIONS_PACKET_REFORMAT) {
- MLX5_SET(flow_context, in_flow_context, packet_reformat_id,
- flow_act->pkt_reformat->id);
- prm_action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
+
+ MLX5_SET(flow_context, in_flow_context, flow_tag,
+ fte->flow_context.flow_tag);
+ MLX5_SET(flow_context, in_flow_context, flow_source,
+ fte->flow_context.flow_source);
+
+ MLX5_SET(flow_context, in_flow_context, extended_destination,
+ extended_dest);
+ if (extended_dest) {
+ u32 action;
+
+ action = fte->action.action &
+ ~MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
+ MLX5_SET(flow_context, in_flow_context, action, action);
+ } else {
+ MLX5_SET(flow_context, in_flow_context, action,
+ fte->action.action);
+ if (fte->action.pkt_reformat)
+ MLX5_SET(flow_context, in_flow_context, packet_reformat_id,
+ fte->action.pkt_reformat->id);
}
- MLX5_SET(flow_context, in_flow_context, destination_list_size,
- dest_size);
+ if (fte->action.modify_hdr)
+ MLX5_SET(flow_context, in_flow_context, modify_header_id,
+ fte->action.modify_hdr->id);
+
+ MLX5_SET(flow_context, in_flow_context, encrypt_decrypt_type,
+ fte->action.crypto.type);
+ MLX5_SET(flow_context, in_flow_context, encrypt_decrypt_obj_id,
+ fte->action.crypto.obj_id);
+
+ vlan = MLX5_ADDR_OF(flow_context, in_flow_context, push_vlan);
+
+ MLX5_SET(vlan, vlan, ethtype, fte->action.vlan[0].ethtype);
+ MLX5_SET(vlan, vlan, vid, fte->action.vlan[0].vid);
+ MLX5_SET(vlan, vlan, prio, fte->action.vlan[0].prio);
+
+ vlan = MLX5_ADDR_OF(flow_context, in_flow_context, push_vlan_2);
+
+ MLX5_SET(vlan, vlan, ethtype, fte->action.vlan[1].ethtype);
+ MLX5_SET(vlan, vlan, vid, fte->action.vlan[1].vid);
+ MLX5_SET(vlan, vlan, prio, fte->action.vlan[1].prio);
+
in_match_value = MLX5_ADDR_OF(flow_context, in_flow_context,
match_value);
- memcpy(in_match_value, match_val, MLX5_ST_SZ_BYTES(fte_match_param));
+ memcpy(in_match_value, &fte->val, sizeof(fte->val));
in_dests = MLX5_ADDR_OF(flow_context, in_flow_context, destination);
+ if (fte->action.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) {
+ int list_size = 0;
- if (dest_size) {
- list_for_each_entry(dst, dests, base.list) {
+ list_for_each_entry(dst, &fte->node.children, node.list) {
+ enum mlx5_flow_destination_type type = dst->dest_attr.type;
+ enum mlx5_ifc_flow_destination_type ifc_type;
unsigned int id;
- MLX5_SET(dest_format_struct, in_dests, destination_type,
- dst->dest_attr.type);
- if (dst->dest_attr.type ==
- MLX5_FLOW_CONTEXT_DEST_TYPE_FLOW_TABLE)
+ if (type == MLX5_FLOW_DESTINATION_TYPE_COUNTER)
+ continue;
+
+ switch (type) {
+ case MLX5_FLOW_DESTINATION_TYPE_NONE:
+ continue;
+ case MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM:
+ id = dst->dest_attr.ft_num;
+ ifc_type = MLX5_IFC_FLOW_DESTINATION_TYPE_FLOW_TABLE;
+ break;
+ case MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE:
id = dst->dest_attr.ft->id;
- else
+ ifc_type = MLX5_IFC_FLOW_DESTINATION_TYPE_FLOW_TABLE;
+ break;
+ case MLX5_FLOW_DESTINATION_TYPE_UPLINK:
+ case MLX5_FLOW_DESTINATION_TYPE_VPORT:
+ MLX5_SET(dest_format_struct, in_dests,
+ destination_eswitch_owner_vhca_id_valid,
+ !!(dst->dest_attr.vport.flags &
+ MLX5_FLOW_DEST_VPORT_VHCA_ID));
+ MLX5_SET(dest_format_struct, in_dests,
+ destination_eswitch_owner_vhca_id,
+ dst->dest_attr.vport.vhca_id);
+ if (type == MLX5_FLOW_DESTINATION_TYPE_UPLINK) {
+ /* destination_id is reserved */
+ id = 0;
+ ifc_type = MLX5_IFC_FLOW_DESTINATION_TYPE_UPLINK;
+ break;
+ }
+ ifc_type = MLX5_IFC_FLOW_DESTINATION_TYPE_VPORT;
+ id = dst->dest_attr.vport.num;
+ if (extended_dest &&
+ dst->dest_attr.vport.pkt_reformat) {
+ MLX5_SET(dest_format_struct, in_dests,
+ packet_reformat,
+ !!(dst->dest_attr.vport.flags &
+ MLX5_FLOW_DEST_VPORT_REFORMAT_ID));
+ MLX5_SET(extended_dest_format, in_dests,
+ packet_reformat_id,
+ dst->dest_attr.vport.pkt_reformat->id);
+ }
+ break;
+ case MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER:
+ id = dst->dest_attr.sampler_id;
+ ifc_type = MLX5_IFC_FLOW_DESTINATION_TYPE_FLOW_SAMPLER;
+ break;
+ case MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE:
+ MLX5_SET(dest_format_struct, in_dests,
+ destination_table_type, dst->dest_attr.ft->type);
+ id = dst->dest_attr.ft->id;
+ ifc_type = MLX5_IFC_FLOW_DESTINATION_TYPE_TABLE_TYPE;
+ break;
+ default:
id = dst->dest_attr.tir_num;
+ ifc_type = MLX5_IFC_FLOW_DESTINATION_TYPE_TIR;
+ }
+
+ MLX5_SET(dest_format_struct, in_dests, destination_type,
+ ifc_type);
MLX5_SET(dest_format_struct, in_dests, destination_id, id);
- in_dests += MLX5_ST_SZ_BYTES(dest_format_struct);
+ in_dests += dst_cnt_size;
+ list_size++;
}
+
+ MLX5_SET(flow_context, in_flow_context, destination_list_size,
+ list_size);
}
- if (flow_act->actions & MLX5_FLOW_ACT_ACTIONS_COUNT) {
- MLX5_SET(dest_format_struct, in_dests, destination_id,
- mlx5_fc_id(flow_act->counter));
- in_dests += MLX5_ST_SZ_BYTES(dest_format_struct);
- MLX5_SET(flow_context, in_flow_context, flow_counter_list_size, 1);
+ if (fte->action.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
+ int max_list_size = BIT(MLX5_CAP_FLOWTABLE_TYPE(dev,
+ log_max_flow_counter,
+ ft->type));
+ int list_size = 0;
+
+ list_for_each_entry(dst, &fte->node.children, node.list) {
+ if (dst->dest_attr.type !=
+ MLX5_FLOW_DESTINATION_TYPE_COUNTER)
+ continue;
+
+ MLX5_SET(flow_counter_list, in_dests, flow_counter_id,
+ dst->dest_attr.counter_id);
+ in_dests += dst_cnt_size;
+ list_size++;
+ }
+ if (list_size > max_list_size) {
+ err = -EINVAL;
+ goto err_out;
+ }
+
+ MLX5_SET(flow_context, in_flow_context, flow_counter_list_size,
+ list_size);
}
- MLX5_SET(flow_context, in_flow_context, action, prm_action);
- err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
- if (!err)
- *fte_status |= FS_FTE_STATUS_EXISTING;
+ if (fte->action.action & MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO) {
+ if (fte->action.exe_aso.type == MLX5_EXE_ASO_FLOW_METER) {
+ mlx5_cmd_set_fte_flow_meter(fte, in_flow_context);
+ } else {
+ err = -EOPNOTSUPP;
+ goto err_out;
+ }
+ }
+ err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
+err_out:
kvfree(in);
+ return err;
+}
+
+static int mlx5_cmd_create_fte(struct mlx5_flow_root_namespace *ns,
+ struct mlx5_flow_table *ft,
+ struct mlx5_flow_group *group,
+ struct fs_fte *fte)
+{
+ struct mlx5_core_dev *dev = ns->dev;
+ unsigned int group_id = group->id;
+
+ return mlx5_cmd_set_fte(dev, 0, 0, ft, group_id, fte);
+}
+
+static int mlx5_cmd_update_fte(struct mlx5_flow_root_namespace *ns,
+ struct mlx5_flow_table *ft,
+ struct mlx5_flow_group *fg,
+ int modify_mask,
+ struct fs_fte *fte)
+{
+ int opmod;
+ struct mlx5_core_dev *dev = ns->dev;
+ int atomic_mod_cap = MLX5_CAP_FLOWTABLE(dev,
+ flow_table_properties_nic_receive.
+ flow_modify_en);
+ if (!atomic_mod_cap)
+ return -EOPNOTSUPP;
+ opmod = 1;
+
+ return mlx5_cmd_set_fte(dev, opmod, modify_mask, ft, fg->id, fte);
+}
+
+static int mlx5_cmd_delete_fte(struct mlx5_flow_root_namespace *ns,
+ struct mlx5_flow_table *ft,
+ struct fs_fte *fte)
+{
+ u32 in[MLX5_ST_SZ_DW(delete_fte_in)] = {};
+ struct mlx5_core_dev *dev = ns->dev;
+ MLX5_SET(delete_fte_in, in, opcode, MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY);
+ MLX5_SET(delete_fte_in, in, table_type, ft->type);
+ MLX5_SET(delete_fte_in, in, table_id, ft->id);
+ MLX5_SET(delete_fte_in, in, flow_index, fte->index);
+ MLX5_SET(delete_fte_in, in, vport_number, ft->vport);
+ MLX5_SET(delete_fte_in, in, other_vport,
+ !!(ft->flags & MLX5_FLOW_TABLE_OTHER_VPORT));
+
+ return mlx5_cmd_exec_in(dev, delete_fte, in);
+}
+
+int mlx5_cmd_fc_bulk_alloc(struct mlx5_core_dev *dev,
+ enum mlx5_fc_bulk_alloc_bitmask alloc_bitmask,
+ u32 *id)
+{
+ u32 out[MLX5_ST_SZ_DW(alloc_flow_counter_out)] = {};
+ u32 in[MLX5_ST_SZ_DW(alloc_flow_counter_in)] = {};
+ int err;
+
+ MLX5_SET(alloc_flow_counter_in, in, opcode,
+ MLX5_CMD_OP_ALLOC_FLOW_COUNTER);
+ MLX5_SET(alloc_flow_counter_in, in, flow_counter_bulk, alloc_bitmask);
+
+ err = mlx5_cmd_exec_inout(dev, alloc_flow_counter, in, out);
+ if (!err)
+ *id = MLX5_GET(alloc_flow_counter_out, out, flow_counter_id);
return err;
}
-int mlx5_cmd_fs_delete_fte(struct mlx5_core_dev *dev,
- u16 vport,
- enum fs_fte_status *fte_status,
- enum fs_ft_type type, unsigned int table_id,
- unsigned int index)
+int mlx5_cmd_fc_alloc(struct mlx5_core_dev *dev, u32 *id)
+{
+ return mlx5_cmd_fc_bulk_alloc(dev, 0, id);
+}
+
+int mlx5_cmd_fc_free(struct mlx5_core_dev *dev, u32 id)
+{
+ u32 in[MLX5_ST_SZ_DW(dealloc_flow_counter_in)] = {};
+
+ MLX5_SET(dealloc_flow_counter_in, in, opcode,
+ MLX5_CMD_OP_DEALLOC_FLOW_COUNTER);
+ MLX5_SET(dealloc_flow_counter_in, in, flow_counter_id, id);
+ return mlx5_cmd_exec_in(dev, dealloc_flow_counter, in);
+}
+
+int mlx5_cmd_fc_query(struct mlx5_core_dev *dev, u32 id,
+ u64 *packets, u64 *bytes)
+{
+ u32 out[MLX5_ST_SZ_BYTES(query_flow_counter_out) +
+ MLX5_ST_SZ_BYTES(traffic_counter)] = {};
+ u32 in[MLX5_ST_SZ_DW(query_flow_counter_in)] = {};
+ void *stats;
+ int err = 0;
+
+ MLX5_SET(query_flow_counter_in, in, opcode,
+ MLX5_CMD_OP_QUERY_FLOW_COUNTER);
+ MLX5_SET(query_flow_counter_in, in, op_mod, 0);
+ MLX5_SET(query_flow_counter_in, in, flow_counter_id, id);
+ err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+ if (err)
+ return err;
+
+ stats = MLX5_ADDR_OF(query_flow_counter_out, out, flow_statistics);
+ *packets = MLX5_GET64(traffic_counter, stats, packets);
+ *bytes = MLX5_GET64(traffic_counter, stats, octets);
+ return 0;
+}
+
+int mlx5_cmd_fc_get_bulk_query_out_len(int bulk_len)
{
- u32 in[MLX5_ST_SZ_DW(delete_fte_in)] = {0};
- u32 out[MLX5_ST_SZ_DW(delete_fte_out)] = {0};
+ return MLX5_ST_SZ_BYTES(query_flow_counter_out) +
+ MLX5_ST_SZ_BYTES(traffic_counter) * bulk_len;
+}
+
+int mlx5_cmd_fc_bulk_query(struct mlx5_core_dev *dev, u32 base_id, int bulk_len,
+ u32 *out)
+{
+ int outlen = mlx5_cmd_fc_get_bulk_query_out_len(bulk_len);
+ u32 in[MLX5_ST_SZ_DW(query_flow_counter_in)] = {};
+
+ MLX5_SET(query_flow_counter_in, in, opcode,
+ MLX5_CMD_OP_QUERY_FLOW_COUNTER);
+ MLX5_SET(query_flow_counter_in, in, flow_counter_id, base_id);
+ MLX5_SET(query_flow_counter_in, in, num_of_counters, bulk_len);
+ return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen);
+}
+
+static int mlx5_cmd_packet_reformat_alloc(struct mlx5_flow_root_namespace *ns,
+ struct mlx5_pkt_reformat_params *params,
+ enum mlx5_flow_namespace_type namespace,
+ struct mlx5_pkt_reformat *pkt_reformat)
+{
+ u32 out[MLX5_ST_SZ_DW(alloc_packet_reformat_context_out)] = {};
+ struct mlx5_core_dev *dev = ns->dev;
+ void *packet_reformat_context_in;
+ int max_encap_size;
+ void *reformat;
+ int inlen;
int err;
+ u32 *in;
- if (!(*fte_status & FS_FTE_STATUS_EXISTING))
- return 0;
+ if (namespace == MLX5_FLOW_NAMESPACE_FDB ||
+ namespace == MLX5_FLOW_NAMESPACE_FDB_BYPASS)
+ max_encap_size = MLX5_CAP_ESW(dev, max_encap_header_size);
+ else
+ max_encap_size = MLX5_CAP_FLOWTABLE(dev, max_encap_header_size);
- if (!dev)
+ if (params->size > max_encap_size) {
+ mlx5_core_warn(dev, "encap size %zd too big, max supported is %d\n",
+ params->size, max_encap_size);
return -EINVAL;
+ }
- MLX5_SET(delete_fte_in, in, opcode, MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY);
- MLX5_SET(delete_fte_in, in, table_type, type);
- MLX5_SET(delete_fte_in, in, table_id, table_id);
- MLX5_SET(delete_fte_in, in, flow_index, index);
- if (vport) {
- MLX5_SET(delete_fte_in, in, vport_number, vport);
- MLX5_SET(delete_fte_in, in, other_vport, 1);
+ in = kzalloc(MLX5_ST_SZ_BYTES(alloc_packet_reformat_context_in) +
+ params->size, GFP_KERNEL);
+ if (!in)
+ return -ENOMEM;
+
+ packet_reformat_context_in = MLX5_ADDR_OF(alloc_packet_reformat_context_in,
+ in, packet_reformat_context);
+ reformat = MLX5_ADDR_OF(packet_reformat_context_in,
+ packet_reformat_context_in,
+ reformat_data);
+ inlen = reformat - (void *)in + params->size;
+
+ MLX5_SET(alloc_packet_reformat_context_in, in, opcode,
+ MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT);
+ MLX5_SET(packet_reformat_context_in, packet_reformat_context_in,
+ reformat_data_size, params->size);
+ MLX5_SET(packet_reformat_context_in, packet_reformat_context_in,
+ reformat_type, params->type);
+ MLX5_SET(packet_reformat_context_in, packet_reformat_context_in,
+ reformat_param_0, params->param_0);
+ MLX5_SET(packet_reformat_context_in, packet_reformat_context_in,
+ reformat_param_1, params->param_1);
+ if (params->data && params->size)
+ memcpy(reformat, params->data, params->size);
+
+ err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
+
+ pkt_reformat->id = MLX5_GET(alloc_packet_reformat_context_out,
+ out, packet_reformat_id);
+ kfree(in);
+ return err;
+}
+
+static void mlx5_cmd_packet_reformat_dealloc(struct mlx5_flow_root_namespace *ns,
+ struct mlx5_pkt_reformat *pkt_reformat)
+{
+ u32 in[MLX5_ST_SZ_DW(dealloc_packet_reformat_context_in)] = {};
+ struct mlx5_core_dev *dev = ns->dev;
+
+ MLX5_SET(dealloc_packet_reformat_context_in, in, opcode,
+ MLX5_CMD_OP_DEALLOC_PACKET_REFORMAT_CONTEXT);
+ MLX5_SET(dealloc_packet_reformat_context_in, in, packet_reformat_id,
+ pkt_reformat->id);
+
+ mlx5_cmd_exec_in(dev, dealloc_packet_reformat_context, in);
+}
+
+static int mlx5_cmd_modify_header_alloc(struct mlx5_flow_root_namespace *ns,
+ u8 namespace, u8 num_actions,
+ void *modify_actions,
+ struct mlx5_modify_hdr *modify_hdr)
+{
+ u32 out[MLX5_ST_SZ_DW(alloc_modify_header_context_out)] = {};
+ int max_actions, actions_size, inlen, err;
+ struct mlx5_core_dev *dev = ns->dev;
+ void *actions_in;
+ u8 table_type;
+ u32 *in;
+
+ switch (namespace) {
+ case MLX5_FLOW_NAMESPACE_FDB:
+ case MLX5_FLOW_NAMESPACE_FDB_BYPASS:
+ max_actions = MLX5_CAP_ESW_FLOWTABLE_FDB(dev, max_modify_header_actions);
+ table_type = FS_FT_FDB;
+ break;
+ case MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC:
+ case MLX5_FLOW_NAMESPACE_KERNEL:
+ case MLX5_FLOW_NAMESPACE_BYPASS:
+ max_actions = MLX5_CAP_FLOWTABLE_NIC_RX(dev, max_modify_header_actions);
+ table_type = FS_FT_NIC_RX;
+ break;
+ case MLX5_FLOW_NAMESPACE_EGRESS:
+ case MLX5_FLOW_NAMESPACE_EGRESS_IPSEC:
+ case MLX5_FLOW_NAMESPACE_EGRESS_MACSEC:
+ max_actions = MLX5_CAP_FLOWTABLE_NIC_TX(dev, max_modify_header_actions);
+ table_type = FS_FT_NIC_TX;
+ break;
+ case MLX5_FLOW_NAMESPACE_ESW_INGRESS:
+ max_actions = MLX5_CAP_ESW_INGRESS_ACL(dev, max_modify_header_actions);
+ table_type = FS_FT_ESW_INGRESS_ACL;
+ break;
+ case MLX5_FLOW_NAMESPACE_RDMA_TX:
+ max_actions = MLX5_CAP_FLOWTABLE_RDMA_TX(dev, max_modify_header_actions);
+ table_type = FS_FT_RDMA_TX;
+ break;
+ default:
+ return -EOPNOTSUPP;
}
- err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
- if (!err)
- *fte_status = 0;
+ if (num_actions > max_actions) {
+ mlx5_core_warn(dev, "too many modify header actions %d, max supported %d\n",
+ num_actions, max_actions);
+ return -EOPNOTSUPP;
+ }
+
+ actions_size = MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto) * num_actions;
+ inlen = MLX5_ST_SZ_BYTES(alloc_modify_header_context_in) + actions_size;
+
+ in = kzalloc(inlen, GFP_KERNEL);
+ if (!in)
+ return -ENOMEM;
+
+ MLX5_SET(alloc_modify_header_context_in, in, opcode,
+ MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT);
+ MLX5_SET(alloc_modify_header_context_in, in, table_type, table_type);
+ MLX5_SET(alloc_modify_header_context_in, in, num_of_actions, num_actions);
+
+ actions_in = MLX5_ADDR_OF(alloc_modify_header_context_in, in, actions);
+ memcpy(actions_in, modify_actions, actions_size);
+ err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
+
+ modify_hdr->id = MLX5_GET(alloc_modify_header_context_out, out, modify_header_id);
+ kfree(in);
return err;
}
-int mlx5_cmd_modify_header_alloc(struct mlx5_core_dev *dev,
- enum mlx5_flow_namespace_type namespace,
- u8 num_actions,
- void *modify_actions,
- struct mlx5_modify_hdr *modify_hdr)
-{
- u32 out[MLX5_ST_SZ_DW(alloc_modify_header_context_out)] = {};
- int max_actions, actions_size, inlen, err;
- void *actions_in;
- u8 table_type;
- u32 *in;
-
- switch (namespace) {
- case MLX5_FLOW_NAMESPACE_FDB:
- max_actions = MLX5_CAP_ESW_FLOWTABLE_FDB(dev, max_modify_header_actions);
- table_type = FS_FT_FDB;
- break;
- case MLX5_FLOW_NAMESPACE_KERNEL:
- case MLX5_FLOW_NAMESPACE_BYPASS:
- max_actions = MLX5_CAP_FLOWTABLE_NIC_RX(dev, max_modify_header_actions);
- table_type = FS_FT_NIC_RX;
- break;
- case MLX5_FLOW_NAMESPACE_ESW_INGRESS:
- max_actions = MLX5_CAP_ESW_INGRESS_ACL(dev, max_modify_header_actions);
- table_type = FS_FT_ESW_INGRESS_ACL;
- break;
- default:
- return -EOPNOTSUPP;
- }
-
- if (num_actions > max_actions) {
- mlx5_core_warn(dev, "too many modify header actions %d, max supported %d\n",
- num_actions, max_actions);
- return -EOPNOTSUPP;
- }
-
- actions_size = MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto) * num_actions;
- inlen = MLX5_ST_SZ_BYTES(alloc_modify_header_context_in) + actions_size;
-
- in = kzalloc(inlen, GFP_KERNEL);
- if (!in)
- return -ENOMEM;
-
- MLX5_SET(alloc_modify_header_context_in, in, opcode,
- MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT);
- MLX5_SET(alloc_modify_header_context_in, in, table_type, table_type);
- MLX5_SET(alloc_modify_header_context_in, in, num_of_actions, num_actions);
-
- actions_in = MLX5_ADDR_OF(alloc_modify_header_context_in, in, actions);
- memcpy(actions_in, modify_actions, actions_size);
-
- err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
-
- modify_hdr->id = MLX5_GET(alloc_modify_header_context_out, out, modify_header_id);
- kfree(in);
-
- return err;
-}
-
-void mlx5_cmd_modify_header_dealloc(struct mlx5_core_dev *dev,
- struct mlx5_modify_hdr *modify_hdr)
-{
- u32 out[MLX5_ST_SZ_DW(dealloc_modify_header_context_out)] = {};
- u32 in[MLX5_ST_SZ_DW(dealloc_modify_header_context_in)] = {};
-
- MLX5_SET(dealloc_modify_header_context_in, in, opcode,
- MLX5_CMD_OP_DEALLOC_MODIFY_HEADER_CONTEXT);
- MLX5_SET(dealloc_modify_header_context_in, in, modify_header_id,
- modify_hdr->id);
-
- mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
-}
-
-int mlx5_cmd_packet_reformat_alloc(struct mlx5_core_dev *dev,
- struct mlx5_pkt_reformat_params *params,
- enum mlx5_flow_namespace_type namespace,
- struct mlx5_pkt_reformat *pkt_reformat)
-{
- u32 out[MLX5_ST_SZ_DW(alloc_packet_reformat_context_out)] = {};
- void *packet_reformat_context_in;
- int max_encap_size;
- void *reformat;
- int inlen;
- int err;
- u32 *in;
-
- if (namespace == MLX5_FLOW_NAMESPACE_FDB)
- max_encap_size = MLX5_CAP_ESW(dev, max_encap_header_size);
- else
- max_encap_size = MLX5_CAP_FLOWTABLE(dev, max_encap_header_size);
-
- if (params->size > max_encap_size) {
- mlx5_core_warn(dev, "encap size %zd too big, max supported is %d\n",
- params->size, max_encap_size);
- return -EINVAL;
- }
-
- in = kzalloc(MLX5_ST_SZ_BYTES(alloc_packet_reformat_context_in) +
- params->size, GFP_KERNEL);
- if (!in)
- return -ENOMEM;
-
- packet_reformat_context_in = MLX5_ADDR_OF(alloc_packet_reformat_context_in,
- in, packet_reformat_context);
- reformat = MLX5_ADDR_OF(packet_reformat_context_in,
- packet_reformat_context_in,
- reformat_data);
- inlen = reformat - (void *)in + params->size;
-
- MLX5_SET(alloc_packet_reformat_context_in, in, opcode,
- MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT);
- MLX5_SET(packet_reformat_context_in, packet_reformat_context_in,
- reformat_data_size, params->size);
- MLX5_SET(packet_reformat_context_in, packet_reformat_context_in,
- reformat_type, params->type);
- MLX5_SET(packet_reformat_context_in, packet_reformat_context_in,
- reformat_param_0, params->param_0);
- MLX5_SET(packet_reformat_context_in, packet_reformat_context_in,
- reformat_param_1, params->param_1);
- if (params->data && params->size)
- memcpy(reformat, params->data, params->size);
-
- err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
-
- pkt_reformat->id = MLX5_GET(alloc_packet_reformat_context_out,
- out, packet_reformat_id);
- kfree(in);
+static void mlx5_cmd_modify_header_dealloc(struct mlx5_flow_root_namespace *ns,
+ struct mlx5_modify_hdr *modify_hdr)
+{
+ u32 in[MLX5_ST_SZ_DW(dealloc_modify_header_context_in)] = {};
+ struct mlx5_core_dev *dev = ns->dev;
+
+ MLX5_SET(dealloc_modify_header_context_in, in, opcode,
+ MLX5_CMD_OP_DEALLOC_MODIFY_HEADER_CONTEXT);
+ MLX5_SET(dealloc_modify_header_context_in, in, modify_header_id,
+ modify_hdr->id);
- return err;
+ mlx5_cmd_exec_in(dev, dealloc_modify_header_context, in);
}
-void mlx5_cmd_packet_reformat_dealloc(struct mlx5_core_dev *dev,
- struct mlx5_pkt_reformat *pkt_reformat)
+static u32 mlx5_cmd_get_capabilities(struct mlx5_flow_root_namespace *ns,
+ enum fs_flow_table_type ft_type)
{
- u32 out[MLX5_ST_SZ_DW(dealloc_packet_reformat_context_out)] = {};
- u32 in[MLX5_ST_SZ_DW(dealloc_packet_reformat_context_in)] = {};
+ return 0;
+}
- MLX5_SET(dealloc_packet_reformat_context_in, in, opcode,
- MLX5_CMD_OP_DEALLOC_PACKET_REFORMAT_CONTEXT);
- MLX5_SET(dealloc_packet_reformat_context_in, in, packet_reformat_id,
- pkt_reformat->id);
+static const struct mlx5_flow_cmds mlx5_flow_cmds = {
+ .create_flow_table = mlx5_cmd_create_flow_table,
+ .destroy_flow_table = mlx5_cmd_destroy_flow_table,
+ .modify_flow_table = mlx5_cmd_modify_flow_table,
+ .create_flow_group = mlx5_cmd_create_flow_group,
+ .destroy_flow_group = mlx5_cmd_destroy_flow_group,
+ .create_fte = mlx5_cmd_create_fte,
+ .update_fte = mlx5_cmd_update_fte,
+ .delete_fte = mlx5_cmd_delete_fte,
+ .update_root_ft = mlx5_cmd_update_root_ft,
+ .packet_reformat_alloc = mlx5_cmd_packet_reformat_alloc,
+ .packet_reformat_dealloc = mlx5_cmd_packet_reformat_dealloc,
+ .modify_header_alloc = mlx5_cmd_modify_header_alloc,
+ .modify_header_dealloc = mlx5_cmd_modify_header_dealloc,
+ .set_peer = mlx5_cmd_stub_set_peer,
+ .create_ns = mlx5_cmd_stub_create_ns,
+ .destroy_ns = mlx5_cmd_stub_destroy_ns,
+ .get_capabilities = mlx5_cmd_get_capabilities,
+};
+
+static const struct mlx5_flow_cmds mlx5_flow_cmd_stubs = {
+ .create_flow_table = mlx5_cmd_stub_create_flow_table,
+ .destroy_flow_table = mlx5_cmd_stub_destroy_flow_table,
+ .modify_flow_table = mlx5_cmd_stub_modify_flow_table,
+ .create_flow_group = mlx5_cmd_stub_create_flow_group,
+ .destroy_flow_group = mlx5_cmd_stub_destroy_flow_group,
+ .create_fte = mlx5_cmd_stub_create_fte,
+ .update_fte = mlx5_cmd_stub_update_fte,
+ .delete_fte = mlx5_cmd_stub_delete_fte,
+ .update_root_ft = mlx5_cmd_stub_update_root_ft,
+ .packet_reformat_alloc = mlx5_cmd_stub_packet_reformat_alloc,
+ .packet_reformat_dealloc = mlx5_cmd_stub_packet_reformat_dealloc,
+ .modify_header_alloc = mlx5_cmd_stub_modify_header_alloc,
+ .modify_header_dealloc = mlx5_cmd_stub_modify_header_dealloc,
+ .set_peer = mlx5_cmd_stub_set_peer,
+ .create_ns = mlx5_cmd_stub_create_ns,
+ .destroy_ns = mlx5_cmd_stub_destroy_ns,
+ .get_capabilities = mlx5_cmd_stub_get_capabilities,
+};
+
+const struct mlx5_flow_cmds *mlx5_fs_cmd_get_fw_cmds(void)
+{
+ return &mlx5_flow_cmds;
+}
+
+static const struct mlx5_flow_cmds *mlx5_fs_cmd_get_stub_cmds(void)
+{
+ return &mlx5_flow_cmd_stubs;
+}
- mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+const struct mlx5_flow_cmds *mlx5_fs_cmd_get_default(enum fs_flow_table_type type)
+{
+ switch (type) {
+ case FS_FT_NIC_RX:
+ case FS_FT_ESW_EGRESS_ACL:
+ case FS_FT_ESW_INGRESS_ACL:
+ case FS_FT_FDB:
+ case FS_FT_SNIFFER_RX:
+ case FS_FT_SNIFFER_TX:
+ case FS_FT_NIC_TX:
+ case FS_FT_RDMA_RX:
+ case FS_FT_RDMA_TX:
+ case FS_FT_PORT_SEL:
+ return mlx5_fs_cmd_get_fw_cmds();
+ default:
+ return mlx5_fs_cmd_get_stub_cmds();
+ }
}
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fs_core.c b/sys/dev/mlx5/mlx5_core/mlx5_fs_core.c
new file mode 100644
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_core/mlx5_fs_core.c
@@ -0,0 +1,3514 @@
+/*
+ * Copyright (c) 2015, Mellanox Technologies. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/module.h>
+#include <dev/mlx5/driver.h>
+#include <dev/mlx5/mlx5_core/mlx5_core.h>
+#include <dev/mlx5/mlx5_core/fs_core.h>
+#include <linux/string.h>
+#include <linux/compiler.h>
+
+#include "eswitch.h"
+#include "fs_ft_pool.h"
+#include "fs_cmd.h"
+
+#define down_write_nested(a, b) down_write(a)
+
+#define INIT_TREE_NODE_ARRAY_SIZE(...) (sizeof((struct init_tree_node[]){__VA_ARGS__}) /\
+ sizeof(struct init_tree_node))
+
+#define ADD_PRIO(num_prios_val, min_level_val, num_levels_val, caps_val,\
+ ...) {.type = FS_TYPE_PRIO,\
+ .min_ft_level = min_level_val,\
+ .num_levels = num_levels_val,\
+ .num_leaf_prios = num_prios_val,\
+ .caps = caps_val,\
+ .children = (struct init_tree_node[]) {__VA_ARGS__},\
+ .ar_size = INIT_TREE_NODE_ARRAY_SIZE(__VA_ARGS__) \
+}
+
+#define ADD_MULTIPLE_PRIO(num_prios_val, num_levels_val, ...)\
+ ADD_PRIO(num_prios_val, 0, num_levels_val, {},\
+ __VA_ARGS__)\
+
+#define ADD_NS(def_miss_act, ...) {.type = FS_TYPE_NAMESPACE, \
+ .def_miss_action = def_miss_act,\
+ .children = (struct init_tree_node[]) {__VA_ARGS__},\
+ .ar_size = INIT_TREE_NODE_ARRAY_SIZE(__VA_ARGS__) \
+}
+
+#define INIT_CAPS_ARRAY_SIZE(...) (sizeof((long[]){__VA_ARGS__}) /\
+ sizeof(long))
+
+#define FS_CAP(cap) (__mlx5_bit_off(flow_table_nic_cap, cap))
+
+#define FS_REQUIRED_CAPS(...) {.arr_sz = INIT_CAPS_ARRAY_SIZE(__VA_ARGS__), \
+ .caps = (long[]) {__VA_ARGS__} }
+
+#define FS_CHAINING_CAPS FS_REQUIRED_CAPS(FS_CAP(flow_table_properties_nic_receive.flow_modify_en), \
+ FS_CAP(flow_table_properties_nic_receive.modify_root), \
+ FS_CAP(flow_table_properties_nic_receive.identified_miss_table_mode), \
+ FS_CAP(flow_table_properties_nic_receive.flow_table_modify))
+
+#define FS_CHAINING_CAPS_EGRESS \
+ FS_REQUIRED_CAPS( \
+ FS_CAP(flow_table_properties_nic_transmit.flow_modify_en), \
+ FS_CAP(flow_table_properties_nic_transmit.modify_root), \
+ FS_CAP(flow_table_properties_nic_transmit \
+ .identified_miss_table_mode), \
+ FS_CAP(flow_table_properties_nic_transmit.flow_table_modify))
+
+#define FS_CHAINING_CAPS_RDMA_TX \
+ FS_REQUIRED_CAPS( \
+ FS_CAP(flow_table_properties_nic_transmit_rdma.flow_modify_en), \
+ FS_CAP(flow_table_properties_nic_transmit_rdma.modify_root), \
+ FS_CAP(flow_table_properties_nic_transmit_rdma \
+ .identified_miss_table_mode), \
+ FS_CAP(flow_table_properties_nic_transmit_rdma \
+ .flow_table_modify))
+
+#define LEFTOVERS_NUM_LEVELS 1
+#define LEFTOVERS_NUM_PRIOS 1
+
+#define RDMA_RX_COUNTERS_PRIO_NUM_LEVELS 1
+#define RDMA_TX_COUNTERS_PRIO_NUM_LEVELS 1
+
+#define BY_PASS_PRIO_NUM_LEVELS 1
+#define BY_PASS_MIN_LEVEL (ETHTOOL_MIN_LEVEL + MLX5_BY_PASS_NUM_PRIOS +\
+ LEFTOVERS_NUM_PRIOS)
+
+#define KERNEL_RX_MACSEC_NUM_PRIOS 1
+#define KERNEL_RX_MACSEC_NUM_LEVELS 2
+#define KERNEL_RX_MACSEC_MIN_LEVEL (BY_PASS_MIN_LEVEL + KERNEL_RX_MACSEC_NUM_PRIOS)
+
+#define ETHTOOL_PRIO_NUM_LEVELS 1
+#define ETHTOOL_NUM_PRIOS 11
+#define ETHTOOL_MIN_LEVEL (KERNEL_MIN_LEVEL + ETHTOOL_NUM_PRIOS)
+/* Promiscuous, Vlan, mac, ttc, inner ttc, {UDP/ANY/aRFS/accel/{esp, esp_err}}, IPsec policy,
+ * IPsec RoCE policy
+ */
+#define KERNEL_NIC_PRIO_NUM_LEVELS 9
+#define KERNEL_NIC_NUM_PRIOS 1
+/* One more level for tc */
+#define KERNEL_MIN_LEVEL (KERNEL_NIC_PRIO_NUM_LEVELS + 1)
+
+#define KERNEL_NIC_TC_NUM_PRIOS 1
+#define KERNEL_NIC_TC_NUM_LEVELS 13
+
+#define ANCHOR_NUM_LEVELS 1
+#define ANCHOR_NUM_PRIOS 1
+#define ANCHOR_MIN_LEVEL (BY_PASS_MIN_LEVEL + 1)
+
+#define OFFLOADS_MAX_FT 2
+#define OFFLOADS_NUM_PRIOS 1
+#define OFFLOADS_MIN_LEVEL (ANCHOR_MIN_LEVEL + OFFLOADS_NUM_PRIOS)
+
+#define LAG_PRIO_NUM_LEVELS 1
+#define LAG_NUM_PRIOS 1
+#define LAG_MIN_LEVEL (OFFLOADS_MIN_LEVEL + KERNEL_RX_MACSEC_MIN_LEVEL + 1)
+
+#define KERNEL_TX_IPSEC_NUM_PRIOS 1
+#define KERNEL_TX_IPSEC_NUM_LEVELS 3
+#define KERNEL_TX_IPSEC_MIN_LEVEL (KERNEL_TX_IPSEC_NUM_LEVELS)
+
+#define KERNEL_TX_MACSEC_NUM_PRIOS 1
+#define KERNEL_TX_MACSEC_NUM_LEVELS 2
+#define KERNEL_TX_MACSEC_MIN_LEVEL (KERNEL_TX_IPSEC_MIN_LEVEL + KERNEL_TX_MACSEC_NUM_PRIOS)
+
+struct node_caps {
+ size_t arr_sz;
+ long *caps;
+};
+
+static struct init_tree_node {
+ enum fs_node_type type;
+ struct init_tree_node *children;
+ int ar_size;
+ struct node_caps caps;
+ int min_ft_level;
+ int num_leaf_prios;
+ int prio;
+ int num_levels;
+ enum mlx5_flow_table_miss_action def_miss_action;
+} root_fs = {
+ .type = FS_TYPE_NAMESPACE,
+ .ar_size = 8,
+ .children = (struct init_tree_node[]){
+ ADD_PRIO(0, BY_PASS_MIN_LEVEL, 0, FS_CHAINING_CAPS,
+ ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
+ ADD_MULTIPLE_PRIO(MLX5_BY_PASS_NUM_PRIOS,
+ BY_PASS_PRIO_NUM_LEVELS))),
+ ADD_PRIO(0, KERNEL_RX_MACSEC_MIN_LEVEL, 0, FS_CHAINING_CAPS,
+ ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
+ ADD_MULTIPLE_PRIO(KERNEL_RX_MACSEC_NUM_PRIOS,
+ KERNEL_RX_MACSEC_NUM_LEVELS))),
+ ADD_PRIO(0, LAG_MIN_LEVEL, 0, FS_CHAINING_CAPS,
+ ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
+ ADD_MULTIPLE_PRIO(LAG_NUM_PRIOS,
+ LAG_PRIO_NUM_LEVELS))),
+ ADD_PRIO(0, OFFLOADS_MIN_LEVEL, 0, FS_CHAINING_CAPS,
+ ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
+ ADD_MULTIPLE_PRIO(OFFLOADS_NUM_PRIOS,
+ OFFLOADS_MAX_FT))),
+ ADD_PRIO(0, ETHTOOL_MIN_LEVEL, 0, FS_CHAINING_CAPS,
+ ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
+ ADD_MULTIPLE_PRIO(ETHTOOL_NUM_PRIOS,
+ ETHTOOL_PRIO_NUM_LEVELS))),
+ ADD_PRIO(0, KERNEL_MIN_LEVEL, 0, {},
+ ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
+ ADD_MULTIPLE_PRIO(KERNEL_NIC_TC_NUM_PRIOS,
+ KERNEL_NIC_TC_NUM_LEVELS),
+ ADD_MULTIPLE_PRIO(KERNEL_NIC_NUM_PRIOS,
+ KERNEL_NIC_PRIO_NUM_LEVELS))),
+ ADD_PRIO(0, BY_PASS_MIN_LEVEL, 0, FS_CHAINING_CAPS,
+ ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
+ ADD_MULTIPLE_PRIO(LEFTOVERS_NUM_PRIOS,
+ LEFTOVERS_NUM_LEVELS))),
+ ADD_PRIO(0, ANCHOR_MIN_LEVEL, 0, {},
+ ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
+ ADD_MULTIPLE_PRIO(ANCHOR_NUM_PRIOS,
+ ANCHOR_NUM_LEVELS))),
+ }
+};
+
+static struct init_tree_node egress_root_fs = {
+ .type = FS_TYPE_NAMESPACE,
+ .ar_size = 3,
+ .children = (struct init_tree_node[]) {
+ ADD_PRIO(0, MLX5_BY_PASS_NUM_PRIOS, 0,
+ FS_CHAINING_CAPS_EGRESS,
+ ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
+ ADD_MULTIPLE_PRIO(MLX5_BY_PASS_NUM_PRIOS,
+ BY_PASS_PRIO_NUM_LEVELS))),
+ ADD_PRIO(0, KERNEL_TX_IPSEC_MIN_LEVEL, 0,
+ FS_CHAINING_CAPS_EGRESS,
+ ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
+ ADD_MULTIPLE_PRIO(KERNEL_TX_IPSEC_NUM_PRIOS,
+ KERNEL_TX_IPSEC_NUM_LEVELS))),
+ ADD_PRIO(0, KERNEL_TX_MACSEC_MIN_LEVEL, 0,
+ FS_CHAINING_CAPS_EGRESS,
+ ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
+ ADD_MULTIPLE_PRIO(KERNEL_TX_MACSEC_NUM_PRIOS,
+ KERNEL_TX_MACSEC_NUM_LEVELS))),
+ }
+};
+
+enum {
+ RDMA_RX_IPSEC_PRIO,
+ RDMA_RX_COUNTERS_PRIO,
+ RDMA_RX_BYPASS_PRIO,
+ RDMA_RX_KERNEL_PRIO,
+};
+
+#define RDMA_RX_IPSEC_NUM_PRIOS 1
+#define RDMA_RX_IPSEC_NUM_LEVELS 2
+#define RDMA_RX_IPSEC_MIN_LEVEL (RDMA_RX_IPSEC_NUM_LEVELS)
+
+#define RDMA_RX_BYPASS_MIN_LEVEL MLX5_BY_PASS_NUM_REGULAR_PRIOS
+#define RDMA_RX_KERNEL_MIN_LEVEL (RDMA_RX_BYPASS_MIN_LEVEL + 1)
+#define RDMA_RX_COUNTERS_MIN_LEVEL (RDMA_RX_KERNEL_MIN_LEVEL + 2)
+
+static struct init_tree_node rdma_rx_root_fs = {
+ .type = FS_TYPE_NAMESPACE,
+ .ar_size = 4,
+ .children = (struct init_tree_node[]) {
+ [RDMA_RX_IPSEC_PRIO] =
+ ADD_PRIO(0, RDMA_RX_IPSEC_MIN_LEVEL, 0,
+ FS_CHAINING_CAPS,
+ ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
+ ADD_MULTIPLE_PRIO(RDMA_RX_IPSEC_NUM_PRIOS,
+ RDMA_RX_IPSEC_NUM_LEVELS))),
+ [RDMA_RX_COUNTERS_PRIO] =
+ ADD_PRIO(0, RDMA_RX_COUNTERS_MIN_LEVEL, 0,
+ FS_CHAINING_CAPS,
+ ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
+ ADD_MULTIPLE_PRIO(MLX5_RDMA_RX_NUM_COUNTERS_PRIOS,
+ RDMA_RX_COUNTERS_PRIO_NUM_LEVELS))),
+ [RDMA_RX_BYPASS_PRIO] =
+ ADD_PRIO(0, RDMA_RX_BYPASS_MIN_LEVEL, 0,
+ FS_CHAINING_CAPS,
+ ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
+ ADD_MULTIPLE_PRIO(MLX5_BY_PASS_NUM_REGULAR_PRIOS,
+ BY_PASS_PRIO_NUM_LEVELS))),
+ [RDMA_RX_KERNEL_PRIO] =
+ ADD_PRIO(0, RDMA_RX_KERNEL_MIN_LEVEL, 0,
+ FS_CHAINING_CAPS,
+ ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_SWITCH_DOMAIN,
+ ADD_MULTIPLE_PRIO(1, 1))),
+ }
+};
+
+enum {
+ RDMA_TX_COUNTERS_PRIO,
+ RDMA_TX_IPSEC_PRIO,
+ RDMA_TX_BYPASS_PRIO,
+};
+
+#define RDMA_TX_BYPASS_MIN_LEVEL MLX5_BY_PASS_NUM_PRIOS
+#define RDMA_TX_COUNTERS_MIN_LEVEL (RDMA_TX_BYPASS_MIN_LEVEL + 1)
+
+#define RDMA_TX_IPSEC_NUM_PRIOS 1
+#define RDMA_TX_IPSEC_PRIO_NUM_LEVELS 1
+#define RDMA_TX_IPSEC_MIN_LEVEL (RDMA_TX_COUNTERS_MIN_LEVEL + RDMA_TX_IPSEC_NUM_PRIOS)
+
+static struct init_tree_node rdma_tx_root_fs = {
+ .type = FS_TYPE_NAMESPACE,
+ .ar_size = 3,
+ .children = (struct init_tree_node[]) {
+ [RDMA_TX_COUNTERS_PRIO] =
+ ADD_PRIO(0, RDMA_TX_COUNTERS_MIN_LEVEL, 0,
+ FS_CHAINING_CAPS,
+ ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
+ ADD_MULTIPLE_PRIO(MLX5_RDMA_TX_NUM_COUNTERS_PRIOS,
+ RDMA_TX_COUNTERS_PRIO_NUM_LEVELS))),
+ [RDMA_TX_IPSEC_PRIO] =
+ ADD_PRIO(0, RDMA_TX_IPSEC_MIN_LEVEL, 0,
+ FS_CHAINING_CAPS,
+ ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
+ ADD_MULTIPLE_PRIO(RDMA_TX_IPSEC_NUM_PRIOS,
+ RDMA_TX_IPSEC_PRIO_NUM_LEVELS))),
+
+ [RDMA_TX_BYPASS_PRIO] =
+ ADD_PRIO(0, RDMA_TX_BYPASS_MIN_LEVEL, 0,
+ FS_CHAINING_CAPS_RDMA_TX,
+ ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
+ ADD_MULTIPLE_PRIO(RDMA_TX_BYPASS_MIN_LEVEL,
+ BY_PASS_PRIO_NUM_LEVELS))),
+ }
+};
+
+enum fs_i_lock_class {
+ FS_LOCK_GRANDPARENT,
+ FS_LOCK_PARENT,
+ FS_LOCK_CHILD
+};
+
+static void del_hw_flow_table(struct fs_node *node);
+static void del_hw_flow_group(struct fs_node *node);
+static void del_hw_fte(struct fs_node *node);
+static void del_sw_flow_table(struct fs_node *node);
+static void del_sw_flow_group(struct fs_node *node);
+static void del_sw_fte(struct fs_node *node);
+static void del_sw_prio(struct fs_node *node);
+static void del_sw_ns(struct fs_node *node);
+/* Delete rule (destination) is special case that
+ * requires to lock the FTE for all the deletion process.
+ */
+static void del_sw_hw_rule(struct fs_node *node);
+static bool mlx5_flow_dests_cmp(struct mlx5_flow_destination *d1,
+ struct mlx5_flow_destination *d2);
+static void cleanup_root_ns(struct mlx5_flow_root_namespace *root_ns);
+static struct mlx5_flow_rule *
+find_flow_rule(struct fs_fte *fte,
+ struct mlx5_flow_destination *dest);
+
+static void tree_init_node(struct fs_node *node,
+ void (*del_hw_func)(struct fs_node *),
+ void (*del_sw_func)(struct fs_node *))
+{
+ refcount_set(&node->refcount, 1);
+ INIT_LIST_HEAD(&node->list);
+ INIT_LIST_HEAD(&node->children);
+ init_rwsem(&node->lock);
+ node->del_hw_func = del_hw_func;
+ node->del_sw_func = del_sw_func;
+ node->active = false;
+}
+
+static void tree_add_node(struct fs_node *node, struct fs_node *parent)
+{
+ if (parent)
+ refcount_inc(&parent->refcount);
+ node->parent = parent;
+
+ /* Parent is the root */
+ if (!parent)
+ node->root = node;
+ else
+ node->root = parent->root;
+}
+
+static int tree_get_node(struct fs_node *node)
+{
+ return refcount_inc_not_zero(&node->refcount);
+}
+
+static void nested_down_read_ref_node(struct fs_node *node,
+ enum fs_i_lock_class class)
+{
+ if (node) {
+ down_read_nested(&node->lock, class);
+ refcount_inc(&node->refcount);
+ }
+}
+
+static void nested_down_write_ref_node(struct fs_node *node,
+ enum fs_i_lock_class class)
+{
+ if (node) {
+ down_write_nested(&node->lock, class);
+ refcount_inc(&node->refcount);
+ }
+}
+
+static void down_write_ref_node(struct fs_node *node, bool locked)
+{
+ if (node) {
+ if (!locked)
+ down_write(&node->lock);
+ refcount_inc(&node->refcount);
+ }
+}
+
+static void up_read_ref_node(struct fs_node *node)
+{
+ refcount_dec(&node->refcount);
+ up_read(&node->lock);
+}
+
+static void up_write_ref_node(struct fs_node *node, bool locked)
+{
+ refcount_dec(&node->refcount);
+ if (!locked)
+ up_write(&node->lock);
+}
+
+static void tree_put_node(struct fs_node *node, bool locked)
+{
+ struct fs_node *parent_node = node->parent;
+
+ if (refcount_dec_and_test(&node->refcount.value)) {
+ if (node->del_hw_func)
+ node->del_hw_func(node);
+ if (parent_node) {
+ down_write_ref_node(parent_node, locked);
+ list_del_init(&node->list);
+ }
+ node->del_sw_func(node);
+ if (parent_node)
+ up_write_ref_node(parent_node, locked);
+ node = NULL;
+ }
+ if (!node && parent_node)
+ tree_put_node(parent_node, locked);
+}
+
+static int tree_remove_node(struct fs_node *node, bool locked)
+{
+ if (refcount_read(&node->refcount) > 1) {
+ refcount_dec(&node->refcount);
+ return -EEXIST;
+ }
+ tree_put_node(node, locked);
+ return 0;
+}
+
+static struct fs_prio *find_prio(struct mlx5_flow_namespace *ns,
+ unsigned int prio)
+{
+ struct fs_prio *iter_prio;
+
+ fs_for_each_prio(iter_prio, ns) {
+ if (iter_prio->prio == prio)
+ return iter_prio;
+ }
+
+ return NULL;
+}
+
+static bool is_fwd_next_action(u32 action)
+{
+ return action & (MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO |
+ MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS);
+}
+
+static bool is_fwd_dest_type(enum mlx5_flow_destination_type type)
+{
+ return type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM ||
+ type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE ||
+ type == MLX5_FLOW_DESTINATION_TYPE_UPLINK ||
+ type == MLX5_FLOW_DESTINATION_TYPE_VPORT ||
+ type == MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER ||
+ type == MLX5_FLOW_DESTINATION_TYPE_TIR ||
+ type == MLX5_FLOW_DESTINATION_TYPE_RANGE ||
+ type == MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE;
+}
+
+static bool check_valid_spec(const struct mlx5_flow_spec *spec)
+{
+ int i;
+
+ for (i = 0; i < MLX5_ST_SZ_DW_MATCH_PARAM; i++)
+ if (spec->match_value[i] & ~spec->match_criteria[i]) {
+ pr_warn("mlx5_core: match_value differs from match_criteria\n");
+ return false;
+ }
+
+ return true;
+}
+
+struct mlx5_flow_root_namespace *find_root(struct fs_node *node)
+{
+ struct fs_node *root;
+ struct mlx5_flow_namespace *ns;
+
+ root = node->root;
+
+ if (WARN_ON(root->type != FS_TYPE_NAMESPACE)) {
+ pr_warn("mlx5: flow steering node is not in tree or garbaged\n");
+ return NULL;
+ }
+
+ ns = container_of(root, struct mlx5_flow_namespace, node);
+ return container_of(ns, struct mlx5_flow_root_namespace, ns);
+}
+
+static inline struct mlx5_flow_steering *get_steering(struct fs_node *node)
+{
+ struct mlx5_flow_root_namespace *root = find_root(node);
+
+ if (root)
+ return root->dev->priv.steering;
+ return NULL;
+}
+
+static inline struct mlx5_core_dev *get_dev(struct fs_node *node)
+{
+ struct mlx5_flow_root_namespace *root = find_root(node);
+
+ if (root)
+ return root->dev;
+ return NULL;
+}
+
+static void del_sw_ns(struct fs_node *node)
+{
+ kfree(node);
+}
+
+static void del_sw_prio(struct fs_node *node)
+{
+ kfree(node);
+}
+
+static void del_hw_flow_table(struct fs_node *node)
+{
+ struct mlx5_flow_root_namespace *root;
+ struct mlx5_flow_table *ft;
+ struct mlx5_core_dev *dev;
+ int err;
+
+ fs_get_obj(ft, node);
+ dev = get_dev(&ft->node);
+ root = find_root(&ft->node);
+
+ if (node->active) {
+ err = root->cmds->destroy_flow_table(root, ft);
+ if (err)
+ mlx5_core_warn(dev, "flow steering can't destroy ft\n");
+ }
+}
+
+static void del_sw_flow_table(struct fs_node *node)
+{
+ struct mlx5_flow_table *ft;
+ struct fs_prio *prio;
+
+ fs_get_obj(ft, node);
+
+ xa_destroy(&ft->fgs_xa);
+ if (ft->node.parent) {
+ fs_get_obj(prio, ft->node.parent);
+ prio->num_ft--;
+ }
+ kfree(ft);
+}
+
+static void modify_fte(struct fs_fte *fte)
+{
+ struct mlx5_flow_root_namespace *root;
+ struct mlx5_flow_table *ft;
+ struct mlx5_flow_group *fg;
+ struct mlx5_core_dev *dev;
+ int err;
+
+ fs_get_obj(fg, fte->node.parent);
+ fs_get_obj(ft, fg->node.parent);
+ dev = get_dev(&fte->node);
+
+ root = find_root(&ft->node);
+ err = root->cmds->update_fte(root, ft, fg, fte->modify_mask, fte);
+ if (err)
+ mlx5_core_warn(dev,
+ "%s can't del rule fg id=%d fte_index=%d\n",
+ __func__, fg->id, fte->index);
+ fte->modify_mask = 0;
+}
+
+static void del_sw_hw_rule(struct fs_node *node)
+{
+ struct mlx5_flow_rule *rule;
+ struct fs_fte *fte;
+
+ fs_get_obj(rule, node);
+ fs_get_obj(fte, rule->node.parent);
+ if (is_fwd_next_action(rule->sw_action)) {
+ mutex_lock(&rule->dest_attr.ft->lock);
+ list_del(&rule->next_ft);
+ mutex_unlock(&rule->dest_attr.ft->lock);
+ }
+
+ if (rule->dest_attr.type == MLX5_FLOW_DESTINATION_TYPE_COUNTER) {
+ --fte->dests_size;
+ fte->modify_mask |=
+ BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_ACTION) |
+ BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_FLOW_COUNTERS);
+ fte->action.action &= ~MLX5_FLOW_CONTEXT_ACTION_COUNT;
+ goto out;
+ }
+
+ if (rule->dest_attr.type == MLX5_FLOW_DESTINATION_TYPE_PORT) {
+ --fte->dests_size;
+ fte->modify_mask |= BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_ACTION);
+ fte->action.action &= ~MLX5_FLOW_CONTEXT_ACTION_ALLOW;
+ goto out;
+ }
+
+ if (is_fwd_dest_type(rule->dest_attr.type)) {
+ --fte->dests_size;
+ --fte->fwd_dests;
+
+ if (!fte->fwd_dests)
+ fte->action.action &=
+ ~MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
+ fte->modify_mask |=
+ BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_DESTINATION_LIST);
+ goto out;
+ }
+out:
+ kfree(rule);
+}
+
+static void del_hw_fte(struct fs_node *node)
+{
+ struct mlx5_flow_root_namespace *root;
+ struct mlx5_flow_table *ft;
+ struct mlx5_flow_group *fg;
+ struct mlx5_core_dev *dev;
+ struct fs_fte *fte;
+ int err;
+
+ fs_get_obj(fte, node);
+ fs_get_obj(fg, fte->node.parent);
+ fs_get_obj(ft, fg->node.parent);
+
+ WARN_ON(fte->dests_size);
+ dev = get_dev(&ft->node);
+ root = find_root(&ft->node);
+ if (node->active) {
+ err = root->cmds->delete_fte(root, ft, fte);
+ if (err)
+ mlx5_core_warn(dev,
+ "flow steering can't delete fte in index %d of flow group id %d\n",
+ fte->index, fg->id);
+ node->active = false;
+ }
+}
+
+static void del_sw_fte(struct fs_node *node)
+{
+ struct mlx5_flow_steering *steering = get_steering(node);
+ struct mlx5_flow_group *fg;
+ struct fs_fte *deleted_fte;
+ struct fs_fte *fte;
+
+ fs_get_obj(fte, node);
+ fs_get_obj(fg, fte->node.parent);
+
+ deleted_fte = xa_erase(&fg->ftes_xa, fte->index);
+ WARN_ON(deleted_fte != fte);
+ ida_free(&fg->fte_allocator, fte->index - fg->start_index);
+ kmem_cache_free(steering->ftes_cache, fte);
+}
+
+static void del_hw_flow_group(struct fs_node *node)
+{
+ struct mlx5_flow_root_namespace *root;
+ struct mlx5_flow_group *fg;
+ struct mlx5_flow_table *ft;
+ struct mlx5_core_dev *dev;
+
+ fs_get_obj(fg, node);
+ fs_get_obj(ft, fg->node.parent);
+ dev = get_dev(&ft->node);
+
+ root = find_root(&ft->node);
+ if (fg->node.active && root->cmds->destroy_flow_group(root, ft, fg))
+ mlx5_core_warn(dev, "flow steering can't destroy fg %d of ft %d\n",
+ fg->id, ft->id);
+}
+
+static void del_sw_flow_group(struct fs_node *node)
+{
+ struct mlx5_flow_steering *steering = get_steering(node);
+ struct mlx5_flow_group *deleted_fg;
+ struct mlx5_flow_group *fg;
+ struct mlx5_flow_table *ft;
+
+ fs_get_obj(fg, node);
+ fs_get_obj(ft, fg->node.parent);
+
+ xa_destroy(&fg->ftes_xa);
+ ida_destroy(&fg->fte_allocator);
+ if (ft->autogroup.active &&
+ fg->max_ftes == ft->autogroup.group_size &&
+ fg->start_index < ft->autogroup.max_fte)
+ ft->autogroup.num_groups--;
+ deleted_fg = xa_erase(&ft->fgs_xa, fg->start_index);
+ WARN_ON(deleted_fg != fg);
+ kmem_cache_free(steering->fgs_cache, fg);
+}
+
+static int insert_fte(struct mlx5_flow_group *fg, struct fs_fte *fte)
+{
+ int index;
+ int ret;
+
+ index = ida_alloc_max(&fg->fte_allocator, fg->max_ftes, GFP_KERNEL);
+ if (index < 0)
+ return index;
+
+ fte->index = index + fg->start_index;
+ ret = xa_insert(&fg->ftes_xa, fte->index, fte, GFP_KERNEL);
+ if (ret)
+ goto err_ida_remove;
+
+ tree_add_node(&fte->node, &fg->node);
+ list_add_tail(&fte->node.list, &fg->node.children);
+ return 0;
+
+err_ida_remove:
+ ida_free(&fg->fte_allocator, index);
+ return ret;
+}
+
+static struct fs_fte *alloc_fte(struct mlx5_flow_table *ft,
+ const struct mlx5_flow_spec *spec,
+ struct mlx5_flow_act *flow_act)
+{
+ struct mlx5_flow_steering *steering = get_steering(&ft->node);
+ struct fs_fte *fte;
+
+ fte = kmem_cache_zalloc(steering->ftes_cache, GFP_KERNEL);
+ if (!fte)
+ return ERR_PTR(-ENOMEM);
+
+ memcpy(fte->val, &spec->match_value, sizeof(fte->val));
+ fte->node.type = FS_TYPE_FLOW_ENTRY;
+ fte->action = *flow_act;
+ fte->flow_context = spec->flow_context;
+
+ tree_init_node(&fte->node, del_hw_fte, del_sw_fte);
+
+ return fte;
+}
+
+static void dealloc_flow_group(struct mlx5_flow_steering *steering,
+ struct mlx5_flow_group *fg)
+{
+ xa_destroy(&fg->ftes_xa);
+ kmem_cache_free(steering->fgs_cache, fg);
+}
+
+static struct mlx5_flow_group *alloc_flow_group(struct mlx5_flow_steering *steering,
+ u8 match_criteria_enable,
+ const void *match_criteria,
+ int start_index,
+ int end_index)
+{
+ struct mlx5_flow_group *fg;
+
+ fg = kmem_cache_zalloc(steering->fgs_cache, GFP_KERNEL);
+ if (!fg)
+ return ERR_PTR(-ENOMEM);
+
+ xa_init(&fg->ftes_xa);
+
+ ida_init(&fg->fte_allocator);
+ fg->mask.match_criteria_enable = match_criteria_enable;
+ memcpy(&fg->mask.match_criteria, match_criteria,
+ sizeof(fg->mask.match_criteria));
+ fg->node.type = FS_TYPE_FLOW_GROUP;
+ fg->start_index = start_index;
+ fg->max_ftes = end_index - start_index + 1;
+
+ return fg;
+}
+
+static struct mlx5_flow_group *alloc_insert_flow_group(struct mlx5_flow_table *ft,
+ u8 match_criteria_enable,
+ const void *match_criteria,
+ int start_index,
+ int end_index,
+ struct list_head *prev)
+{
+ struct mlx5_flow_steering *steering = get_steering(&ft->node);
+ struct mlx5_flow_group *fg;
+ int ret;
+
+ fg = alloc_flow_group(steering, match_criteria_enable, match_criteria,
+ start_index, end_index);
+ if (IS_ERR(fg))
+ return fg;
+
+ /* initialize refcnt, add to parent list */
+ ret = xa_insert(&ft->fgs_xa, fg->start_index, fg, GFP_KERNEL);
+ if (ret) {
+ dealloc_flow_group(steering, fg);
+ return ERR_PTR(ret);
+ }
+
+ tree_init_node(&fg->node, del_hw_flow_group, del_sw_flow_group);
+ tree_add_node(&fg->node, &ft->node);
+ /* Add node to group list */
+ list_add(&fg->node.list, prev);
+ atomic_inc(&ft->node.version);
+
+ return fg;
+}
+
+static struct mlx5_flow_table *alloc_flow_table(int level, u16 vport,
+ enum fs_flow_table_type table_type,
+ enum fs_flow_table_op_mod op_mod,
+ u32 flags)
+{
+ struct mlx5_flow_table *ft;
+
+ ft = kzalloc(sizeof(*ft), GFP_KERNEL);
+ if (!ft)
+ return ERR_PTR(-ENOMEM);
+
+ xa_init(&ft->fgs_xa);
+
+ ft->level = level;
+ ft->node.type = FS_TYPE_FLOW_TABLE;
+ ft->op_mod = op_mod;
+ ft->type = table_type;
+ ft->vport = vport;
+ ft->flags = flags;
+ INIT_LIST_HEAD(&ft->fwd_rules);
+ mutex_init(&ft->lock);
+
+ return ft;
+}
+
+/* If reverse is false, then we search for the first flow table in the
+ * root sub-tree from start(closest from right), else we search for the
+ * last flow table in the root sub-tree till start(closest from left).
+ */
+static struct mlx5_flow_table *find_closest_ft_recursive(struct fs_node *root,
+ struct list_head *start,
+ bool reverse)
+{
+#define list_advance_entry(pos, reverse) \
+ ((reverse) ? list_prev_entry(pos, list) : list_next_entry(pos, list))
+
+#define list_for_each_advance_continue(pos, head, reverse) \
+ for (pos = list_advance_entry(pos, reverse); \
+ &pos->list != (head); \
+ pos = list_advance_entry(pos, reverse))
+
+ struct fs_node *iter = list_entry(start, struct fs_node, list);
+ struct mlx5_flow_table *ft = NULL;
+
+ if (!root || root->type == FS_TYPE_PRIO_CHAINS)
+ return NULL;
+
+ list_for_each_advance_continue(iter, &root->children, reverse) {
+ if (iter->type == FS_TYPE_FLOW_TABLE) {
+ fs_get_obj(ft, iter);
+ return ft;
+ }
+ ft = find_closest_ft_recursive(iter, &iter->children, reverse);
+ if (ft)
+ return ft;
+ }
+
+ return ft;
+}
+
+/* If reverse is false then return the first flow table in next priority of
+ * prio in the tree, else return the last flow table in the previous priority
+ * of prio in the tree.
+ */
+static struct mlx5_flow_table *find_closest_ft(struct fs_prio *prio, bool reverse)
+{
+ struct mlx5_flow_table *ft = NULL;
+ struct fs_node *curr_node;
+ struct fs_node *parent;
+
+ parent = prio->node.parent;
+ curr_node = &prio->node;
+ while (!ft && parent) {
+ ft = find_closest_ft_recursive(parent, &curr_node->list, reverse);
+ curr_node = parent;
+ parent = curr_node->parent;
+ }
+ return ft;
+}
+
+/* Assuming all the tree is locked by mutex chain lock */
+static struct mlx5_flow_table *find_next_chained_ft(struct fs_prio *prio)
+{
+ return find_closest_ft(prio, false);
+}
+
+/* Assuming all the tree is locked by mutex chain lock */
+static struct mlx5_flow_table *find_prev_chained_ft(struct fs_prio *prio)
+{
+ return find_closest_ft(prio, true);
+}
+
+static struct mlx5_flow_table *find_next_fwd_ft(struct mlx5_flow_table *ft,
+ struct mlx5_flow_act *flow_act)
+{
+ struct fs_prio *prio;
+ bool next_ns;
+
+ next_ns = flow_act->action & MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS;
+ fs_get_obj(prio, next_ns ? ft->ns->node.parent : ft->node.parent);
+
+ return find_next_chained_ft(prio);
+}
+
+static int connect_fts_in_prio(struct mlx5_core_dev *dev,
+ struct fs_prio *prio,
+ struct mlx5_flow_table *ft)
+{
+ struct mlx5_flow_root_namespace *root = find_root(&prio->node);
+ struct mlx5_flow_table *iter;
+ int err;
+
+ fs_for_each_ft(iter, prio) {
+ err = root->cmds->modify_flow_table(root, iter, ft);
+ if (err) {
+ mlx5_core_err(dev,
+ "Failed to modify flow table id %d, type %d, err %d\n",
+ iter->id, iter->type, err);
+ /* The driver is out of sync with the FW */
+ return err;
+ }
+ }
+ return 0;
+}
+
+/* Connect flow tables from previous priority of prio to ft */
+static int connect_prev_fts(struct mlx5_core_dev *dev,
+ struct mlx5_flow_table *ft,
+ struct fs_prio *prio)
+{
+ struct mlx5_flow_table *prev_ft;
+
+ prev_ft = find_prev_chained_ft(prio);
+ if (prev_ft) {
+ struct fs_prio *prev_prio;
+
+ fs_get_obj(prev_prio, prev_ft->node.parent);
+ return connect_fts_in_prio(dev, prev_prio, ft);
+ }
+ return 0;
+}
+
+static int update_root_ft_create(struct mlx5_flow_table *ft, struct fs_prio
+ *prio)
+{
+ struct mlx5_flow_root_namespace *root = find_root(&prio->node);
+ struct mlx5_ft_underlay_qp *uqp;
+ int min_level = INT_MAX;
+ int err = 0;
+ u32 qpn;
+
+ if (root->root_ft)
+ min_level = root->root_ft->level;
+
+ if (ft->level >= min_level)
+ return 0;
+
+ if (list_empty(&root->underlay_qpns)) {
+ /* Don't set any QPN (zero) in case QPN list is empty */
+ qpn = 0;
+ err = root->cmds->update_root_ft(root, ft, qpn, false);
+ } else {
+ list_for_each_entry(uqp, &root->underlay_qpns, list) {
+ qpn = uqp->qpn;
+ err = root->cmds->update_root_ft(root, ft,
+ qpn, false);
+ if (err)
+ break;
+ }
+ }
+
+ if (err)
+ mlx5_core_warn(root->dev,
+ "Update root flow table of id(%u) qpn(%d) failed\n",
+ ft->id, qpn);
+ else
+ root->root_ft = ft;
+
+ return err;
+}
+
+static int _mlx5_modify_rule_destination(struct mlx5_flow_rule *rule,
+ struct mlx5_flow_destination *dest)
+{
+ struct mlx5_flow_root_namespace *root;
+ struct mlx5_flow_table *ft;
+ struct mlx5_flow_group *fg;
+ struct fs_fte *fte;
+ int modify_mask = BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_DESTINATION_LIST);
+ int err = 0;
+
+ fs_get_obj(fte, rule->node.parent);
+ if (!(fte->action.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST))
+ return -EINVAL;
+ down_write_ref_node(&fte->node, false);
+ fs_get_obj(fg, fte->node.parent);
+ fs_get_obj(ft, fg->node.parent);
+
+ memcpy(&rule->dest_attr, dest, sizeof(*dest));
+ root = find_root(&ft->node);
+ err = root->cmds->update_fte(root, ft, fg,
+ modify_mask, fte);
+ up_write_ref_node(&fte->node, false);
+
+ return err;
+}
+
+int mlx5_modify_rule_destination(struct mlx5_flow_handle *handle,
+ struct mlx5_flow_destination *new_dest,
+ struct mlx5_flow_destination *old_dest)
+{
+ int i;
+
+ if (!old_dest) {
+ if (handle->num_rules != 1)
+ return -EINVAL;
+ return _mlx5_modify_rule_destination(handle->rule[0],
+ new_dest);
+ }
+
+ for (i = 0; i < handle->num_rules; i++) {
+ if (mlx5_flow_dests_cmp(new_dest, &handle->rule[i]->dest_attr))
+ return _mlx5_modify_rule_destination(handle->rule[i],
+ new_dest);
+ }
+
+ return -EINVAL;
+}
+
+/* Modify/set FWD rules that point on old_next_ft to point on new_next_ft */
+static int connect_fwd_rules(struct mlx5_core_dev *dev,
+ struct mlx5_flow_table *new_next_ft,
+ struct mlx5_flow_table *old_next_ft)
+{
+ struct mlx5_flow_destination dest = {};
+ struct mlx5_flow_rule *iter;
+ int err = 0;
+
+ /* new_next_ft and old_next_ft could be NULL only
+ * when we create/destroy the anchor flow table.
+ */
+ if (!new_next_ft || !old_next_ft)
+ return 0;
+
+ dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
+ dest.ft = new_next_ft;
+
+ mutex_lock(&old_next_ft->lock);
+ list_splice_init(&old_next_ft->fwd_rules, &new_next_ft->fwd_rules);
+ mutex_unlock(&old_next_ft->lock);
+ list_for_each_entry(iter, &new_next_ft->fwd_rules, next_ft) {
+ if ((iter->sw_action & MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS) &&
+ iter->ft->ns == new_next_ft->ns)
+ continue;
+
+ err = _mlx5_modify_rule_destination(iter, &dest);
+ if (err)
+ pr_err("mlx5_core: failed to modify rule to point on flow table %d\n",
+ new_next_ft->id);
+ }
+ return 0;
+}
+
+static int connect_flow_table(struct mlx5_core_dev *dev, struct mlx5_flow_table *ft,
+ struct fs_prio *prio)
+{
+ struct mlx5_flow_table *next_ft, *first_ft;
+ int err = 0;
+
+ /* Connect_prev_fts and update_root_ft_create are mutually exclusive */
+
+ first_ft = list_first_entry_or_null(&prio->node.children,
+ struct mlx5_flow_table, node.list);
+ if (!first_ft || first_ft->level > ft->level) {
+ err = connect_prev_fts(dev, ft, prio);
+ if (err)
+ return err;
+
+ next_ft = first_ft ? first_ft : find_next_chained_ft(prio);
+ err = connect_fwd_rules(dev, ft, next_ft);
+ if (err)
+ return err;
+ }
+
+ if (MLX5_CAP_FLOWTABLE(dev,
+ flow_table_properties_nic_receive.modify_root))
+ err = update_root_ft_create(ft, prio);
+ return err;
+}
+
+static void list_add_flow_table(struct mlx5_flow_table *ft,
+ struct fs_prio *prio)
+{
+ struct list_head *prev = &prio->node.children;
+ struct mlx5_flow_table *iter;
+
+ fs_for_each_ft(iter, prio) {
+ if (iter->level > ft->level)
+ break;
+ prev = &iter->node.list;
+ }
+ list_add(&ft->node.list, prev);
+}
+
+static struct mlx5_flow_table *__mlx5_create_flow_table(struct mlx5_flow_namespace *ns,
+ struct mlx5_flow_table_attr *ft_attr,
+ enum fs_flow_table_op_mod op_mod,
+ u16 vport)
+{
+ struct mlx5_flow_root_namespace *root = find_root(&ns->node);
+ bool unmanaged = ft_attr->flags & MLX5_FLOW_TABLE_UNMANAGED;
+ struct mlx5_flow_table *next_ft;
+ struct fs_prio *fs_prio = NULL;
+ struct mlx5_flow_table *ft;
+ int err;
+
+ if (!root) {
+ pr_err("mlx5: flow steering failed to find root of namespace\n");
+ return ERR_PTR(-ENODEV);
+ }
+
+ mutex_lock(&root->chain_lock);
+ fs_prio = find_prio(ns, ft_attr->prio);
+ if (!fs_prio) {
+ err = -EINVAL;
+ goto unlock_root;
+ }
+ if (!unmanaged) {
+ /* The level is related to the
+ * priority level range.
+ */
+ if (ft_attr->level >= fs_prio->num_levels) {
+ err = -ENOSPC;
+ goto unlock_root;
+ }
+
+ ft_attr->level += fs_prio->start_level;
+ }
+
+ /* The level is related to the
+ * priority level range.
+ */
+ ft = alloc_flow_table(ft_attr->level,
+ vport,
+ root->table_type,
+ op_mod, ft_attr->flags);
+ if (IS_ERR(ft)) {
+ err = PTR_ERR(ft);
+ goto unlock_root;
+ }
+
+ tree_init_node(&ft->node, del_hw_flow_table, del_sw_flow_table);
+ next_ft = unmanaged ? ft_attr->next_ft :
+ find_next_chained_ft(fs_prio);
+ ft->def_miss_action = ns->def_miss_action;
+ ft->ns = ns;
+ err = root->cmds->create_flow_table(root, ft, ft_attr, next_ft);
+ if (err)
+ goto free_ft;
+
+ if (!unmanaged) {
+ err = connect_flow_table(root->dev, ft, fs_prio);
+ if (err)
+ goto destroy_ft;
+ }
+
+ ft->node.active = true;
+ down_write_ref_node(&fs_prio->node, false);
+ if (!unmanaged) {
+ tree_add_node(&ft->node, &fs_prio->node);
+ list_add_flow_table(ft, fs_prio);
+ } else {
+ ft->node.root = fs_prio->node.root;
+ }
+ fs_prio->num_ft++;
+ up_write_ref_node(&fs_prio->node, false);
+ mutex_unlock(&root->chain_lock);
+ return ft;
+destroy_ft:
+ root->cmds->destroy_flow_table(root, ft);
+free_ft:
+ xa_destroy(&ft->fgs_xa);
+ kfree(ft);
+unlock_root:
+ mutex_unlock(&root->chain_lock);
+ return ERR_PTR(err);
+}
+
+struct mlx5_flow_table *mlx5_create_flow_table(struct mlx5_flow_namespace *ns,
+ struct mlx5_flow_table_attr *ft_attr)
+{
+ return __mlx5_create_flow_table(ns, ft_attr, FS_FT_OP_MOD_NORMAL, 0);
+}
+EXPORT_SYMBOL(mlx5_create_flow_table);
+
+u32 mlx5_flow_table_id(struct mlx5_flow_table *ft)
+{
+ return ft->id;
+}
+EXPORT_SYMBOL(mlx5_flow_table_id);
+
+struct mlx5_flow_table *
+mlx5_create_vport_flow_table(struct mlx5_flow_namespace *ns,
+ struct mlx5_flow_table_attr *ft_attr, u16 vport)
+{
+ return __mlx5_create_flow_table(ns, ft_attr, FS_FT_OP_MOD_NORMAL, vport);
+}
+
+struct mlx5_flow_table*
+mlx5_create_lag_demux_flow_table(struct mlx5_flow_namespace *ns,
+ int prio, u32 level)
+{
+ struct mlx5_flow_table_attr ft_attr = {};
+
+ ft_attr.level = level;
+ ft_attr.prio = prio;
+ ft_attr.max_fte = 1;
+
+ return __mlx5_create_flow_table(ns, &ft_attr, FS_FT_OP_MOD_LAG_DEMUX, 0);
+}
+EXPORT_SYMBOL(mlx5_create_lag_demux_flow_table);
+
+#define MAX_FLOW_GROUP_SIZE BIT(24)
+struct mlx5_flow_table*
+mlx5_create_auto_grouped_flow_table(struct mlx5_flow_namespace *ns,
+ struct mlx5_flow_table_attr *ft_attr)
+{
+ int num_reserved_entries = ft_attr->autogroup.num_reserved_entries;
+ int max_num_groups = ft_attr->autogroup.max_num_groups;
+ struct mlx5_flow_table *ft;
+ int autogroups_max_fte;
+
+ ft = mlx5_create_flow_table(ns, ft_attr);
+ if (IS_ERR(ft))
+ return ft;
+
+ autogroups_max_fte = ft->max_fte - num_reserved_entries;
+ if (max_num_groups > autogroups_max_fte)
+ goto err_validate;
+ if (num_reserved_entries > ft->max_fte)
+ goto err_validate;
+
+ /* Align the number of groups according to the largest group size */
+ if (autogroups_max_fte / (max_num_groups + 1) > MAX_FLOW_GROUP_SIZE)
+ max_num_groups = (autogroups_max_fte / MAX_FLOW_GROUP_SIZE) - 1;
+
+ ft->autogroup.active = true;
+ ft->autogroup.required_groups = max_num_groups;
+ ft->autogroup.max_fte = autogroups_max_fte;
+ /* We save place for flow groups in addition to max types */
+ ft->autogroup.group_size = autogroups_max_fte / (max_num_groups + 1);
+
+ return ft;
+
+err_validate:
+ mlx5_destroy_flow_table(ft);
+ return ERR_PTR(-ENOSPC);
+}
+EXPORT_SYMBOL(mlx5_create_auto_grouped_flow_table);
+
+struct mlx5_flow_group *mlx5_create_flow_group(struct mlx5_flow_table *ft,
+ u32 *fg_in)
+{
+ struct mlx5_flow_root_namespace *root = find_root(&ft->node);
+ void *match_criteria = MLX5_ADDR_OF(create_flow_group_in,
+ fg_in, match_criteria);
+ u8 match_criteria_enable = MLX5_GET(create_flow_group_in,
+ fg_in,
+ match_criteria_enable);
+ int start_index = MLX5_GET(create_flow_group_in, fg_in,
+ start_flow_index);
+ int end_index = MLX5_GET(create_flow_group_in, fg_in,
+ end_flow_index);
+ struct mlx5_flow_group *fg;
+ int err;
+
+ if (ft->autogroup.active && start_index < ft->autogroup.max_fte)
+ return ERR_PTR(-EPERM);
+
+ down_write_ref_node(&ft->node, false);
+ fg = alloc_insert_flow_group(ft, match_criteria_enable, match_criteria,
+ start_index, end_index,
+ ft->node.children.prev);
+ up_write_ref_node(&ft->node, false);
+ if (IS_ERR(fg))
+ return fg;
+
+ err = root->cmds->create_flow_group(root, ft, fg_in, fg);
+ if (err) {
+ tree_put_node(&fg->node, false);
+ return ERR_PTR(err);
+ }
+ fg->node.active = true;
+
+ return fg;
+}
+EXPORT_SYMBOL(mlx5_create_flow_group);
+
+static struct mlx5_flow_rule *alloc_rule(struct mlx5_flow_destination *dest)
+{
+ struct mlx5_flow_rule *rule;
+
+ rule = kzalloc(sizeof(*rule), GFP_KERNEL);
+ if (!rule)
+ return NULL;
+
+ INIT_LIST_HEAD(&rule->next_ft);
+ rule->node.type = FS_TYPE_FLOW_DEST;
+ if (dest)
+ memcpy(&rule->dest_attr, dest, sizeof(*dest));
+ else
+ rule->dest_attr.type = MLX5_FLOW_DESTINATION_TYPE_NONE;
+
+ return rule;
+}
+
+static struct mlx5_flow_handle *alloc_handle(int num_rules)
+{
+ struct mlx5_flow_handle *handle;
+
+ handle = kzalloc(struct_size(handle, rule, num_rules), GFP_KERNEL);
+ if (!handle)
+ return NULL;
+
+ handle->num_rules = num_rules;
+
+ return handle;
+}
+
+static void destroy_flow_handle(struct fs_fte *fte,
+ struct mlx5_flow_handle *handle,
+ struct mlx5_flow_destination *dest,
+ int i)
+{
+ for (; --i >= 0;) {
+ if (refcount_dec_and_test(&handle->rule[i]->node.refcount.value)) {
+ fte->dests_size--;
+ list_del(&handle->rule[i]->node.list);
+ kfree(handle->rule[i]);
+ }
+ }
+ kfree(handle);
+}
+
+static struct mlx5_flow_handle *
+create_flow_handle(struct fs_fte *fte,
+ struct mlx5_flow_destination *dest,
+ int dest_num,
+ int *modify_mask,
+ bool *new_rule)
+{
+ struct mlx5_flow_handle *handle;
+ struct mlx5_flow_rule *rule = NULL;
+ static int count = BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_FLOW_COUNTERS);
+ static int dst = BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_DESTINATION_LIST);
+ int type;
+ int i = 0;
+
+ handle = alloc_handle((dest_num) ? dest_num : 1);
+ if (!handle)
+ return ERR_PTR(-ENOMEM);
+
+ do {
+ if (dest) {
+ rule = find_flow_rule(fte, dest + i);
+ if (rule) {
+ refcount_inc(&rule->node.refcount);
+ goto rule_found;
+ }
+ }
+
+ *new_rule = true;
+ rule = alloc_rule(dest + i);
+ if (!rule)
+ goto free_rules;
+
+ /* Add dest to dests list- we need flow tables to be in the
+ * end of the list for forward to next prio rules.
+ */
+ tree_init_node(&rule->node, NULL, del_sw_hw_rule);
+ if (dest &&
+ dest[i].type != MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE)
+ list_add(&rule->node.list, &fte->node.children);
+ else
+ list_add_tail(&rule->node.list, &fte->node.children);
+ if (dest) {
+ fte->dests_size++;
+
+ if (is_fwd_dest_type(dest[i].type))
+ fte->fwd_dests++;
+
+ type = dest[i].type ==
+ MLX5_FLOW_DESTINATION_TYPE_COUNTER;
+ *modify_mask |= type ? count : dst;
+ }
+rule_found:
+ handle->rule[i] = rule;
+ } while (++i < dest_num);
+
+ return handle;
+
+free_rules:
+ destroy_flow_handle(fte, handle, dest, i);
+ return ERR_PTR(-ENOMEM);
+}
+
+/* fte should not be deleted while calling this function */
+static struct mlx5_flow_handle *
+add_rule_fte(struct fs_fte *fte,
+ struct mlx5_flow_group *fg,
+ struct mlx5_flow_destination *dest,
+ int dest_num,
+ bool update_action)
+{
+ struct mlx5_flow_root_namespace *root;
+ struct mlx5_flow_handle *handle;
+ struct mlx5_flow_table *ft;
+ int modify_mask = 0;
+ int err;
+ bool new_rule = false;
+
+ handle = create_flow_handle(fte, dest, dest_num, &modify_mask,
+ &new_rule);
+ if (IS_ERR(handle) || !new_rule)
+ goto out;
+
+ if (update_action)
+ modify_mask |= BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_ACTION);
+
+ fs_get_obj(ft, fg->node.parent);
+ root = find_root(&fg->node);
+ if (!(fte->status & FS_FTE_STATUS_EXISTING))
+ err = root->cmds->create_fte(root, ft, fg, fte);
+ else
+ err = root->cmds->update_fte(root, ft, fg, modify_mask, fte);
+ if (err)
+ goto free_handle;
+
+ fte->node.active = true;
+ fte->status |= FS_FTE_STATUS_EXISTING;
+ atomic_inc(&fg->node.version);
+
+out:
+ return handle;
+
+free_handle:
+ destroy_flow_handle(fte, handle, dest, handle->num_rules);
+ return ERR_PTR(err);
+}
+
+static struct mlx5_flow_group *alloc_auto_flow_group(struct mlx5_flow_table *ft,
+ const struct mlx5_flow_spec *spec)
+{
+ struct list_head *prev = &ft->node.children;
+ u32 max_fte = ft->autogroup.max_fte;
+ unsigned int candidate_index = 0;
+ unsigned int group_size = 0;
+ struct mlx5_flow_group *fg;
+
+ if (!ft->autogroup.active)
+ return ERR_PTR(-ENOENT);
+
+ if (ft->autogroup.num_groups < ft->autogroup.required_groups)
+ group_size = ft->autogroup.group_size;
+
+ /* max_fte == ft->autogroup.max_types */
+ if (group_size == 0)
+ group_size = 1;
+
+ /* sorted by start_index */
+ fs_for_each_fg(fg, ft) {
+ if (candidate_index + group_size > fg->start_index)
+ candidate_index = fg->start_index + fg->max_ftes;
+ else
+ break;
+ prev = &fg->node.list;
+ }
+
+ if (candidate_index + group_size > max_fte)
+ return ERR_PTR(-ENOSPC);
+
+ fg = alloc_insert_flow_group(ft,
+ spec->match_criteria_enable,
+ spec->match_criteria,
+ candidate_index,
+ candidate_index + group_size - 1,
+ prev);
+ if (IS_ERR(fg))
+ goto out;
+
+ if (group_size == ft->autogroup.group_size)
+ ft->autogroup.num_groups++;
+
+out:
+ return fg;
+}
+
+static int create_auto_flow_group(struct mlx5_flow_table *ft,
+ struct mlx5_flow_group *fg)
+{
+ struct mlx5_flow_root_namespace *root = find_root(&ft->node);
+ int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
+ void *match_criteria_addr;
+ u8 src_esw_owner_mask_on;
+ void *misc;
+ int err;
+ u32 *in;
+
+ in = kvzalloc(inlen, GFP_KERNEL);
+ if (!in)
+ return -ENOMEM;
+
+ MLX5_SET(create_flow_group_in, in, match_criteria_enable,
+ fg->mask.match_criteria_enable);
+ MLX5_SET(create_flow_group_in, in, start_flow_index, fg->start_index);
+ MLX5_SET(create_flow_group_in, in, end_flow_index, fg->start_index +
+ fg->max_ftes - 1);
+
+ misc = MLX5_ADDR_OF(fte_match_param, fg->mask.match_criteria,
+ misc_parameters);
+ src_esw_owner_mask_on = !!MLX5_GET(fte_match_set_misc, misc,
+ source_eswitch_owner_vhca_id);
+ MLX5_SET(create_flow_group_in, in,
+ source_eswitch_owner_vhca_id_valid, src_esw_owner_mask_on);
+
+ match_criteria_addr = MLX5_ADDR_OF(create_flow_group_in,
+ in, match_criteria);
+ memcpy(match_criteria_addr, fg->mask.match_criteria,
+ sizeof(fg->mask.match_criteria));
+
+ err = root->cmds->create_flow_group(root, ft, in, fg);
+ if (!err) {
+ fg->node.active = true;
+ }
+
+ kvfree(in);
+ return err;
+}
+
+static bool mlx5_flow_dests_cmp(struct mlx5_flow_destination *d1,
+ struct mlx5_flow_destination *d2)
+{
+ if (d1->type == d2->type) {
+ if (((d1->type == MLX5_FLOW_DESTINATION_TYPE_VPORT ||
+ d1->type == MLX5_FLOW_DESTINATION_TYPE_UPLINK) &&
+ d1->vport.num == d2->vport.num &&
+ d1->vport.flags == d2->vport.flags &&
+ ((d1->vport.flags & MLX5_FLOW_DEST_VPORT_VHCA_ID) ?
+ (d1->vport.vhca_id == d2->vport.vhca_id) : true) &&
+ ((d1->vport.flags & MLX5_FLOW_DEST_VPORT_REFORMAT_ID) ?
+ (d1->vport.pkt_reformat->id ==
+ d2->vport.pkt_reformat->id) : true)) ||
+ (d1->type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE &&
+ d1->ft == d2->ft) ||
+ (d1->type == MLX5_FLOW_DESTINATION_TYPE_TIR &&
+ d1->tir_num == d2->tir_num) ||
+ (d1->type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM &&
+ d1->ft_num == d2->ft_num) ||
+ (d1->type == MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER &&
+ d1->sampler_id == d2->sampler_id) ||
+ (d1->type == MLX5_FLOW_DESTINATION_TYPE_RANGE &&
+ d1->range.field == d2->range.field &&
+ d1->range.hit_ft == d2->range.hit_ft &&
+ d1->range.miss_ft == d2->range.miss_ft &&
+ d1->range.min == d2->range.min &&
+ d1->range.max == d2->range.max))
+ return true;
+ }
+
+ return false;
+}
+
+static struct mlx5_flow_rule *find_flow_rule(struct fs_fte *fte,
+ struct mlx5_flow_destination *dest)
+{
+ struct mlx5_flow_rule *rule;
+
+ list_for_each_entry(rule, &fte->node.children, node.list) {
+ if (mlx5_flow_dests_cmp(&rule->dest_attr, dest))
+ return rule;
+ }
+ return NULL;
+}
+
+static bool check_conflicting_actions_vlan(const struct mlx5_fs_vlan *vlan0,
+ const struct mlx5_fs_vlan *vlan1)
+{
+ return vlan0->ethtype != vlan1->ethtype ||
+ vlan0->vid != vlan1->vid ||
+ vlan0->prio != vlan1->prio;
+}
+
+static bool check_conflicting_actions(const struct mlx5_flow_act *act1,
+ const struct mlx5_flow_act *act2)
+{
+ u32 action1 = act1->action;
+ u32 action2 = act2->action;
+ u32 xored_actions;
+
+ xored_actions = action1 ^ action2;
+
+ /* if one rule only wants to count, it's ok */
+ if (action1 == MLX5_FLOW_CONTEXT_ACTION_COUNT ||
+ action2 == MLX5_FLOW_CONTEXT_ACTION_COUNT)
+ return false;
+
+ if (xored_actions & (MLX5_FLOW_CONTEXT_ACTION_DROP |
+ MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT |
+ MLX5_FLOW_CONTEXT_ACTION_DECAP |
+ MLX5_FLOW_CONTEXT_ACTION_MOD_HDR |
+ MLX5_FLOW_CONTEXT_ACTION_VLAN_POP |
+ MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH |
+ MLX5_FLOW_CONTEXT_ACTION_VLAN_POP_2 |
+ MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2))
+ return true;
+
+ if (action1 & MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT &&
+ act1->pkt_reformat != act2->pkt_reformat)
+ return true;
+
+ if (action1 & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR &&
+ act1->modify_hdr != act2->modify_hdr)
+ return true;
+
+ if (action1 & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH &&
+ check_conflicting_actions_vlan(&act1->vlan[0], &act2->vlan[0]))
+ return true;
+
+ if (action1 & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2 &&
+ check_conflicting_actions_vlan(&act1->vlan[1], &act2->vlan[1]))
+ return true;
+
+ return false;
+}
+
+static int check_conflicting_ftes(struct fs_fte *fte,
+ const struct mlx5_flow_context *flow_context,
+ const struct mlx5_flow_act *flow_act)
+{
+ if (check_conflicting_actions(flow_act, &fte->action)) {
+ mlx5_core_warn(get_dev(&fte->node),
+ "Found two FTEs with conflicting actions\n");
+ return -EEXIST;
+ }
+
+ if ((flow_context->flags & FLOW_CONTEXT_HAS_TAG) &&
+ fte->flow_context.flow_tag != flow_context->flow_tag) {
+ mlx5_core_warn(get_dev(&fte->node),
+ "FTE flow tag %u already exists with different flow tag %u\n",
+ fte->flow_context.flow_tag,
+ flow_context->flow_tag);
+ return -EEXIST;
+ }
+
+ return 0;
+}
+
+static struct mlx5_flow_handle *add_rule_fg(struct mlx5_flow_group *fg,
+ const struct mlx5_flow_spec *spec,
+ struct mlx5_flow_act *flow_act,
+ struct mlx5_flow_destination *dest,
+ int dest_num,
+ struct fs_fte *fte)
+{
+ struct mlx5_flow_handle *handle;
+ int old_action;
+ int i;
+ int ret;
+
+ ret = check_conflicting_ftes(fte, &spec->flow_context, flow_act);
+ if (ret)
+ return ERR_PTR(ret);
+
+ old_action = fte->action.action;
+ fte->action.action |= flow_act->action;
+ handle = add_rule_fte(fte, fg, dest, dest_num,
+ old_action != flow_act->action);
+ if (IS_ERR(handle)) {
+ fte->action.action = old_action;
+ return handle;
+ }
+
+ for (i = 0; i < handle->num_rules; i++) {
+ if (refcount_read(&handle->rule[i]->node.refcount) == 1) {
+ tree_add_node(&handle->rule[i]->node, &fte->node);
+ }
+ }
+ return handle;
+}
+
+static bool counter_is_valid(u32 action)
+{
+ return (action & (MLX5_FLOW_CONTEXT_ACTION_DROP |
+ MLX5_FLOW_CONTEXT_ACTION_ALLOW |
+ MLX5_FLOW_CONTEXT_ACTION_FWD_DEST));
+}
+
+static bool dest_is_valid(struct mlx5_flow_destination *dest,
+ struct mlx5_flow_act *flow_act,
+ struct mlx5_flow_table *ft)
+{
+ bool ignore_level = flow_act->flags & FLOW_ACT_IGNORE_FLOW_LEVEL;
+ u32 action = flow_act->action;
+
+ if (dest && (dest->type == MLX5_FLOW_DESTINATION_TYPE_COUNTER))
+ return counter_is_valid(action);
+
+ if (!(action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST))
+ return true;
+
+ if (ignore_level) {
+ if (ft->type != FS_FT_FDB &&
+ ft->type != FS_FT_NIC_RX &&
+ ft->type != FS_FT_NIC_TX)
+ return false;
+
+ if (dest->type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE &&
+ ft->type != dest->ft->type)
+ return false;
+ }
+
+ if (!dest || ((dest->type ==
+ MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE) &&
+ (dest->ft->level <= ft->level && !ignore_level)))
+ return false;
+ return true;
+}
+
+struct match_list {
+ struct list_head list;
+ struct mlx5_flow_group *g;
+};
+
+static void free_match_list(struct match_list *head, bool ft_locked)
+{
+ struct match_list *iter, *match_tmp;
+
+ list_for_each_entry_safe(iter, match_tmp, &head->list,
+ list) {
+ tree_put_node(&iter->g->node, ft_locked);
+ list_del(&iter->list);
+ kfree(iter);
+ }
+}
+
+#define xa_for_each_rcu(xa, index, entry) \
+ for ((entry) = NULL, (index) = 0; \
+ ((entry) = xa_next(xa, &index, (entry) != NULL)) != NULL; )
+
+static int build_match_list(struct match_list *match_head,
+ struct mlx5_flow_table *ft,
+ const struct mlx5_flow_spec *spec,
+ struct mlx5_flow_group *fg,
+ bool ft_locked)
+{
+ struct mlx5_flow_group *g;
+ unsigned long id;
+
+ rcu_read_lock();
+ INIT_LIST_HEAD(&match_head->list);
+ xa_for_each_rcu(&ft->fgs_xa, id, g) {
+ struct match_list *curr_match;
+
+ if (memcmp(&g->mask, spec, sizeof_field(struct mlx5_flow_group,
+ mask)))
+ continue;
+
+ if (fg && fg != g)
+ continue;
+
+ if (unlikely(!tree_get_node(&g->node)))
+ continue;
+
+ curr_match = kmalloc(sizeof(*curr_match), GFP_ATOMIC);
+ if (!curr_match) {
+ rcu_read_unlock();
+ free_match_list(match_head, ft_locked);
+ return -ENOMEM;
+ }
+ curr_match->g = g;
+ list_add_tail(&curr_match->list, &match_head->list);
+ }
+ rcu_read_unlock();
+ return 0;
+}
+
+static u64 matched_fgs_get_version(struct list_head *match_head)
+{
+ struct match_list *iter;
+ u64 version = 0;
+
+ list_for_each_entry(iter, match_head, list)
+ version += (u64)atomic_read(&iter->g->node.version);
+ return version;
+}
+
+static struct fs_fte *
+lookup_fte_locked(struct mlx5_flow_group *g,
+ const u32 *match_value,
+ bool take_write)
+{
+ struct fs_fte *fte_tmp;
+ unsigned long index;
+
+ if (take_write)
+ nested_down_write_ref_node(&g->node, FS_LOCK_PARENT);
+ else
+ nested_down_read_ref_node(&g->node, FS_LOCK_PARENT);
+ xa_for_each_rcu(&g->ftes_xa, index, fte_tmp) {
+ if (!memcmp(match_value, fte_tmp->val, sizeof_field(struct fs_fte, val)))
+ break;
+ }
+ if (!fte_tmp || !tree_get_node(&fte_tmp->node)) {
+ fte_tmp = NULL;
+ goto out;
+ }
+ if (!fte_tmp->node.active) {
+ tree_put_node(&fte_tmp->node, false);
+ fte_tmp = NULL;
+ goto out;
+ }
+
+ nested_down_write_ref_node(&fte_tmp->node, FS_LOCK_CHILD);
+out:
+ if (take_write)
+ up_write_ref_node(&g->node, false);
+ else
+ up_read_ref_node(&g->node);
+ return fte_tmp;
+}
+
+static struct mlx5_flow_handle *
+try_add_to_existing_fg(struct mlx5_flow_table *ft,
+ struct list_head *match_head,
+ const struct mlx5_flow_spec *spec,
+ struct mlx5_flow_act *flow_act,
+ struct mlx5_flow_destination *dest,
+ int dest_num,
+ int ft_version)
+{
+ struct mlx5_flow_steering *steering = get_steering(&ft->node);
+ struct mlx5_flow_group *g;
+ struct mlx5_flow_handle *rule;
+ struct match_list *iter;
+ bool take_write = false;
+ struct fs_fte *fte;
+ u64 version = 0;
+ int err;
+
+ fte = alloc_fte(ft, spec, flow_act);
+ if (IS_ERR(fte))
+ return ERR_PTR(-ENOMEM);
+
+search_again_locked:
+ if (flow_act->flags & FLOW_ACT_NO_APPEND)
+ goto skip_search;
+ version = matched_fgs_get_version(match_head);
+ /* Try to find an fte with identical match value and attempt update its
+ * action.
+ */
+ list_for_each_entry(iter, match_head, list) {
+ struct fs_fte *fte_tmp;
+
+ g = iter->g;
+ fte_tmp = lookup_fte_locked(g, spec->match_value, take_write);
+ if (!fte_tmp)
+ continue;
+ rule = add_rule_fg(g, spec, flow_act, dest, dest_num, fte_tmp);
+ /* No error check needed here, because insert_fte() is not called */
+ up_write_ref_node(&fte_tmp->node, false);
+ tree_put_node(&fte_tmp->node, false);
+ kmem_cache_free(steering->ftes_cache, fte);
+ return rule;
+ }
+
+skip_search:
+ /* No group with matching fte found, or we skipped the search.
+ * Try to add a new fte to any matching fg.
+ */
+
+ /* Check the ft version, for case that new flow group
+ * was added while the fgs weren't locked
+ */
+ if (atomic_read(&ft->node.version) != ft_version) {
+ rule = ERR_PTR(-EAGAIN);
+ goto out;
+ }
+
+ /* Check the fgs version. If version have changed it could be that an
+ * FTE with the same match value was added while the fgs weren't
+ * locked.
+ */
+ if (!(flow_act->flags & FLOW_ACT_NO_APPEND) &&
+ version != matched_fgs_get_version(match_head)) {
+ take_write = true;
+ goto search_again_locked;
+ }
+
+ list_for_each_entry(iter, match_head, list) {
+ g = iter->g;
+
+ nested_down_write_ref_node(&g->node, FS_LOCK_PARENT);
+
+ if (!g->node.active) {
+ up_write_ref_node(&g->node, false);
+ continue;
+ }
+
+ err = insert_fte(g, fte);
+ if (err) {
+ up_write_ref_node(&g->node, false);
+ if (err == -ENOSPC)
+ continue;
+ kmem_cache_free(steering->ftes_cache, fte);
+ return ERR_PTR(err);
+ }
+
+ nested_down_write_ref_node(&fte->node, FS_LOCK_CHILD);
+ up_write_ref_node(&g->node, false);
+ rule = add_rule_fg(g, spec, flow_act, dest, dest_num, fte);
+ up_write_ref_node(&fte->node, false);
+ if (IS_ERR(rule))
+ tree_put_node(&fte->node, false);
+ return rule;
+ }
+ rule = ERR_PTR(-ENOENT);
+out:
+ kmem_cache_free(steering->ftes_cache, fte);
+ return rule;
+}
+
+static struct mlx5_flow_handle *
+_mlx5_add_flow_rules(struct mlx5_flow_table *ft,
+ const struct mlx5_flow_spec *spec,
+ struct mlx5_flow_act *flow_act,
+ struct mlx5_flow_destination *dest,
+ int dest_num)
+
+{
+ struct mlx5_flow_steering *steering = get_steering(&ft->node);
+ struct mlx5_flow_handle *rule;
+ struct match_list match_head;
+ struct mlx5_flow_group *g;
+ bool take_write = false;
+ struct fs_fte *fte;
+ int version;
+ int err;
+ int i;
+
+ if (!check_valid_spec(spec))
+ return ERR_PTR(-EINVAL);
+
+ if (flow_act->fg && ft->autogroup.active)
+ return ERR_PTR(-EINVAL);
+
+ if (dest && dest_num <= 0)
+ return ERR_PTR(-EINVAL);
+
+ for (i = 0; i < dest_num; i++) {
+ if (!dest_is_valid(&dest[i], flow_act, ft))
+ return ERR_PTR(-EINVAL);
+ }
+ nested_down_read_ref_node(&ft->node, FS_LOCK_GRANDPARENT);
+search_again_locked:
+ version = atomic_read(&ft->node.version);
+
+ /* Collect all fgs which has a matching match_criteria */
+ err = build_match_list(&match_head, ft, spec, flow_act->fg, take_write);
+ if (err) {
+ if (take_write)
+ up_write_ref_node(&ft->node, false);
+ else
+ up_read_ref_node(&ft->node);
+ return ERR_PTR(err);
+ }
+
+ if (!take_write)
+ up_read_ref_node(&ft->node);
+
+ rule = try_add_to_existing_fg(ft, &match_head.list, spec, flow_act, dest,
+ dest_num, version);
+ free_match_list(&match_head, take_write);
+ if (!IS_ERR(rule) ||
+ (PTR_ERR(rule) != -ENOENT && PTR_ERR(rule) != -EAGAIN)) {
+ if (take_write)
+ up_write_ref_node(&ft->node, false);
+ return rule;
+ }
+
+ if (!take_write) {
+ nested_down_write_ref_node(&ft->node, FS_LOCK_GRANDPARENT);
+ take_write = true;
+ }
+
+ if (PTR_ERR(rule) == -EAGAIN ||
+ version != atomic_read(&ft->node.version))
+ goto search_again_locked;
+
+ g = alloc_auto_flow_group(ft, spec);
+ if (IS_ERR(g)) {
+ rule = ERR_CAST(g);
+ up_write_ref_node(&ft->node, false);
+ return rule;
+ }
+
+ fte = alloc_fte(ft, spec, flow_act);
+ if (IS_ERR(fte)) {
+ up_write_ref_node(&ft->node, false);
+ err = PTR_ERR(fte);
+ goto err_alloc_fte;
+ }
+
+ nested_down_write_ref_node(&g->node, FS_LOCK_PARENT);
+ up_write_ref_node(&ft->node, false);
+
+ err = create_auto_flow_group(ft, g);
+ if (err)
+ goto err_release_fg;
+
+ err = insert_fte(g, fte);
+ if (err)
+ goto err_release_fg;
+
+ nested_down_write_ref_node(&fte->node, FS_LOCK_CHILD);
+ up_write_ref_node(&g->node, false);
+ rule = add_rule_fg(g, spec, flow_act, dest, dest_num, fte);
+ up_write_ref_node(&fte->node, false);
+ if (IS_ERR(rule))
+ tree_put_node(&fte->node, false);
+ tree_put_node(&g->node, false);
+ return rule;
+
+err_release_fg:
+ up_write_ref_node(&g->node, false);
+ kmem_cache_free(steering->ftes_cache, fte);
+err_alloc_fte:
+ tree_put_node(&g->node, false);
+ return ERR_PTR(err);
+}
+
+static bool fwd_next_prio_supported(struct mlx5_flow_table *ft)
+{
+ return ((ft->type == FS_FT_NIC_RX) &&
+ (MLX5_CAP_FLOWTABLE(get_dev(&ft->node), nic_rx_multi_path_tirs)));
+}
+
+struct mlx5_flow_handle *
+mlx5_add_flow_rules(struct mlx5_flow_table *ft,
+ const struct mlx5_flow_spec *spec,
+ struct mlx5_flow_act *flow_act,
+ struct mlx5_flow_destination *dest,
+ int num_dest)
+{
+ struct mlx5_flow_root_namespace *root = find_root(&ft->node);
+ static const struct mlx5_flow_spec zero_spec = {};
+ struct mlx5_flow_destination *gen_dest = NULL;
+ struct mlx5_flow_table *next_ft = NULL;
+ struct mlx5_flow_handle *handle = NULL;
+ u32 sw_action = flow_act->action;
+ int i;
+
+ if (!spec)
+ spec = &zero_spec;
+
+ if (!is_fwd_next_action(sw_action))
+ return _mlx5_add_flow_rules(ft, spec, flow_act, dest, num_dest);
+
+ if (!fwd_next_prio_supported(ft))
+ return ERR_PTR(-EOPNOTSUPP);
+
+ mutex_lock(&root->chain_lock);
+ next_ft = find_next_fwd_ft(ft, flow_act);
+ if (!next_ft) {
+ handle = ERR_PTR(-EOPNOTSUPP);
+ goto unlock;
+ }
+
+ gen_dest = kcalloc(num_dest + 1, sizeof(*dest),
+ GFP_KERNEL);
+ if (!gen_dest) {
+ handle = ERR_PTR(-ENOMEM);
+ goto unlock;
+ }
+ for (i = 0; i < num_dest; i++)
+ gen_dest[i] = dest[i];
+ gen_dest[i].type =
+ MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
+ gen_dest[i].ft = next_ft;
+ dest = gen_dest;
+ num_dest++;
+ flow_act->action &= ~(MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO |
+ MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS);
+ flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
+ handle = _mlx5_add_flow_rules(ft, spec, flow_act, dest, num_dest);
+ if (IS_ERR(handle))
+ goto unlock;
+
+ if (list_empty(&handle->rule[num_dest - 1]->next_ft)) {
+ mutex_lock(&next_ft->lock);
+ list_add(&handle->rule[num_dest - 1]->next_ft,
+ &next_ft->fwd_rules);
+ mutex_unlock(&next_ft->lock);
+ handle->rule[num_dest - 1]->sw_action = sw_action;
+ handle->rule[num_dest - 1]->ft = ft;
+ }
+unlock:
+ mutex_unlock(&root->chain_lock);
+ kfree(gen_dest);
+ return handle;
+}
+EXPORT_SYMBOL(mlx5_add_flow_rules);
+
+void mlx5_del_flow_rules(struct mlx5_flow_handle **pp)
+{
+ struct mlx5_flow_handle *handle;
+ struct fs_fte *fte;
+ int i;
+
+ handle = *pp;
+ *pp = NULL;
+ if (IS_ERR_OR_NULL(handle))
+ return;
+
+ /* In order to consolidate the HW changes we lock the FTE for other
+ * changes, and increase its refcount, in order not to perform the
+ * "del" functions of the FTE. Will handle them here.
+ * The removal of the rules is done under locked FTE.
+ * After removing all the handle's rules, if there are remaining
+ * rules, it means we just need to modify the FTE in FW, and
+ * unlock/decrease the refcount we increased before.
+ * Otherwise, it means the FTE should be deleted. First delete the
+ * FTE in FW. Then, unlock the FTE, and proceed the tree_put_node of
+ * the FTE, which will handle the last decrease of the refcount, as
+ * well as required handling of its parent.
+ */
+ fs_get_obj(fte, handle->rule[0]->node.parent);
+ down_write_ref_node(&fte->node, false);
+ for (i = handle->num_rules - 1; i >= 0; i--)
+ tree_remove_node(&handle->rule[i]->node, true);
+ if (list_empty(&fte->node.children)) {
+ fte->node.del_hw_func(&fte->node);
+ /* Avoid double call to del_hw_fte */
+ fte->node.del_hw_func = NULL;
+ up_write_ref_node(&fte->node, false);
+ tree_put_node(&fte->node, false);
+ } else if (fte->dests_size) {
+ if (fte->modify_mask)
+ modify_fte(fte);
+ up_write_ref_node(&fte->node, false);
+ } else {
+ up_write_ref_node(&fte->node, false);
+ }
+ kfree(handle);
+}
+EXPORT_SYMBOL(mlx5_del_flow_rules);
+
+/* Assuming prio->node.children(flow tables) is sorted by level */
+static struct mlx5_flow_table *find_next_ft(struct mlx5_flow_table *ft)
+{
+ struct fs_prio *prio;
+
+ fs_get_obj(prio, ft->node.parent);
+
+ if (!list_is_last(&ft->node.list, &prio->node.children))
+ return list_next_entry(ft, node.list);
+ return find_next_chained_ft(prio);
+}
+
+static int update_root_ft_destroy(struct mlx5_flow_table *ft)
+{
+ struct mlx5_flow_root_namespace *root = find_root(&ft->node);
+ struct mlx5_ft_underlay_qp *uqp;
+ struct mlx5_flow_table *new_root_ft = NULL;
+ int err = 0;
+ u32 qpn;
+
+ if (root->root_ft != ft)
+ return 0;
+
+ new_root_ft = find_next_ft(ft);
+ if (!new_root_ft) {
+ root->root_ft = NULL;
+ return 0;
+ }
+
+ if (list_empty(&root->underlay_qpns)) {
+ /* Don't set any QPN (zero) in case QPN list is empty */
+ qpn = 0;
+ err = root->cmds->update_root_ft(root, new_root_ft,
+ qpn, false);
+ } else {
+ list_for_each_entry(uqp, &root->underlay_qpns, list) {
+ qpn = uqp->qpn;
+ err = root->cmds->update_root_ft(root,
+ new_root_ft, qpn,
+ false);
+ if (err)
+ break;
+ }
+ }
+
+ if (err)
+ mlx5_core_warn(root->dev,
+ "Update root flow table of id(%u) qpn(%d) failed\n",
+ ft->id, qpn);
+ else
+ root->root_ft = new_root_ft;
+
+ return 0;
+}
+
+/* Connect flow table from previous priority to
+ * the next flow table.
+ */
+static int disconnect_flow_table(struct mlx5_flow_table *ft)
+{
+ struct mlx5_core_dev *dev = get_dev(&ft->node);
+ struct mlx5_flow_table *next_ft;
+ struct fs_prio *prio;
+ int err = 0;
+
+ err = update_root_ft_destroy(ft);
+ if (err)
+ return err;
+
+ fs_get_obj(prio, ft->node.parent);
+ if (!(list_first_entry(&prio->node.children,
+ struct mlx5_flow_table,
+ node.list) == ft))
+ return 0;
+
+ next_ft = find_next_ft(ft);
+ err = connect_fwd_rules(dev, next_ft, ft);
+ if (err)
+ return err;
+
+ err = connect_prev_fts(dev, next_ft, prio);
+ if (err)
+ mlx5_core_warn(dev, "Failed to disconnect flow table %d\n",
+ ft->id);
+ return err;
+}
+
+int mlx5_destroy_flow_table(struct mlx5_flow_table *ft)
+{
+ struct mlx5_flow_root_namespace *root = find_root(&ft->node);
+ int err = 0;
+
+ mutex_lock(&root->chain_lock);
+ if (!(ft->flags & MLX5_FLOW_TABLE_UNMANAGED))
+ err = disconnect_flow_table(ft);
+ if (err) {
+ mutex_unlock(&root->chain_lock);
+ return err;
+ }
+ if (tree_remove_node(&ft->node, false))
+ mlx5_core_warn(get_dev(&ft->node), "Flow table %d wasn't destroyed, refcount > 1\n",
+ ft->id);
+ mutex_unlock(&root->chain_lock);
+
+ return err;
+}
+EXPORT_SYMBOL(mlx5_destroy_flow_table);
+
+void mlx5_destroy_flow_group(struct mlx5_flow_group *fg)
+{
+ if (tree_remove_node(&fg->node, false))
+ mlx5_core_warn(get_dev(&fg->node), "Flow group %d wasn't destroyed, refcount > 1\n",
+ fg->id);
+}
+EXPORT_SYMBOL(mlx5_destroy_flow_group);
+
+struct mlx5_flow_namespace *mlx5_get_fdb_sub_ns(struct mlx5_core_dev *dev,
+ int n)
+{
+ struct mlx5_flow_steering *steering = dev->priv.steering;
+
+ if (!steering || !steering->fdb_sub_ns)
+ return NULL;
+
+ return steering->fdb_sub_ns[n];
+}
+EXPORT_SYMBOL(mlx5_get_fdb_sub_ns);
+
+static bool is_nic_rx_ns(enum mlx5_flow_namespace_type type)
+{
+ switch (type) {
+ case MLX5_FLOW_NAMESPACE_BYPASS:
+ case MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC:
+ case MLX5_FLOW_NAMESPACE_LAG:
+ case MLX5_FLOW_NAMESPACE_OFFLOADS:
+ case MLX5_FLOW_NAMESPACE_ETHTOOL:
+ case MLX5_FLOW_NAMESPACE_KERNEL:
+ case MLX5_FLOW_NAMESPACE_LEFTOVERS:
+ case MLX5_FLOW_NAMESPACE_ANCHOR:
+ return true;
+ default:
+ return false;
+ }
+}
+
+struct mlx5_flow_namespace *mlx5_get_flow_namespace(struct mlx5_core_dev *dev,
+ enum mlx5_flow_namespace_type type)
+{
+ struct mlx5_flow_steering *steering = dev->priv.steering;
+ struct mlx5_flow_root_namespace *root_ns;
+ int prio = 0;
+ struct fs_prio *fs_prio;
+ struct mlx5_flow_namespace *ns;
+
+ if (!steering)
+ return NULL;
+
+ switch (type) {
+ case MLX5_FLOW_NAMESPACE_FDB:
+ if (steering->fdb_root_ns)
+ return &steering->fdb_root_ns->ns;
+ return NULL;
+ case MLX5_FLOW_NAMESPACE_PORT_SEL:
+ if (steering->port_sel_root_ns)
+ return &steering->port_sel_root_ns->ns;
+ return NULL;
+ case MLX5_FLOW_NAMESPACE_SNIFFER_RX:
+ if (steering->sniffer_rx_root_ns)
+ return &steering->sniffer_rx_root_ns->ns;
+ return NULL;
+ case MLX5_FLOW_NAMESPACE_SNIFFER_TX:
+ if (steering->sniffer_tx_root_ns)
+ return &steering->sniffer_tx_root_ns->ns;
+ return NULL;
+ case MLX5_FLOW_NAMESPACE_FDB_BYPASS:
+ root_ns = steering->fdb_root_ns;
+ prio = FDB_BYPASS_PATH;
+ break;
+ case MLX5_FLOW_NAMESPACE_EGRESS:
+ case MLX5_FLOW_NAMESPACE_EGRESS_IPSEC:
+ case MLX5_FLOW_NAMESPACE_EGRESS_MACSEC:
+ root_ns = steering->egress_root_ns;
+ prio = type - MLX5_FLOW_NAMESPACE_EGRESS;
+ break;
+ case MLX5_FLOW_NAMESPACE_RDMA_RX:
+ root_ns = steering->rdma_rx_root_ns;
+ prio = RDMA_RX_BYPASS_PRIO;
+ break;
+ case MLX5_FLOW_NAMESPACE_RDMA_RX_KERNEL:
+ root_ns = steering->rdma_rx_root_ns;
+ prio = RDMA_RX_KERNEL_PRIO;
+ break;
+ case MLX5_FLOW_NAMESPACE_RDMA_TX:
+ root_ns = steering->rdma_tx_root_ns;
+ break;
+ case MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS:
+ root_ns = steering->rdma_rx_root_ns;
+ prio = RDMA_RX_COUNTERS_PRIO;
+ break;
+ case MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS:
+ root_ns = steering->rdma_tx_root_ns;
+ prio = RDMA_TX_COUNTERS_PRIO;
+ break;
+ case MLX5_FLOW_NAMESPACE_RDMA_RX_IPSEC:
+ root_ns = steering->rdma_rx_root_ns;
+ prio = RDMA_RX_IPSEC_PRIO;
+ break;
+ case MLX5_FLOW_NAMESPACE_RDMA_TX_IPSEC:
+ root_ns = steering->rdma_tx_root_ns;
+ prio = RDMA_TX_IPSEC_PRIO;
+ break;
+ default: /* Must be NIC RX */
+ WARN_ON(!is_nic_rx_ns(type));
+ root_ns = steering->root_ns;
+ prio = type;
+ break;
+ }
+
+ if (!root_ns)
+ return NULL;
+
+ fs_prio = find_prio(&root_ns->ns, prio);
+ if (!fs_prio)
+ return NULL;
+
+ ns = list_first_entry(&fs_prio->node.children,
+ typeof(*ns),
+ node.list);
+
+ return ns;
+}
+EXPORT_SYMBOL(mlx5_get_flow_namespace);
+
+struct mlx5_flow_namespace *mlx5_get_flow_vport_acl_namespace(struct mlx5_core_dev *dev,
+ enum mlx5_flow_namespace_type type,
+ int vport)
+{
+ struct mlx5_flow_steering *steering = dev->priv.steering;
+
+ if (!steering)
+ return NULL;
+
+ switch (type) {
+ case MLX5_FLOW_NAMESPACE_ESW_EGRESS:
+ if (vport >= steering->esw_egress_acl_vports)
+ return NULL;
+ if (steering->esw_egress_root_ns &&
+ steering->esw_egress_root_ns[vport])
+ return &steering->esw_egress_root_ns[vport]->ns;
+ else
+ return NULL;
+ case MLX5_FLOW_NAMESPACE_ESW_INGRESS:
+ if (vport >= steering->esw_ingress_acl_vports)
+ return NULL;
+ if (steering->esw_ingress_root_ns &&
+ steering->esw_ingress_root_ns[vport])
+ return &steering->esw_ingress_root_ns[vport]->ns;
+ else
+ return NULL;
+ default:
+ return NULL;
+ }
+}
+
+static struct fs_prio *_fs_create_prio(struct mlx5_flow_namespace *ns,
+ unsigned int prio,
+ int num_levels,
+ enum fs_node_type type)
+{
+ struct fs_prio *fs_prio;
+
+ fs_prio = kzalloc(sizeof(*fs_prio), GFP_KERNEL);
+ if (!fs_prio)
+ return ERR_PTR(-ENOMEM);
+
+ fs_prio->node.type = type;
+ tree_init_node(&fs_prio->node, NULL, del_sw_prio);
+ tree_add_node(&fs_prio->node, &ns->node);
+ fs_prio->num_levels = num_levels;
+ fs_prio->prio = prio;
+ list_add_tail(&fs_prio->node.list, &ns->node.children);
+
+ return fs_prio;
+}
+
+static struct fs_prio *fs_create_prio_chained(struct mlx5_flow_namespace *ns,
+ unsigned int prio,
+ int num_levels)
+{
+ return _fs_create_prio(ns, prio, num_levels, FS_TYPE_PRIO_CHAINS);
+}
+
+static struct fs_prio *fs_create_prio(struct mlx5_flow_namespace *ns,
+ unsigned int prio, int num_levels)
+{
+ return _fs_create_prio(ns, prio, num_levels, FS_TYPE_PRIO);
+}
+
+static struct mlx5_flow_namespace *fs_init_namespace(struct mlx5_flow_namespace
+ *ns)
+{
+ ns->node.type = FS_TYPE_NAMESPACE;
+
+ return ns;
+}
+
+static struct mlx5_flow_namespace *fs_create_namespace(struct fs_prio *prio,
+ int def_miss_act)
+{
+ struct mlx5_flow_namespace *ns;
+
+ ns = kzalloc(sizeof(*ns), GFP_KERNEL);
+ if (!ns)
+ return ERR_PTR(-ENOMEM);
+
+ fs_init_namespace(ns);
+ ns->def_miss_action = def_miss_act;
+ tree_init_node(&ns->node, NULL, del_sw_ns);
+ tree_add_node(&ns->node, &prio->node);
+ list_add_tail(&ns->node.list, &prio->node.children);
+
+ return ns;
+}
+
+static int create_leaf_prios(struct mlx5_flow_namespace *ns, int prio,
+ struct init_tree_node *prio_metadata)
+{
+ struct fs_prio *fs_prio;
+ int i;
+
+ for (i = 0; i < prio_metadata->num_leaf_prios; i++) {
+ fs_prio = fs_create_prio(ns, prio++, prio_metadata->num_levels);
+ if (IS_ERR(fs_prio))
+ return PTR_ERR(fs_prio);
+ }
+ return 0;
+}
+
+#define FLOW_TABLE_BIT_SZ 1
+#define GET_FLOW_TABLE_CAP(dev, offset) \
+ ((be32_to_cpu(*((__be32 *)(dev->hca_caps_cur[MLX5_CAP_FLOW_TABLE]) + \
+ offset / 32)) >> \
+ (32 - FLOW_TABLE_BIT_SZ - (offset & 0x1f))) & FLOW_TABLE_BIT_SZ)
+static bool has_required_caps(struct mlx5_core_dev *dev, struct node_caps *caps)
+{
+ int i;
+
+ for (i = 0; i < caps->arr_sz; i++) {
+ if (!GET_FLOW_TABLE_CAP(dev, caps->caps[i]))
+ return false;
+ }
+ return true;
+}
+
+static int init_root_tree_recursive(struct mlx5_flow_steering *steering,
+ struct init_tree_node *init_node,
+ struct fs_node *fs_parent_node,
+ struct init_tree_node *init_parent_node,
+ int prio)
+{
+ int max_ft_level = MLX5_CAP_FLOWTABLE(steering->dev,
+ flow_table_properties_nic_receive.
+ max_ft_level);
+ struct mlx5_flow_namespace *fs_ns;
+ struct fs_prio *fs_prio;
+ struct fs_node *base;
+ int i;
+ int err;
+
+ if (init_node->type == FS_TYPE_PRIO) {
+ if ((init_node->min_ft_level > max_ft_level) ||
+ !has_required_caps(steering->dev, &init_node->caps))
+ return 0;
+
+ fs_get_obj(fs_ns, fs_parent_node);
+ if (init_node->num_leaf_prios)
+ return create_leaf_prios(fs_ns, prio, init_node);
+ fs_prio = fs_create_prio(fs_ns, prio, init_node->num_levels);
+ if (IS_ERR(fs_prio))
+ return PTR_ERR(fs_prio);
+ base = &fs_prio->node;
+ } else if (init_node->type == FS_TYPE_NAMESPACE) {
+ fs_get_obj(fs_prio, fs_parent_node);
+ fs_ns = fs_create_namespace(fs_prio, init_node->def_miss_action);
+ if (IS_ERR(fs_ns))
+ return PTR_ERR(fs_ns);
+ base = &fs_ns->node;
+ } else {
+ return -EINVAL;
+ }
+ prio = 0;
+ for (i = 0; i < init_node->ar_size; i++) {
+ err = init_root_tree_recursive(steering, &init_node->children[i],
+ base, init_node, prio);
+ if (err)
+ return err;
+ if (init_node->children[i].type == FS_TYPE_PRIO &&
+ init_node->children[i].num_leaf_prios) {
+ prio += init_node->children[i].num_leaf_prios;
+ }
+ }
+
+ return 0;
+}
+
+static int init_root_tree(struct mlx5_flow_steering *steering,
+ struct init_tree_node *init_node,
+ struct fs_node *fs_parent_node)
+{
+ int err;
+ int i;
+
+ for (i = 0; i < init_node->ar_size; i++) {
+ err = init_root_tree_recursive(steering, &init_node->children[i],
+ fs_parent_node,
+ init_node, i);
+ if (err)
+ return err;
+ }
+ return 0;
+}
+
+static void del_sw_root_ns(struct fs_node *node)
+{
+ struct mlx5_flow_root_namespace *root_ns;
+ struct mlx5_flow_namespace *ns;
+
+ fs_get_obj(ns, node);
+ root_ns = container_of(ns, struct mlx5_flow_root_namespace, ns);
+ mutex_destroy(&root_ns->chain_lock);
+ kfree(node);
+}
+
+static struct mlx5_flow_root_namespace
+*create_root_ns(struct mlx5_flow_steering *steering,
+ enum fs_flow_table_type table_type)
+{
+ const struct mlx5_flow_cmds *cmds = mlx5_fs_cmd_get_default(table_type);
+ struct mlx5_flow_root_namespace *root_ns;
+ struct mlx5_flow_namespace *ns;
+
+ /* Create the root namespace */
+ root_ns = kzalloc(sizeof(*root_ns), GFP_KERNEL);
+ if (!root_ns)
+ return NULL;
+
+ root_ns->dev = steering->dev;
+ root_ns->table_type = table_type;
+ root_ns->cmds = cmds;
+
+ INIT_LIST_HEAD(&root_ns->underlay_qpns);
+
+ ns = &root_ns->ns;
+ fs_init_namespace(ns);
+ mutex_init(&root_ns->chain_lock);
+ tree_init_node(&ns->node, NULL, del_sw_root_ns);
+ tree_add_node(&ns->node, NULL);
+
+ return root_ns;
+}
+
+static void set_prio_attrs_in_prio(struct fs_prio *prio, int acc_level);
+
+static int set_prio_attrs_in_ns(struct mlx5_flow_namespace *ns, int acc_level)
+{
+ struct fs_prio *prio;
+
+ fs_for_each_prio(prio, ns) {
+ /* This updates prio start_level and num_levels */
+ set_prio_attrs_in_prio(prio, acc_level);
+ acc_level += prio->num_levels;
+ }
+ return acc_level;
+}
+
+static void set_prio_attrs_in_prio(struct fs_prio *prio, int acc_level)
+{
+ struct mlx5_flow_namespace *ns;
+ int acc_level_ns = acc_level;
+
+ prio->start_level = acc_level;
+ fs_for_each_ns(ns, prio) {
+ /* This updates start_level and num_levels of ns's priority descendants */
+ acc_level_ns = set_prio_attrs_in_ns(ns, acc_level);
+
+ /* If this a prio with chains, and we can jump from one chain
+ * (namespace) to another, so we accumulate the levels
+ */
+ if (prio->node.type == FS_TYPE_PRIO_CHAINS)
+ acc_level = acc_level_ns;
+ }
+
+ if (!prio->num_levels)
+ prio->num_levels = acc_level_ns - prio->start_level;
+ WARN_ON(prio->num_levels < acc_level_ns - prio->start_level);
+}
+
+static void set_prio_attrs(struct mlx5_flow_root_namespace *root_ns)
+{
+ struct mlx5_flow_namespace *ns = &root_ns->ns;
+ struct fs_prio *prio;
+ int start_level = 0;
+
+ fs_for_each_prio(prio, ns) {
+ set_prio_attrs_in_prio(prio, start_level);
+ start_level += prio->num_levels;
+ }
+}
+
+#define ANCHOR_PRIO 0
+#define ANCHOR_SIZE 1
+#define ANCHOR_LEVEL 0
+static int create_anchor_flow_table(struct mlx5_flow_steering *steering)
+{
+ struct mlx5_flow_namespace *ns = NULL;
+ struct mlx5_flow_table_attr ft_attr = {};
+ struct mlx5_flow_table *ft;
+
+ ns = mlx5_get_flow_namespace(steering->dev, MLX5_FLOW_NAMESPACE_ANCHOR);
+ if (WARN_ON(!ns))
+ return -EINVAL;
+
+ ft_attr.max_fte = ANCHOR_SIZE;
+ ft_attr.level = ANCHOR_LEVEL;
+ ft_attr.prio = ANCHOR_PRIO;
+
+ ft = mlx5_create_flow_table(ns, &ft_attr);
+ if (IS_ERR(ft)) {
+ mlx5_core_err(steering->dev, "Failed to create last anchor flow table");
+ return PTR_ERR(ft);
+ }
+ return 0;
+}
+
+static int init_root_ns(struct mlx5_flow_steering *steering)
+{
+ int err;
+
+ steering->root_ns = create_root_ns(steering, FS_FT_NIC_RX);
+ if (!steering->root_ns)
+ return -ENOMEM;
+
+ err = init_root_tree(steering, &root_fs, &steering->root_ns->ns.node);
+ if (err)
+ goto out_err;
+
+ set_prio_attrs(steering->root_ns);
+ err = create_anchor_flow_table(steering);
+ if (err)
+ goto out_err;
+
+ return 0;
+
+out_err:
+ cleanup_root_ns(steering->root_ns);
+ steering->root_ns = NULL;
+ return err;
+}
+
+static void clean_tree(struct fs_node *node)
+{
+ if (node) {
+ struct fs_node *iter;
+ struct fs_node *temp;
+
+ tree_get_node(node);
+ list_for_each_entry_safe(iter, temp, &node->children, list)
+ clean_tree(iter);
+ tree_put_node(node, false);
+ tree_remove_node(node, false);
+ }
+}
+
+static void cleanup_root_ns(struct mlx5_flow_root_namespace *root_ns)
+{
+ if (!root_ns)
+ return;
+
+ clean_tree(&root_ns->ns.node);
+}
+
+static int init_sniffer_tx_root_ns(struct mlx5_flow_steering *steering)
+{
+ struct fs_prio *prio;
+
+ steering->sniffer_tx_root_ns = create_root_ns(steering, FS_FT_SNIFFER_TX);
+ if (!steering->sniffer_tx_root_ns)
+ return -ENOMEM;
+
+ /* Create single prio */
+ prio = fs_create_prio(&steering->sniffer_tx_root_ns->ns, 0, 1);
+ return PTR_ERR_OR_ZERO(prio);
+}
+
+static int init_sniffer_rx_root_ns(struct mlx5_flow_steering *steering)
+{
+ struct fs_prio *prio;
+
+ steering->sniffer_rx_root_ns = create_root_ns(steering, FS_FT_SNIFFER_RX);
+ if (!steering->sniffer_rx_root_ns)
+ return -ENOMEM;
+
+ /* Create single prio */
+ prio = fs_create_prio(&steering->sniffer_rx_root_ns->ns, 0, 1);
+ return PTR_ERR_OR_ZERO(prio);
+}
+
+#define PORT_SEL_NUM_LEVELS 3
+static int init_port_sel_root_ns(struct mlx5_flow_steering *steering)
+{
+ struct fs_prio *prio;
+
+ steering->port_sel_root_ns = create_root_ns(steering, FS_FT_PORT_SEL);
+ if (!steering->port_sel_root_ns)
+ return -ENOMEM;
+
+ /* Create single prio */
+ prio = fs_create_prio(&steering->port_sel_root_ns->ns, 0,
+ PORT_SEL_NUM_LEVELS);
+ return PTR_ERR_OR_ZERO(prio);
+}
+
+static int init_rdma_rx_root_ns(struct mlx5_flow_steering *steering)
+{
+ int err;
+
+ steering->rdma_rx_root_ns = create_root_ns(steering, FS_FT_RDMA_RX);
+ if (!steering->rdma_rx_root_ns)
+ return -ENOMEM;
+
+ err = init_root_tree(steering, &rdma_rx_root_fs,
+ &steering->rdma_rx_root_ns->ns.node);
+ if (err)
+ goto out_err;
+
+ set_prio_attrs(steering->rdma_rx_root_ns);
+
+ return 0;
+
+out_err:
+ cleanup_root_ns(steering->rdma_rx_root_ns);
+ steering->rdma_rx_root_ns = NULL;
+ return err;
+}
+
+static int init_rdma_tx_root_ns(struct mlx5_flow_steering *steering)
+{
+ int err;
+
+ steering->rdma_tx_root_ns = create_root_ns(steering, FS_FT_RDMA_TX);
+ if (!steering->rdma_tx_root_ns)
+ return -ENOMEM;
+
+ err = init_root_tree(steering, &rdma_tx_root_fs,
+ &steering->rdma_tx_root_ns->ns.node);
+ if (err)
+ goto out_err;
+
+ set_prio_attrs(steering->rdma_tx_root_ns);
+
+ return 0;
+
+out_err:
+ cleanup_root_ns(steering->rdma_tx_root_ns);
+ steering->rdma_tx_root_ns = NULL;
+ return err;
+}
+
+/* FT and tc chains are stored in the same array so we can re-use the
+ * mlx5_get_fdb_sub_ns() and tc api for FT chains.
+ * When creating a new ns for each chain store it in the first available slot.
+ * Assume tc chains are created and stored first and only then the FT chain.
+ */
+static void store_fdb_sub_ns_prio_chain(struct mlx5_flow_steering *steering,
+ struct mlx5_flow_namespace *ns)
+{
+ int chain = 0;
+
+ while (steering->fdb_sub_ns[chain])
+ ++chain;
+
+ steering->fdb_sub_ns[chain] = ns;
+}
+
+static int create_fdb_sub_ns_prio_chain(struct mlx5_flow_steering *steering,
+ struct fs_prio *maj_prio)
+{
+ struct mlx5_flow_namespace *ns;
+ struct fs_prio *min_prio;
+ int prio;
+
+ ns = fs_create_namespace(maj_prio, MLX5_FLOW_TABLE_MISS_ACTION_DEF);
+ if (IS_ERR(ns))
+ return PTR_ERR(ns);
+
+ for (prio = 0; prio < FDB_TC_MAX_PRIO; prio++) {
+ min_prio = fs_create_prio(ns, prio, FDB_TC_LEVELS_PER_PRIO);
+ if (IS_ERR(min_prio))
+ return PTR_ERR(min_prio);
+ }
+
+ store_fdb_sub_ns_prio_chain(steering, ns);
+
+ return 0;
+}
+
+static int create_fdb_chains(struct mlx5_flow_steering *steering,
+ int fs_prio,
+ int chains)
+{
+ struct fs_prio *maj_prio;
+ int levels;
+ int chain;
+ int err;
+
+ levels = FDB_TC_LEVELS_PER_PRIO * FDB_TC_MAX_PRIO * chains;
+ maj_prio = fs_create_prio_chained(&steering->fdb_root_ns->ns,
+ fs_prio,
+ levels);
+ if (IS_ERR(maj_prio))
+ return PTR_ERR(maj_prio);
+
+ for (chain = 0; chain < chains; chain++) {
+ err = create_fdb_sub_ns_prio_chain(steering, maj_prio);
+ if (err)
+ return err;
+ }
+
+ return 0;
+}
+
+static int create_fdb_fast_path(struct mlx5_flow_steering *steering)
+{
+ int err;
+
+ steering->fdb_sub_ns = kcalloc(FDB_NUM_CHAINS,
+ sizeof(*steering->fdb_sub_ns),
+ GFP_KERNEL);
+ if (!steering->fdb_sub_ns)
+ return -ENOMEM;
+
+ err = create_fdb_chains(steering, FDB_TC_OFFLOAD, FDB_TC_MAX_CHAIN + 1);
+ if (err)
+ return err;
+
+ err = create_fdb_chains(steering, FDB_FT_OFFLOAD, 1);
+ if (err)
+ return err;
+
+ return 0;
+}
+
+static int create_fdb_bypass(struct mlx5_flow_steering *steering)
+{
+ struct mlx5_flow_namespace *ns;
+ struct fs_prio *prio;
+ int i;
+
+ prio = fs_create_prio(&steering->fdb_root_ns->ns, FDB_BYPASS_PATH, 0);
+ if (IS_ERR(prio))
+ return PTR_ERR(prio);
+
+ ns = fs_create_namespace(prio, MLX5_FLOW_TABLE_MISS_ACTION_DEF);
+ if (IS_ERR(ns))
+ return PTR_ERR(ns);
+
+ for (i = 0; i < MLX5_BY_PASS_NUM_REGULAR_PRIOS; i++) {
+ prio = fs_create_prio(ns, i, 1);
+ if (IS_ERR(prio))
+ return PTR_ERR(prio);
+ }
+ return 0;
+}
+
+static void cleanup_fdb_root_ns(struct mlx5_flow_steering *steering)
+{
+ cleanup_root_ns(steering->fdb_root_ns);
+ steering->fdb_root_ns = NULL;
+ kfree(steering->fdb_sub_ns);
+ steering->fdb_sub_ns = NULL;
+}
+
+static int init_fdb_root_ns(struct mlx5_flow_steering *steering)
+{
+ struct fs_prio *maj_prio;
+ int err;
+
+ steering->fdb_root_ns = create_root_ns(steering, FS_FT_FDB);
+ if (!steering->fdb_root_ns)
+ return -ENOMEM;
+
+ err = create_fdb_bypass(steering);
+ if (err)
+ goto out_err;
+
+ err = create_fdb_fast_path(steering);
+ if (err)
+ goto out_err;
+
+ maj_prio = fs_create_prio(&steering->fdb_root_ns->ns, FDB_TC_MISS, 1);
+ if (IS_ERR(maj_prio)) {
+ err = PTR_ERR(maj_prio);
+ goto out_err;
+ }
+
+ maj_prio = fs_create_prio(&steering->fdb_root_ns->ns, FDB_BR_OFFLOAD, 4);
+ if (IS_ERR(maj_prio)) {
+ err = PTR_ERR(maj_prio);
+ goto out_err;
+ }
+
+ maj_prio = fs_create_prio(&steering->fdb_root_ns->ns, FDB_SLOW_PATH, 1);
+ if (IS_ERR(maj_prio)) {
+ err = PTR_ERR(maj_prio);
+ goto out_err;
+ }
+
+ /* We put this priority last, knowing that nothing will get here
+ * unless explicitly forwarded to. This is possible because the
+ * slow path tables have catch all rules and nothing gets passed
+ * those tables.
+ */
+ maj_prio = fs_create_prio(&steering->fdb_root_ns->ns, FDB_PER_VPORT, 1);
+ if (IS_ERR(maj_prio)) {
+ err = PTR_ERR(maj_prio);
+ goto out_err;
+ }
+
+ set_prio_attrs(steering->fdb_root_ns);
+ return 0;
+
+out_err:
+ cleanup_fdb_root_ns(steering);
+ return err;
+}
+
+static int init_egress_acl_root_ns(struct mlx5_flow_steering *steering, int vport)
+{
+ struct fs_prio *prio;
+
+ steering->esw_egress_root_ns[vport] = create_root_ns(steering, FS_FT_ESW_EGRESS_ACL);
+ if (!steering->esw_egress_root_ns[vport])
+ return -ENOMEM;
+
+ /* create 1 prio*/
+ prio = fs_create_prio(&steering->esw_egress_root_ns[vport]->ns, 0, 1);
+ return PTR_ERR_OR_ZERO(prio);
+}
+
+static int init_ingress_acl_root_ns(struct mlx5_flow_steering *steering, int vport)
+{
+ struct fs_prio *prio;
+
+ steering->esw_ingress_root_ns[vport] = create_root_ns(steering, FS_FT_ESW_INGRESS_ACL);
+ if (!steering->esw_ingress_root_ns[vport])
+ return -ENOMEM;
+
+ /* create 1 prio*/
+ prio = fs_create_prio(&steering->esw_ingress_root_ns[vport]->ns, 0, 1);
+ return PTR_ERR_OR_ZERO(prio);
+}
+
+int mlx5_fs_egress_acls_init(struct mlx5_core_dev *dev, int total_vports)
+{
+ struct mlx5_flow_steering *steering = dev->priv.steering;
+ int err;
+ int i;
+
+ steering->esw_egress_root_ns =
+ kcalloc(total_vports,
+ sizeof(*steering->esw_egress_root_ns),
+ GFP_KERNEL);
+ if (!steering->esw_egress_root_ns)
+ return -ENOMEM;
+
+ for (i = 0; i < total_vports; i++) {
+ err = init_egress_acl_root_ns(steering, i);
+ if (err)
+ goto cleanup_root_ns;
+ }
+ steering->esw_egress_acl_vports = total_vports;
+ return 0;
+
+cleanup_root_ns:
+ for (i--; i >= 0; i--)
+ cleanup_root_ns(steering->esw_egress_root_ns[i]);
+ kfree(steering->esw_egress_root_ns);
+ steering->esw_egress_root_ns = NULL;
+ return err;
+}
+
+void mlx5_fs_egress_acls_cleanup(struct mlx5_core_dev *dev)
+{
+ struct mlx5_flow_steering *steering = dev->priv.steering;
+ int i;
+
+ if (!steering->esw_egress_root_ns)
+ return;
+
+ for (i = 0; i < steering->esw_egress_acl_vports; i++)
+ cleanup_root_ns(steering->esw_egress_root_ns[i]);
+
+ kfree(steering->esw_egress_root_ns);
+ steering->esw_egress_root_ns = NULL;
+}
+
+int mlx5_fs_ingress_acls_init(struct mlx5_core_dev *dev, int total_vports)
+{
+ struct mlx5_flow_steering *steering = dev->priv.steering;
+ int err;
+ int i;
+
+ steering->esw_ingress_root_ns =
+ kcalloc(total_vports,
+ sizeof(*steering->esw_ingress_root_ns),
+ GFP_KERNEL);
+ if (!steering->esw_ingress_root_ns)
+ return -ENOMEM;
+
+ for (i = 0; i < total_vports; i++) {
+ err = init_ingress_acl_root_ns(steering, i);
+ if (err)
+ goto cleanup_root_ns;
+ }
+ steering->esw_ingress_acl_vports = total_vports;
+ return 0;
+
+cleanup_root_ns:
+ for (i--; i >= 0; i--)
+ cleanup_root_ns(steering->esw_ingress_root_ns[i]);
+ kfree(steering->esw_ingress_root_ns);
+ steering->esw_ingress_root_ns = NULL;
+ return err;
+}
+
+void mlx5_fs_ingress_acls_cleanup(struct mlx5_core_dev *dev)
+{
+ struct mlx5_flow_steering *steering = dev->priv.steering;
+ int i;
+
+ if (!steering->esw_ingress_root_ns)
+ return;
+
+ for (i = 0; i < steering->esw_ingress_acl_vports; i++)
+ cleanup_root_ns(steering->esw_ingress_root_ns[i]);
+
+ kfree(steering->esw_ingress_root_ns);
+ steering->esw_ingress_root_ns = NULL;
+}
+
+u32 mlx5_fs_get_capabilities(struct mlx5_core_dev *dev, enum mlx5_flow_namespace_type type)
+{
+ struct mlx5_flow_root_namespace *root;
+ struct mlx5_flow_namespace *ns;
+
+ ns = mlx5_get_flow_namespace(dev, type);
+ if (!ns)
+ return 0;
+
+ root = find_root(&ns->node);
+ if (!root)
+ return 0;
+
+ return root->cmds->get_capabilities(root, root->table_type);
+}
+
+static int init_egress_root_ns(struct mlx5_flow_steering *steering)
+{
+ int err;
+
+ steering->egress_root_ns = create_root_ns(steering,
+ FS_FT_NIC_TX);
+ if (!steering->egress_root_ns)
+ return -ENOMEM;
+
+ err = init_root_tree(steering, &egress_root_fs,
+ &steering->egress_root_ns->ns.node);
+ if (err)
+ goto cleanup;
+ set_prio_attrs(steering->egress_root_ns);
+ return 0;
+cleanup:
+ cleanup_root_ns(steering->egress_root_ns);
+ steering->egress_root_ns = NULL;
+ return err;
+}
+
+void mlx5_fs_core_cleanup(struct mlx5_core_dev *dev)
+{
+ struct mlx5_flow_steering *steering = dev->priv.steering;
+
+ cleanup_root_ns(steering->root_ns);
+ cleanup_fdb_root_ns(steering);
+ cleanup_root_ns(steering->port_sel_root_ns);
+ cleanup_root_ns(steering->sniffer_rx_root_ns);
+ cleanup_root_ns(steering->sniffer_tx_root_ns);
+ cleanup_root_ns(steering->rdma_rx_root_ns);
+ cleanup_root_ns(steering->rdma_tx_root_ns);
+ cleanup_root_ns(steering->egress_root_ns);
+}
+
+int mlx5_fs_core_init(struct mlx5_core_dev *dev)
+{
+ struct mlx5_flow_steering *steering = dev->priv.steering;
+ int err;
+
+ if ((((MLX5_CAP_GEN(dev, port_type) == MLX5_CAP_PORT_TYPE_ETH) &&
+ (MLX5_CAP_GEN(dev, nic_flow_table))) ||
+ ((MLX5_CAP_GEN(dev, port_type) == MLX5_CAP_PORT_TYPE_IB) &&
+ MLX5_CAP_GEN(dev, ipoib_enhanced_offloads))) &&
+ MLX5_CAP_FLOWTABLE_NIC_RX(dev, ft_support)) {
+ err = init_root_ns(steering);
+ if (err)
+ goto err;
+ }
+
+ if (MLX5_ESWITCH_MANAGER(dev)) {
+ if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev, ft_support)) {
+ err = init_fdb_root_ns(steering);
+ if (err)
+ goto err;
+ }
+ }
+
+ if (MLX5_CAP_FLOWTABLE_SNIFFER_RX(dev, ft_support)) {
+ err = init_sniffer_rx_root_ns(steering);
+ if (err)
+ goto err;
+ }
+
+ if (MLX5_CAP_FLOWTABLE_SNIFFER_TX(dev, ft_support)) {
+ err = init_sniffer_tx_root_ns(steering);
+ if (err)
+ goto err;
+ }
+
+ if (MLX5_CAP_FLOWTABLE_PORT_SELECTION(dev, ft_support)) {
+ err = init_port_sel_root_ns(steering);
+ if (err)
+ goto err;
+ }
+
+ if (MLX5_CAP_FLOWTABLE_RDMA_RX(dev, ft_support) &&
+ MLX5_CAP_FLOWTABLE_RDMA_RX(dev, table_miss_action_domain)) {
+ err = init_rdma_rx_root_ns(steering);
+ if (err)
+ goto err;
+ }
+
+ if (MLX5_CAP_FLOWTABLE_RDMA_TX(dev, ft_support)) {
+ err = init_rdma_tx_root_ns(steering);
+ if (err)
+ goto err;
+ }
+
+ if (MLX5_CAP_FLOWTABLE_NIC_TX(dev, ft_support)) {
+ err = init_egress_root_ns(steering);
+ if (err)
+ goto err;
+ }
+
+ return 0;
+
+err:
+ mlx5_fs_core_cleanup(dev);
+ return err;
+}
+
+void mlx5_fs_core_free(struct mlx5_core_dev *dev)
+{
+ struct mlx5_flow_steering *steering = dev->priv.steering;
+
+ kmem_cache_destroy(steering->ftes_cache);
+ kmem_cache_destroy(steering->fgs_cache);
+ kfree(steering);
+ mlx5_ft_pool_destroy(dev);
+ mlx5_cleanup_fc_stats(dev);
+}
+
+int mlx5_fs_core_alloc(struct mlx5_core_dev *dev)
+{
+ struct mlx5_flow_steering *steering;
+ int err = 0;
+
+ err = mlx5_init_fc_stats(dev);
+ if (err)
+ return err;
+
+ err = mlx5_ft_pool_init(dev);
+ if (err)
+ goto err;
+
+ steering = kzalloc(sizeof(*steering), GFP_KERNEL);
+ if (!steering) {
+ err = -ENOMEM;
+ goto err;
+ }
+
+ steering->dev = dev;
+ dev->priv.steering = steering;
+
+ steering->mode = MLX5_FLOW_STEERING_MODE_DMFS;
+
+ steering->fgs_cache = kmem_cache_create("mlx5_fs_fgs",
+ sizeof(struct mlx5_flow_group), 0,
+ 0, NULL);
+ steering->ftes_cache = kmem_cache_create("mlx5_fs_ftes", sizeof(struct fs_fte), 0,
+ 0, NULL);
+ if (!steering->ftes_cache || !steering->fgs_cache) {
+ err = -ENOMEM;
+ goto err;
+ }
+
+ return 0;
+
+err:
+ mlx5_fs_core_free(dev);
+ return err;
+}
+
+int mlx5_fs_add_rx_underlay_qpn(struct mlx5_core_dev *dev, u32 underlay_qpn)
+{
+ struct mlx5_flow_root_namespace *root = dev->priv.steering->root_ns;
+ struct mlx5_ft_underlay_qp *new_uqp;
+ int err = 0;
+
+ new_uqp = kzalloc(sizeof(*new_uqp), GFP_KERNEL);
+ if (!new_uqp)
+ return -ENOMEM;
+
+ mutex_lock(&root->chain_lock);
+
+ if (!root->root_ft) {
+ err = -EINVAL;
+ goto update_ft_fail;
+ }
+
+ err = root->cmds->update_root_ft(root, root->root_ft, underlay_qpn,
+ false);
+ if (err) {
+ mlx5_core_warn(dev, "Failed adding underlay QPN (%u) to root FT err(%d)\n",
+ underlay_qpn, err);
+ goto update_ft_fail;
+ }
+
+ new_uqp->qpn = underlay_qpn;
+ list_add_tail(&new_uqp->list, &root->underlay_qpns);
+
+ mutex_unlock(&root->chain_lock);
+
+ return 0;
+
+update_ft_fail:
+ mutex_unlock(&root->chain_lock);
+ kfree(new_uqp);
+ return err;
+}
+EXPORT_SYMBOL(mlx5_fs_add_rx_underlay_qpn);
+
+int mlx5_fs_remove_rx_underlay_qpn(struct mlx5_core_dev *dev, u32 underlay_qpn)
+{
+ struct mlx5_flow_root_namespace *root = dev->priv.steering->root_ns;
+ struct mlx5_ft_underlay_qp *uqp;
+ bool found = false;
+ int err = 0;
+
+ mutex_lock(&root->chain_lock);
+ list_for_each_entry(uqp, &root->underlay_qpns, list) {
+ if (uqp->qpn == underlay_qpn) {
+ found = true;
+ break;
+ }
+ }
+
+ if (!found) {
+ mlx5_core_warn(dev, "Failed finding underlay qp (%u) in qpn list\n",
+ underlay_qpn);
+ err = -EINVAL;
+ goto out;
+ }
+
+ err = root->cmds->update_root_ft(root, root->root_ft, underlay_qpn,
+ true);
+ if (err)
+ mlx5_core_warn(dev, "Failed removing underlay QPN (%u) from root FT err(%d)\n",
+ underlay_qpn, err);
+
+ list_del(&uqp->list);
+ mutex_unlock(&root->chain_lock);
+ kfree(uqp);
+
+ return 0;
+
+out:
+ mutex_unlock(&root->chain_lock);
+ return err;
+}
+EXPORT_SYMBOL(mlx5_fs_remove_rx_underlay_qpn);
+
+static struct mlx5_flow_root_namespace
+*get_root_namespace(struct mlx5_core_dev *dev, enum mlx5_flow_namespace_type ns_type)
+{
+ struct mlx5_flow_namespace *ns;
+
+ if (ns_type == MLX5_FLOW_NAMESPACE_ESW_EGRESS ||
+ ns_type == MLX5_FLOW_NAMESPACE_ESW_INGRESS)
+ ns = mlx5_get_flow_vport_acl_namespace(dev, ns_type, 0);
+ else
+ ns = mlx5_get_flow_namespace(dev, ns_type);
+ if (!ns)
+ return NULL;
+
+ return find_root(&ns->node);
+}
+
+struct mlx5_modify_hdr *mlx5_modify_header_alloc(struct mlx5_core_dev *dev,
+ enum mlx5_flow_namespace_type ns_type,
+ u8 num_actions,
+ void *modify_actions)
+{
+ struct mlx5_flow_root_namespace *root;
+ struct mlx5_modify_hdr *modify_hdr;
+ int err;
+
+ root = get_root_namespace(dev, ns_type);
+ if (!root)
+ return ERR_PTR(-EOPNOTSUPP);
+
+ modify_hdr = kzalloc(sizeof(*modify_hdr), GFP_KERNEL);
+ if (!modify_hdr)
+ return ERR_PTR(-ENOMEM);
+
+ modify_hdr->ns_type = ns_type;
+ err = root->cmds->modify_header_alloc(root, ns_type, num_actions,
+ modify_actions, modify_hdr);
+ if (err) {
+ kfree(modify_hdr);
+ return ERR_PTR(err);
+ }
+
+ return modify_hdr;
+}
+EXPORT_SYMBOL(mlx5_modify_header_alloc);
+
+void mlx5_modify_header_dealloc(struct mlx5_core_dev *dev,
+ struct mlx5_modify_hdr *modify_hdr)
+{
+ struct mlx5_flow_root_namespace *root;
+
+ root = get_root_namespace(dev, modify_hdr->ns_type);
+ if (WARN_ON(!root))
+ return;
+ root->cmds->modify_header_dealloc(root, modify_hdr);
+ kfree(modify_hdr);
+}
+EXPORT_SYMBOL(mlx5_modify_header_dealloc);
+
+struct mlx5_pkt_reformat *mlx5_packet_reformat_alloc(struct mlx5_core_dev *dev,
+ struct mlx5_pkt_reformat_params *params,
+ enum mlx5_flow_namespace_type ns_type)
+{
+ struct mlx5_pkt_reformat *pkt_reformat;
+ struct mlx5_flow_root_namespace *root;
+ int err;
+
+ root = get_root_namespace(dev, ns_type);
+ if (!root)
+ return ERR_PTR(-EOPNOTSUPP);
+
+ pkt_reformat = kzalloc(sizeof(*pkt_reformat), GFP_KERNEL);
+ if (!pkt_reformat)
+ return ERR_PTR(-ENOMEM);
+
+ pkt_reformat->ns_type = ns_type;
+ pkt_reformat->reformat_type = params->type;
+ err = root->cmds->packet_reformat_alloc(root, params, ns_type,
+ pkt_reformat);
+ if (err) {
+ kfree(pkt_reformat);
+ return ERR_PTR(err);
+ }
+
+ return pkt_reformat;
+}
+EXPORT_SYMBOL(mlx5_packet_reformat_alloc);
+
+void mlx5_packet_reformat_dealloc(struct mlx5_core_dev *dev,
+ struct mlx5_pkt_reformat *pkt_reformat)
+{
+ struct mlx5_flow_root_namespace *root;
+
+ root = get_root_namespace(dev, pkt_reformat->ns_type);
+ if (WARN_ON(!root))
+ return;
+ root->cmds->packet_reformat_dealloc(root, pkt_reformat);
+ kfree(pkt_reformat);
+}
+EXPORT_SYMBOL(mlx5_packet_reformat_dealloc);
+
+int mlx5_flow_namespace_set_peer(struct mlx5_flow_root_namespace *ns,
+ struct mlx5_flow_root_namespace *peer_ns)
+{
+ if (peer_ns && ns->mode != peer_ns->mode) {
+ mlx5_core_err(ns->dev,
+ "Can't peer namespace of different steering mode\n");
+ return -EINVAL;
+ }
+
+ return ns->cmds->set_peer(ns, peer_ns);
+}
+
+/* This function should be called only at init stage of the namespace.
+ * It is not safe to call this function while steering operations
+ * are executed in the namespace.
+ */
+int mlx5_flow_namespace_set_mode(struct mlx5_flow_namespace *ns,
+ enum mlx5_flow_steering_mode mode)
+{
+ struct mlx5_flow_root_namespace *root;
+ const struct mlx5_flow_cmds *cmds;
+ int err;
+
+ root = find_root(&ns->node);
+ if (&root->ns != ns)
+ /* Can't set cmds to non root namespace */
+ return -EINVAL;
+
+ if (root->table_type != FS_FT_FDB)
+ return -EOPNOTSUPP;
+
+ if (root->mode == mode)
+ return 0;
+
+ cmds = mlx5_fs_cmd_get_fw_cmds();
+ if (!cmds)
+ return -EOPNOTSUPP;
+
+ err = cmds->create_ns(root);
+ if (err) {
+ mlx5_core_err(root->dev, "Failed to create flow namespace (%d)\n",
+ err);
+ return err;
+ }
+
+ root->cmds->destroy_ns(root);
+ root->cmds = cmds;
+ root->mode = mode;
+
+ return 0;
+}
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fs_counters.c b/sys/dev/mlx5/mlx5_core/mlx5_fs_counters.c
--- a/sys/dev/mlx5/mlx5_core/mlx5_fs_counters.c
+++ b/sys/dev/mlx5/mlx5_core/mlx5_fs_counters.c
@@ -31,7 +31,7 @@
#include <linux/rbtree.h>
#include <dev/mlx5/mlx5_core/mlx5_core.h>
#include <dev/mlx5/mlx5_core/fs_core.h>
-#include <dev/mlx5/mlx5_core/mlx5_fc_cmd.h>
+#include <dev/mlx5/mlx5_core/fs_cmd.h>
#define MLX5_FC_STATS_PERIOD msecs_to_jiffies(1000)
#define MLX5_FC_BULK_QUERY_ALLOC_PERIOD msecs_to_jiffies(180 * 1000)
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fs_ft_pool.c b/sys/dev/mlx5/mlx5_core/mlx5_fs_ft_pool.c
new file mode 100644
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_core/mlx5_fs_ft_pool.c
@@ -0,0 +1,85 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+/* Copyright (c) 2021 Mellanox Technologies. */
+
+#include "fs_ft_pool.h"
+
+/* Firmware currently has 4 pool of 4 sizes that it supports (FT_POOLS),
+ * and a virtual memory region of 16M (MLX5_FT_SIZE), this region is duplicated
+ * for each flow table pool. We can allocate up to 16M of each pool,
+ * and we keep track of how much we used via mlx5_ft_pool_get_avail_sz.
+ * Firmware doesn't report any of this for now.
+ * ESW_POOL is expected to be sorted from large to small and match firmware
+ * pools.
+ */
+#define FT_SIZE (16 * 1024 * 1024)
+static const unsigned int FT_POOLS[] = { 4 * 1024 * 1024,
+ 1 * 1024 * 1024,
+ 64 * 1024,
+ 128,
+ 1 /* size for termination tables */ };
+struct mlx5_ft_pool {
+ int ft_left[ARRAY_SIZE(FT_POOLS)];
+};
+
+int mlx5_ft_pool_init(struct mlx5_core_dev *dev)
+{
+ struct mlx5_ft_pool *ft_pool;
+ int i;
+
+ ft_pool = kzalloc(sizeof(*ft_pool), GFP_KERNEL);
+ if (!ft_pool)
+ return -ENOMEM;
+
+ for (i = ARRAY_SIZE(FT_POOLS) - 1; i >= 0; i--)
+ ft_pool->ft_left[i] = FT_SIZE / FT_POOLS[i];
+
+ dev->priv.ft_pool = ft_pool;
+ return 0;
+}
+
+void mlx5_ft_pool_destroy(struct mlx5_core_dev *dev)
+{
+ kfree(dev->priv.ft_pool);
+}
+
+int
+mlx5_ft_pool_get_avail_sz(struct mlx5_core_dev *dev, enum fs_flow_table_type table_type,
+ int desired_size)
+{
+ u32 max_ft_size = 1 << MLX5_CAP_FLOWTABLE_TYPE(dev, log_max_ft_size, table_type);
+ int i, found_i = -1;
+
+ for (i = ARRAY_SIZE(FT_POOLS) - 1; i >= 0; i--) {
+ if (dev->priv.ft_pool->ft_left[i] && FT_POOLS[i] >= desired_size &&
+ FT_POOLS[i] <= max_ft_size) {
+ found_i = i;
+ if (desired_size != POOL_NEXT_SIZE)
+ break;
+ }
+ }
+
+ if (found_i != -1) {
+ --dev->priv.ft_pool->ft_left[found_i];
+ return FT_POOLS[found_i];
+ }
+
+ return 0;
+}
+
+void
+mlx5_ft_pool_put_sz(struct mlx5_core_dev *dev, int sz)
+{
+ int i;
+
+ if (!sz)
+ return;
+
+ for (i = ARRAY_SIZE(FT_POOLS) - 1; i >= 0; i--) {
+ if (sz == FT_POOLS[i]) {
+ ++dev->priv.ft_pool->ft_left[i];
+ return;
+ }
+ }
+
+ WARN_ONCE(1, "Couldn't find size %d in flow table size pool", sz);
+}
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fs_tcp.c b/sys/dev/mlx5/mlx5_core/mlx5_fs_tcp.c
--- a/sys/dev/mlx5/mlx5_core/mlx5_fs_tcp.c
+++ b/sys/dev/mlx5/mlx5_core/mlx5_fs_tcp.c
@@ -81,12 +81,12 @@
#endif
void
-mlx5e_accel_fs_del_inpcb(struct mlx5_flow_rule *rule)
+mlx5e_accel_fs_del_inpcb(struct mlx5_flow_handle *rule)
{
- mlx5_del_flow_rule(&rule);
+ mlx5_del_flow_rules(&rule);
}
-struct mlx5_flow_rule *
+struct mlx5_flow_handle *
mlx5e_accel_fs_add_inpcb(struct mlx5e_priv *priv,
struct inpcb *inp, uint32_t tirn, uint32_t flow_tag,
uint16_t vlan_id)
@@ -96,18 +96,17 @@
#if defined(INET) || defined(INET6)
struct mlx5e_accel_fs_tcp *fs_tcp = &priv->fts.accel_tcp;
#endif
- struct mlx5_flow_rule *flow;
+ struct mlx5_flow_handle *flow;
struct mlx5_flow_spec *spec;
- struct mlx5_flow_act flow_act = {
- .actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG,
- .flow_tag = flow_tag,
- };
+ struct mlx5_flow_act flow_act = {};
spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
if (!spec)
return (ERR_PTR(-ENOMEM));
spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
+ spec->flow_context.flags = FLOW_CONTEXT_HAS_TAG;
+ spec->flow_context.flow_tag = flow_tag;
INP_RLOCK(inp);
/* Set VLAN ID to match, if any. */
@@ -160,13 +159,9 @@
dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
dest.tir_num = tirn;
+ flow_act.action = MLX5_FLOW_RULE_FWD_ACTION_DEST;
- flow = mlx5_add_flow_rule(ft->t, spec->match_criteria_enable,
- spec->match_criteria,
- spec->match_value,
- MLX5_FLOW_RULE_FWD_ACTION_DEST,
- &flow_act,
- &dest);
+ flow = mlx5_add_flow_rules(ft->t, spec, &flow_act, &dest, 1);
out:
kvfree(spec);
return (flow);
@@ -175,18 +170,18 @@
static int
accel_fs_tcp_add_default_rule(struct mlx5e_priv *priv, int type)
{
- static u32 match_criteria[MLX5_ST_SZ_DW(fte_match_param)];
- static u32 match_value[MLX5_ST_SZ_DW(fte_match_param)];
+ static struct mlx5_flow_spec spec = {};
struct mlx5_flow_destination dest = {};
struct mlx5e_accel_fs_tcp *fs_tcp;
- struct mlx5_flow_rule *rule;
+ struct mlx5_flow_handle *rule;
struct mlx5_flow_act flow_act = {
- .actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG,
- .flow_tag = MLX5_FS_DEFAULT_FLOW_TAG,
+ .action = MLX5_FLOW_RULE_FWD_ACTION_DEST,
};
fs_tcp = &priv->fts.accel_tcp;
+ spec.flow_context.flags = FLOW_CONTEXT_HAS_TAG;
+ spec.flow_context.flow_tag = MLX5_FS_DEFAULT_FLOW_TAG;
dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
/*
@@ -197,10 +192,11 @@
* of flow tables.
*/
dest.ft = (type == MLX5E_ACCEL_FS_TCP_NUM_TYPES - 1) ?
- priv->fts.vlan.t : fs_tcp->tables[type + 1].t;
+ ((priv->fts.ipsec_ft) ? priv->fts.ipsec_ft : priv->fts.vlan.t) :
+ fs_tcp->tables[type + 1].t;
- rule = mlx5_add_flow_rule(fs_tcp->tables[type].t, 0, match_criteria, match_value,
- MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
+ rule = mlx5_add_flow_rules(fs_tcp->tables[type].t, &spec, &flow_act,
+ &dest, 1);
if (IS_ERR(rule))
return (PTR_ERR(rule));
@@ -317,11 +313,13 @@
accel_fs_tcp_create_table(struct mlx5e_priv *priv, int type)
{
struct mlx5e_flow_table *ft = &priv->fts.accel_tcp.tables[type];
+ struct mlx5_flow_table_attr ft_attr = {};
int err;
ft->num_groups = 0;
- ft->t = mlx5_create_flow_table(priv->fts.accel_tcp.ns, 0, "tcp",
- MLX5E_ACCEL_FS_TCP_TABLE_SIZE);
+ ft_attr.max_fte = MLX5E_ACCEL_FS_TCP_TABLE_SIZE;
+ ft_attr.level = type;
+ ft->t = mlx5_create_flow_table(priv->fts.accel_tcp.ns, &ft_attr);
if (IS_ERR(ft->t)) {
err = PTR_ERR(ft->t);
ft->t = NULL;
@@ -365,7 +363,7 @@
return;
for (i = 0; i < MLX5E_ACCEL_FS_TCP_NUM_TYPES; i++) {
- mlx5_del_flow_rule(&priv->fts.accel_tcp.default_rules[i]);
+ mlx5_del_flow_rules(&priv->fts.accel_tcp.default_rules[i]);
accel_fs_tcp_destroy_table(priv, i);
}
}
@@ -402,7 +400,7 @@
err_destroy_rules:
while (i--)
- mlx5_del_flow_rule(&priv->fts.accel_tcp.default_rules[i]);
+ mlx5_del_flow_rules(&priv->fts.accel_tcp.default_rules[i]);
i = MLX5E_ACCEL_FS_TCP_NUM_TYPES;
err_destroy_tables:
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fs_tree.c b/sys/dev/mlx5/mlx5_core/mlx5_fs_tree.c
deleted file mode 100644
--- a/sys/dev/mlx5/mlx5_core/mlx5_fs_tree.c
+++ /dev/null
@@ -1,2871 +0,0 @@
-/*-
- * Copyright (c) 2013-2021, Mellanox Technologies, Ltd. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * $FreeBSD$
- */
-
-#include "opt_rss.h"
-
-#include <linux/module.h>
-#include <dev/mlx5/driver.h>
-#include <dev/mlx5/mlx5_core/mlx5_core.h>
-#include <dev/mlx5/mlx5_core/fs_core.h>
-#include <linux/string.h>
-#include <linux/compiler.h>
-
-#define INIT_TREE_NODE_ARRAY_SIZE(...) (sizeof((struct init_tree_node[]){__VA_ARGS__}) /\
- sizeof(struct init_tree_node))
-
-#define ADD_PRIO(name_val, flags_val, min_level_val, max_ft_val, caps_val, \
- ...) {.type = FS_TYPE_PRIO,\
- .name = name_val,\
- .min_ft_level = min_level_val,\
- .flags = flags_val,\
- .max_ft = max_ft_val,\
- .caps = caps_val,\
- .children = (struct init_tree_node[]) {__VA_ARGS__},\
- .ar_size = INIT_TREE_NODE_ARRAY_SIZE(__VA_ARGS__) \
-}
-
-#define ADD_FT_PRIO(name_val, flags_val, max_ft_val, ...)\
- ADD_PRIO(name_val, flags_val, 0, max_ft_val, {},\
- __VA_ARGS__)\
-
-#define ADD_NS(name_val, ...) {.type = FS_TYPE_NAMESPACE,\
- .name = name_val,\
- .children = (struct init_tree_node[]) {__VA_ARGS__},\
- .ar_size = INIT_TREE_NODE_ARRAY_SIZE(__VA_ARGS__) \
-}
-
-#define INIT_CAPS_ARRAY_SIZE(...) (sizeof((long[]){__VA_ARGS__}) /\
- sizeof(long))
-
-#define FS_CAP(cap) (__mlx5_bit_off(flow_table_nic_cap, cap))
-
-#define FS_REQUIRED_CAPS(...) {.arr_sz = INIT_CAPS_ARRAY_SIZE(__VA_ARGS__), \
- .caps = (long[]) {__VA_ARGS__}}
-
-/* Flowtable sizes: */
-#define BYPASS_MAX_FT 5
-#define BYPASS_PRIO_MAX_FT 1
-#define OFFLOADS_MAX_FT 2
-#define KERNEL_MAX_FT 5
-#define LEFTOVER_MAX_FT 1
-
-/* Flowtable levels: */
-#define OFFLOADS_MIN_LEVEL 3
-#define KERNEL_MIN_LEVEL (OFFLOADS_MIN_LEVEL + 1)
-#define LEFTOVER_MIN_LEVEL (KERNEL_MIN_LEVEL + 1)
-#define BYPASS_MIN_LEVEL (MLX5_NUM_BYPASS_FTS + LEFTOVER_MIN_LEVEL)
-
-struct node_caps {
- size_t arr_sz;
- long *caps;
-};
-
-struct init_tree_node {
- enum fs_type type;
- const char *name;
- struct init_tree_node *children;
- int ar_size;
- struct node_caps caps;
- u8 flags;
- int min_ft_level;
- int prio;
- int max_ft;
-} root_fs = {
- .type = FS_TYPE_NAMESPACE,
- .name = "root",
- .ar_size = 4,
- .children = (struct init_tree_node[]) {
- ADD_PRIO("by_pass_prio", 0, BYPASS_MIN_LEVEL, 0,
- FS_REQUIRED_CAPS(FS_CAP(flow_table_properties_nic_receive.flow_modify_en),
- FS_CAP(flow_table_properties_nic_receive.modify_root)),
- ADD_NS("by_pass_ns",
- ADD_FT_PRIO("prio0", 0,
- BYPASS_PRIO_MAX_FT),
- ADD_FT_PRIO("prio1", 0,
- BYPASS_PRIO_MAX_FT),
- ADD_FT_PRIO("prio2", 0,
- BYPASS_PRIO_MAX_FT),
- ADD_FT_PRIO("prio3", 0,
- BYPASS_PRIO_MAX_FT),
- ADD_FT_PRIO("prio4", 0,
- BYPASS_PRIO_MAX_FT),
- ADD_FT_PRIO("prio5", 0,
- BYPASS_PRIO_MAX_FT),
- ADD_FT_PRIO("prio6", 0,
- BYPASS_PRIO_MAX_FT),
- ADD_FT_PRIO("prio7", 0,
- BYPASS_PRIO_MAX_FT),
- ADD_FT_PRIO("prio-mcast", 0,
- BYPASS_PRIO_MAX_FT))),
- ADD_PRIO("offloads_prio", 0, OFFLOADS_MIN_LEVEL, 0, {},
- ADD_NS("offloads_ns",
- ADD_FT_PRIO("prio_offloads-0", 0,
- OFFLOADS_MAX_FT))),
- ADD_PRIO("kernel_prio", 0, KERNEL_MIN_LEVEL, 0, {},
- ADD_NS("kernel_ns",
- ADD_FT_PRIO("prio_kernel-0", 0,
- KERNEL_MAX_FT))),
- ADD_PRIO("leftovers_prio", MLX5_CORE_FS_PRIO_SHARED,
- LEFTOVER_MIN_LEVEL, 0,
- FS_REQUIRED_CAPS(FS_CAP(flow_table_properties_nic_receive.flow_modify_en),
- FS_CAP(flow_table_properties_nic_receive.modify_root)),
- ADD_NS("leftover_ns",
- ADD_FT_PRIO("leftovers_prio-0",
- MLX5_CORE_FS_PRIO_SHARED,
- LEFTOVER_MAX_FT)))
- }
-};
-
-/* Tree creation functions */
-
-static struct mlx5_flow_root_namespace *find_root(struct fs_base *node)
-{
- struct fs_base *parent;
-
- /* Make sure we only read it once while we go up the tree */
- while ((parent = node->parent))
- node = parent;
-
- if (node->type != FS_TYPE_NAMESPACE) {
- return NULL;
- }
-
- return container_of(container_of(node,
- struct mlx5_flow_namespace,
- base),
- struct mlx5_flow_root_namespace,
- ns);
-}
-
-static inline struct mlx5_core_dev *fs_get_dev(struct fs_base *node)
-{
- struct mlx5_flow_root_namespace *root = find_root(node);
-
- if (root)
- return root->dev;
- return NULL;
-}
-
-static void fs_init_node(struct fs_base *node,
- unsigned int refcount)
-{
- kref_init(&node->refcount);
- atomic_set(&node->users_refcount, refcount);
- init_completion(&node->complete);
- INIT_LIST_HEAD(&node->list);
- mutex_init(&node->lock);
-}
-
-static void _fs_add_node(struct fs_base *node,
- const char *name,
- struct fs_base *parent)
-{
- if (parent)
- atomic_inc(&parent->users_refcount);
- node->name = kstrdup_const(name, GFP_KERNEL);
- node->parent = parent;
-}
-
-static void fs_add_node(struct fs_base *node,
- struct fs_base *parent, const char *name,
- unsigned int refcount)
-{
- fs_init_node(node, refcount);
- _fs_add_node(node, name, parent);
-}
-
-static void _fs_put(struct fs_base *node, void (*kref_cb)(struct kref *kref),
- bool parent_locked);
-
-static void fs_del_dst(struct mlx5_flow_rule *dst);
-static void _fs_del_ft(struct mlx5_flow_table *ft);
-static void fs_del_fg(struct mlx5_flow_group *fg);
-static void fs_del_fte(struct fs_fte *fte);
-
-static void cmd_remove_node(struct fs_base *base)
-{
- switch (base->type) {
- case FS_TYPE_FLOW_DEST:
- fs_del_dst(container_of(base, struct mlx5_flow_rule, base));
- break;
- case FS_TYPE_FLOW_TABLE:
- _fs_del_ft(container_of(base, struct mlx5_flow_table, base));
- break;
- case FS_TYPE_FLOW_GROUP:
- fs_del_fg(container_of(base, struct mlx5_flow_group, base));
- break;
- case FS_TYPE_FLOW_ENTRY:
- fs_del_fte(container_of(base, struct fs_fte, base));
- break;
- default:
- break;
- }
-}
-
-static void __fs_remove_node(struct kref *kref)
-{
- struct fs_base *node = container_of(kref, struct fs_base, refcount);
-
- if (node->parent)
- mutex_lock(&node->parent->lock);
- mutex_lock(&node->lock);
- cmd_remove_node(node);
- mutex_unlock(&node->lock);
- complete(&node->complete);
- if (node->parent) {
- mutex_unlock(&node->parent->lock);
- _fs_put(node->parent, _fs_remove_node, false);
- }
-}
-
-void _fs_remove_node(struct kref *kref)
-{
- struct fs_base *node = container_of(kref, struct fs_base, refcount);
-
- __fs_remove_node(kref);
- kfree_const(node->name);
- kfree(node);
-}
-
-static void fs_get(struct fs_base *node)
-{
- atomic_inc(&node->users_refcount);
-}
-
-static void _fs_put(struct fs_base *node, void (*kref_cb)(struct kref *kref),
- bool parent_locked)
-{
- struct fs_base *parent_node = node->parent;
-
- if (parent_node && !parent_locked)
- mutex_lock(&parent_node->lock);
- if (atomic_dec_and_test(&node->users_refcount)) {
- if (parent_node) {
- /*remove from parent's list*/
- list_del_init(&node->list);
- mutex_unlock(&parent_node->lock);
- }
- kref_put(&node->refcount, kref_cb);
- if (parent_node && parent_locked)
- mutex_lock(&parent_node->lock);
- } else if (parent_node && !parent_locked) {
- mutex_unlock(&parent_node->lock);
- }
-}
-
-static void fs_put(struct fs_base *node)
-{
- _fs_put(node, __fs_remove_node, false);
-}
-
-static void fs_put_parent_locked(struct fs_base *node)
-{
- _fs_put(node, __fs_remove_node, true);
-}
-
-static void fs_remove_node(struct fs_base *node)
-{
- fs_put(node);
- wait_for_completion(&node->complete);
- kfree_const(node->name);
- kfree(node);
-}
-
-static void fs_remove_node_parent_locked(struct fs_base *node)
-{
- fs_put_parent_locked(node);
- wait_for_completion(&node->complete);
- kfree_const(node->name);
- kfree(node);
-}
-
-static struct fs_fte *fs_alloc_fte(u32 sw_action,
- struct mlx5_flow_act *flow_act,
- u32 *match_value,
- unsigned int index)
-{
- struct fs_fte *fte;
-
-
- fte = kzalloc(sizeof(*fte), GFP_KERNEL);
- if (!fte)
- return ERR_PTR(-ENOMEM);
-
- memcpy(fte->val, match_value, sizeof(fte->val));
- fte->base.type = FS_TYPE_FLOW_ENTRY;
- fte->dests_size = 0;
- fte->index = index;
- INIT_LIST_HEAD(&fte->dests);
- fte->flow_act = *flow_act;
- fte->sw_action = sw_action;
-
- return fte;
-}
-
-static struct fs_fte *alloc_star_ft_entry(struct mlx5_flow_table *ft,
- struct mlx5_flow_group *fg,
- u32 *match_value,
- unsigned int index)
-{
- int err;
- struct fs_fte *fte;
- struct mlx5_flow_rule *dst;
- struct mlx5_flow_act flow_act = {
- .actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG,
- .flow_tag = MLX5_FS_DEFAULT_FLOW_TAG,
- };
-
- if (fg->num_ftes == fg->max_ftes)
- return ERR_PTR(-ENOSPC);
-
- fte = fs_alloc_fte(MLX5_FLOW_RULE_FWD_ACTION_DEST,
- &flow_act, match_value, index);
- if (IS_ERR(fte))
- return fte;
-
- /*create dst*/
- dst = kzalloc(sizeof(*dst), GFP_KERNEL);
- if (!dst) {
- err = -ENOMEM;
- goto free_fte;
- }
-
- fte->base.parent = &fg->base;
- fte->dests_size = 1;
- dst->dest_attr.type = MLX5_FLOW_CONTEXT_DEST_TYPE_FLOW_TABLE;
- dst->base.parent = &fte->base;
- list_add(&dst->base.list, &fte->dests);
- /* assumed that the callee creates the star rules sorted by index */
- list_add_tail(&fte->base.list, &fg->ftes);
- fg->num_ftes++;
-
- return fte;
-
-free_fte:
- kfree(fte);
- return ERR_PTR(err);
-}
-
-/* assume that fte can't be changed */
-static void free_star_fte_entry(struct fs_fte *fte)
-{
- struct mlx5_flow_group *fg;
- struct mlx5_flow_rule *dst, *temp;
-
- fs_get_parent(fg, fte);
-
- list_for_each_entry_safe(dst, temp, &fte->dests, base.list) {
- fte->dests_size--;
- list_del(&dst->base.list);
- kfree(dst);
- }
-
- list_del(&fte->base.list);
- fg->num_ftes--;
- kfree(fte);
-}
-
-static struct mlx5_flow_group *fs_alloc_fg(u32 *create_fg_in)
-{
- struct mlx5_flow_group *fg;
- void *match_criteria = MLX5_ADDR_OF(create_flow_group_in,
- create_fg_in, match_criteria);
- u8 match_criteria_enable = MLX5_GET(create_flow_group_in,
- create_fg_in,
- match_criteria_enable);
- fg = kzalloc(sizeof(*fg), GFP_KERNEL);
- if (!fg)
- return ERR_PTR(-ENOMEM);
-
- INIT_LIST_HEAD(&fg->ftes);
- fg->mask.match_criteria_enable = match_criteria_enable;
- memcpy(&fg->mask.match_criteria, match_criteria,
- sizeof(fg->mask.match_criteria));
- fg->base.type = FS_TYPE_FLOW_GROUP;
- fg->start_index = MLX5_GET(create_flow_group_in, create_fg_in,
- start_flow_index);
- fg->max_ftes = MLX5_GET(create_flow_group_in, create_fg_in,
- end_flow_index) - fg->start_index + 1;
- return fg;
-}
-
-static struct mlx5_flow_table *find_next_ft(struct fs_prio *prio);
-static struct mlx5_flow_table *find_prev_ft(struct mlx5_flow_table *curr,
- struct fs_prio *prio);
-
-/* assumed src_ft and dst_ft can't be freed */
-static int fs_set_star_rule(struct mlx5_core_dev *dev,
- struct mlx5_flow_table *src_ft,
- struct mlx5_flow_table *dst_ft)
-{
- struct mlx5_flow_rule *src_dst;
- struct fs_fte *src_fte;
- int err = 0;
- u32 *match_value;
- int match_len = MLX5_ST_SZ_BYTES(fte_match_param);
-
- src_dst = list_first_entry(&src_ft->star_rule.fte->dests,
- struct mlx5_flow_rule, base.list);
- match_value = mlx5_vzalloc(match_len);
- if (!match_value) {
- mlx5_core_warn(dev, "failed to allocate inbox\n");
- return -ENOMEM;
- }
- /*Create match context*/
-
- fs_get_parent(src_fte, src_dst);
-
- src_dst->dest_attr.ft = dst_ft;
- if (dst_ft) {
- err = mlx5_cmd_fs_set_fte(dev,
- src_ft->vport,
- &src_fte->status,
- match_value, src_ft->type,
- src_ft->id, src_fte->index,
- src_ft->star_rule.fg->id,
- &src_fte->flow_act,
- src_fte->sw_action,
- src_fte->dests_size,
- &src_fte->dests);
- if (err)
- goto free;
-
- fs_get(&dst_ft->base);
- } else {
- mlx5_cmd_fs_delete_fte(dev,
- src_ft->vport,
- &src_fte->status,
- src_ft->type, src_ft->id,
- src_fte->index);
- }
-
-free:
- kvfree(match_value);
- return err;
-}
-
-static int connect_prev_fts(struct fs_prio *locked_prio,
- struct fs_prio *prev_prio,
- struct mlx5_flow_table *next_ft)
-{
- struct mlx5_flow_table *iter;
- int err = 0;
- struct mlx5_core_dev *dev = fs_get_dev(&prev_prio->base);
-
- if (!dev)
- return -ENODEV;
-
- mutex_lock(&prev_prio->base.lock);
- fs_for_each_ft(iter, prev_prio) {
- struct mlx5_flow_rule *src_dst =
- list_first_entry(&iter->star_rule.fte->dests,
- struct mlx5_flow_rule, base.list);
- struct mlx5_flow_table *prev_ft = src_dst->dest_attr.ft;
-
- if (prev_ft == next_ft)
- continue;
-
- err = fs_set_star_rule(dev, iter, next_ft);
- if (err) {
- mlx5_core_warn(dev,
- "mlx5: flow steering can't connect prev and next\n");
- goto unlock;
- } else {
- /* Assume ft's prio is locked */
- if (prev_ft) {
- struct fs_prio *prio;
-
- fs_get_parent(prio, prev_ft);
- if (prio == locked_prio)
- fs_put_parent_locked(&prev_ft->base);
- else
- fs_put(&prev_ft->base);
- }
- }
- }
-
-unlock:
- mutex_unlock(&prev_prio->base.lock);
- return 0;
-}
-
-static int create_star_rule(struct mlx5_flow_table *ft, struct fs_prio *prio)
-{
- struct mlx5_flow_group *fg;
- int err;
- u32 *fg_in;
- u32 *match_value;
- struct mlx5_flow_table *next_ft;
- struct mlx5_flow_table *prev_ft;
- struct mlx5_flow_root_namespace *root = find_root(&prio->base);
- int fg_inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
- int match_len = MLX5_ST_SZ_BYTES(fte_match_param);
-
- fg_in = mlx5_vzalloc(fg_inlen);
- if (!fg_in) {
- mlx5_core_warn(root->dev, "failed to allocate inbox\n");
- return -ENOMEM;
- }
-
- match_value = mlx5_vzalloc(match_len);
- if (!match_value) {
- mlx5_core_warn(root->dev, "failed to allocate inbox\n");
- kvfree(fg_in);
- return -ENOMEM;
- }
-
- MLX5_SET(create_flow_group_in, fg_in, start_flow_index, ft->max_fte);
- MLX5_SET(create_flow_group_in, fg_in, end_flow_index, ft->max_fte);
- fg = fs_alloc_fg(fg_in);
- if (IS_ERR(fg)) {
- err = PTR_ERR(fg);
- goto out;
- }
- ft->star_rule.fg = fg;
- err = mlx5_cmd_fs_create_fg(fs_get_dev(&prio->base),
- fg_in, ft->vport, ft->type,
- ft->id,
- &fg->id);
- if (err)
- goto free_fg;
-
- ft->star_rule.fte = alloc_star_ft_entry(ft, fg,
- match_value,
- ft->max_fte);
- if (IS_ERR(ft->star_rule.fte))
- goto free_star_rule;
-
- mutex_lock(&root->fs_chain_lock);
- next_ft = find_next_ft(prio);
- err = fs_set_star_rule(root->dev, ft, next_ft);
- if (err) {
- mutex_unlock(&root->fs_chain_lock);
- goto free_star_rule;
- }
- if (next_ft) {
- struct fs_prio *parent;
-
- fs_get_parent(parent, next_ft);
- fs_put(&next_ft->base);
- }
- prev_ft = find_prev_ft(ft, prio);
- if (prev_ft) {
- struct fs_prio *prev_parent;
-
- fs_get_parent(prev_parent, prev_ft);
-
- err = connect_prev_fts(NULL, prev_parent, ft);
- if (err) {
- mutex_unlock(&root->fs_chain_lock);
- goto destroy_chained_star_rule;
- }
- fs_put(&prev_ft->base);
- }
- mutex_unlock(&root->fs_chain_lock);
- kvfree(fg_in);
- kvfree(match_value);
-
- return 0;
-
-destroy_chained_star_rule:
- fs_set_star_rule(fs_get_dev(&prio->base), ft, NULL);
- if (next_ft)
- fs_put(&next_ft->base);
-free_star_rule:
- free_star_fte_entry(ft->star_rule.fte);
- mlx5_cmd_fs_destroy_fg(fs_get_dev(&ft->base), ft->vport,
- ft->type, ft->id,
- fg->id);
-free_fg:
- kfree(fg);
-out:
- kvfree(fg_in);
- kvfree(match_value);
- return err;
-}
-
-static void destroy_star_rule(struct mlx5_flow_table *ft, struct fs_prio *prio)
-{
- int err;
- struct mlx5_flow_root_namespace *root;
- struct mlx5_core_dev *dev = fs_get_dev(&prio->base);
- struct mlx5_flow_table *prev_ft, *next_ft;
- struct fs_prio *prev_prio;
-
- WARN_ON(!dev);
-
- root = find_root(&prio->base);
- if (!root)
- mlx5_core_err(dev,
- "flow steering failed to find root of priority %s",
- prio->base.name);
-
- /* In order to ensure atomic deletion, first update
- * prev ft to point on the next ft.
- */
- mutex_lock(&root->fs_chain_lock);
- prev_ft = find_prev_ft(ft, prio);
- next_ft = find_next_ft(prio);
- if (prev_ft) {
- fs_get_parent(prev_prio, prev_ft);
- /*Prev is connected to ft, only if ft is the first(last) in the prio*/
- err = connect_prev_fts(prio, prev_prio, next_ft);
- if (err)
- mlx5_core_warn(root->dev,
- "flow steering can't connect prev and next of flow table\n");
- fs_put(&prev_ft->base);
- }
-
- err = fs_set_star_rule(root->dev, ft, NULL);
- /*One put is for fs_get in find next ft*/
- if (next_ft) {
- fs_put(&next_ft->base);
- if (!err)
- fs_put(&next_ft->base);
- }
-
- mutex_unlock(&root->fs_chain_lock);
- err = mlx5_cmd_fs_destroy_fg(dev, ft->vport, ft->type, ft->id,
- ft->star_rule.fg->id);
- if (err)
- mlx5_core_warn(dev,
- "flow steering can't destroy star entry group(index:%d) of ft:%s\n", ft->star_rule.fg->start_index,
- ft->base.name);
- free_star_fte_entry(ft->star_rule.fte);
-
- kfree(ft->star_rule.fg);
- ft->star_rule.fg = NULL;
-}
-
-static struct fs_prio *find_prio(struct mlx5_flow_namespace *ns,
- unsigned int prio)
-{
- struct fs_prio *iter_prio;
-
- fs_for_each_prio(iter_prio, ns) {
- if (iter_prio->prio == prio)
- return iter_prio;
- }
-
- return NULL;
-}
-
-static unsigned int _alloc_new_level(struct fs_prio *prio,
- struct mlx5_flow_namespace *match);
-
-static unsigned int __alloc_new_level(struct mlx5_flow_namespace *ns,
- struct fs_prio *prio)
-{
- unsigned int level = 0;
- struct fs_prio *p;
-
- if (!ns)
- return 0;
-
- mutex_lock(&ns->base.lock);
- fs_for_each_prio(p, ns) {
- if (p != prio)
- level += p->max_ft;
- else
- break;
- }
- mutex_unlock(&ns->base.lock);
-
- fs_get_parent(prio, ns);
- if (prio)
- WARN_ON(prio->base.type != FS_TYPE_PRIO);
-
- return level + _alloc_new_level(prio, ns);
-}
-
-/* Called under lock of priority, hence locking all upper objects */
-static unsigned int _alloc_new_level(struct fs_prio *prio,
- struct mlx5_flow_namespace *match)
-{
- struct mlx5_flow_namespace *ns;
- struct fs_base *it;
- unsigned int level = 0;
-
- if (!prio)
- return 0;
-
- mutex_lock(&prio->base.lock);
- fs_for_each_ns_or_ft_reverse(it, prio) {
- if (it->type == FS_TYPE_NAMESPACE) {
- struct fs_prio *p;
-
- fs_get_obj(ns, it);
-
- if (match != ns) {
- mutex_lock(&ns->base.lock);
- fs_for_each_prio(p, ns)
- level += p->max_ft;
- mutex_unlock(&ns->base.lock);
- } else {
- break;
- }
- } else {
- struct mlx5_flow_table *ft;
-
- fs_get_obj(ft, it);
- mutex_unlock(&prio->base.lock);
- return level + ft->level + 1;
- }
- }
-
- fs_get_parent(ns, prio);
- mutex_unlock(&prio->base.lock);
- return __alloc_new_level(ns, prio) + level;
-}
-
-static unsigned int alloc_new_level(struct fs_prio *prio)
-{
- return _alloc_new_level(prio, NULL);
-}
-
-static int update_root_ft_create(struct mlx5_flow_root_namespace *root,
- struct mlx5_flow_table *ft)
-{
- int err = 0;
- int min_level = INT_MAX;
-
- if (root->root_ft)
- min_level = root->root_ft->level;
-
- if (ft->level < min_level)
- err = mlx5_cmd_update_root_ft(root->dev, ft->type,
- ft->id);
- else
- return err;
-
- if (err)
- mlx5_core_warn(root->dev, "Update root flow table of id=%u failed\n",
- ft->id);
- else
- root->root_ft = ft;
-
- return err;
-}
-
-static struct mlx5_flow_table *_create_ft_common(struct mlx5_flow_namespace *ns,
- u16 vport,
- struct fs_prio *fs_prio,
- int max_fte,
- const char *name)
-{
- struct mlx5_flow_table *ft;
- int err;
- int log_table_sz;
- int ft_size;
- char gen_name[20];
- struct mlx5_flow_root_namespace *root = find_root(&ns->base);
- struct mlx5_core_dev *dev = fs_get_dev(&ns->base);
-
- if (!root) {
- mlx5_core_err(dev,
- "flow steering failed to find root of namespace %s",
- ns->base.name);
- return ERR_PTR(-ENODEV);
- }
-
- if (fs_prio->num_ft == fs_prio->max_ft)
- return ERR_PTR(-ENOSPC);
-
- ft = kzalloc(sizeof(*ft), GFP_KERNEL);
- if (!ft)
- return ERR_PTR(-ENOMEM);
-
- fs_init_node(&ft->base, 1);
- INIT_LIST_HEAD(&ft->fgs);
-
- /* Temporarily WA until we expose the level set in the API */
- if (root->table_type == FS_FT_ESW_EGRESS_ACL ||
- root->table_type == FS_FT_ESW_INGRESS_ACL)
- ft->level = 0;
- else
- ft->level = alloc_new_level(fs_prio);
-
- ft->base.type = FS_TYPE_FLOW_TABLE;
- ft->vport = vport;
- ft->type = root->table_type;
- /*Two entries are reserved for star rules*/
- ft_size = roundup_pow_of_two(max_fte + 2);
- /*User isn't aware to those rules*/
- ft->max_fte = ft_size - 2;
- log_table_sz = ilog2(ft_size);
-
- if (name == NULL || name[0] == '\0') {
- snprintf(gen_name, sizeof(gen_name), "flow_table_%u", ft->id);
- name = gen_name;
- }
-
- err = mlx5_cmd_fs_create_ft(root->dev, ft->vport, ft->type,
- ft->level, log_table_sz, name, &ft->id);
- if (err)
- goto free_ft;
-
- err = create_star_rule(ft, fs_prio);
- if (err)
- goto del_ft;
-
- if ((root->table_type == FS_FT_NIC_RX) && MLX5_CAP_FLOWTABLE(root->dev,
- flow_table_properties_nic_receive.modify_root)) {
- err = update_root_ft_create(root, ft);
- if (err)
- goto destroy_star_rule;
- }
-
- _fs_add_node(&ft->base, name, &fs_prio->base);
-
- list_add_tail(&ft->base.list, &fs_prio->objs);
- fs_prio->num_ft++;
-
- return ft;
-
-destroy_star_rule:
- destroy_star_rule(ft, fs_prio);
-del_ft:
- mlx5_cmd_fs_destroy_ft(root->dev, ft->vport, ft->type, ft->id);
-free_ft:
- kfree(ft);
- return ERR_PTR(err);
-}
-
-static struct mlx5_flow_table *create_ft_common(struct mlx5_flow_namespace *ns,
- u16 vport,
- unsigned int prio,
- int max_fte,
- const char *name)
-{
- struct fs_prio *fs_prio = NULL;
- fs_prio = find_prio(ns, prio);
- if (!fs_prio)
- return ERR_PTR(-EINVAL);
-
- return _create_ft_common(ns, vport, fs_prio, max_fte, name);
-}
-
-
-static struct mlx5_flow_table *find_first_ft_in_ns(struct mlx5_flow_namespace *ns,
- struct list_head *start);
-
-static struct mlx5_flow_table *find_first_ft_in_prio(struct fs_prio *prio,
- struct list_head *start);
-
-static struct mlx5_flow_table *mlx5_create_autogrouped_shared_flow_table(struct fs_prio *fs_prio)
-{
- struct mlx5_flow_table *ft;
-
- ft = find_first_ft_in_prio(fs_prio, &fs_prio->objs);
- if (ft) {
- ft->shared_refcount++;
- return ft;
- }
-
- return NULL;
-}
-
-struct mlx5_flow_table *mlx5_create_auto_grouped_flow_table(struct mlx5_flow_namespace *ns,
- int prio,
- const char *name,
- int num_flow_table_entries,
- int max_num_groups,
- int num_reserved_entries)
-{
- struct mlx5_flow_table *ft = NULL;
- struct fs_prio *fs_prio;
- bool is_shared_prio;
-
- if (max_num_groups > (num_flow_table_entries - num_reserved_entries))
- return ERR_PTR(-EINVAL);
- if (num_reserved_entries > num_flow_table_entries)
- return ERR_PTR(-EINVAL);
-
- fs_prio = find_prio(ns, prio);
- if (!fs_prio)
- return ERR_PTR(-EINVAL);
-
- is_shared_prio = fs_prio->flags & MLX5_CORE_FS_PRIO_SHARED;
- if (is_shared_prio) {
- mutex_lock(&fs_prio->shared_lock);
- ft = mlx5_create_autogrouped_shared_flow_table(fs_prio);
- }
-
- if (ft)
- goto return_ft;
-
- ft = create_ft_common(ns, 0, prio, num_flow_table_entries,
- name);
- if (IS_ERR(ft))
- goto return_ft;
-
- ft->autogroup.active = true;
- ft->autogroup.max_types = max_num_groups;
- ft->autogroup.max_fte = num_flow_table_entries - num_reserved_entries;
- /* We save place for flow groups in addition to max types */
- ft->autogroup.group_size = ft->autogroup.max_fte / (max_num_groups + 1);
-
- if (is_shared_prio)
- ft->shared_refcount = 1;
-
-return_ft:
- if (is_shared_prio)
- mutex_unlock(&fs_prio->shared_lock);
- return ft;
-}
-EXPORT_SYMBOL(mlx5_create_auto_grouped_flow_table);
-
-struct mlx5_flow_table *mlx5_create_vport_flow_table(struct mlx5_flow_namespace *ns,
- u16 vport,
- int prio,
- const char *name,
- int num_flow_table_entries)
-{
- return create_ft_common(ns, vport, prio, num_flow_table_entries, name);
-}
-EXPORT_SYMBOL(mlx5_create_vport_flow_table);
-
-struct mlx5_flow_table *mlx5_create_flow_table(struct mlx5_flow_namespace *ns,
- int prio,
- const char *name,
- int num_flow_table_entries)
-{
- return create_ft_common(ns, 0, prio, num_flow_table_entries, name);
-}
-EXPORT_SYMBOL(mlx5_create_flow_table);
-
-static void _fs_del_ft(struct mlx5_flow_table *ft)
-{
- int err;
- struct mlx5_core_dev *dev = fs_get_dev(&ft->base);
- struct fs_prio *prio;
-
- err = mlx5_cmd_fs_destroy_ft(dev, ft->vport, ft->type, ft->id);
- if (err)
- mlx5_core_warn(dev, "flow steering can't destroy ft %s\n",
- ft->base.name);
-
- fs_get_parent(prio, ft);
- prio->num_ft--;
-}
-
-static int update_root_ft_destroy(struct mlx5_flow_root_namespace *root,
- struct mlx5_flow_table *ft)
-{
- int err = 0;
- struct fs_prio *prio;
- struct mlx5_flow_table *next_ft = NULL;
- struct mlx5_flow_table *put_ft = NULL;
-
- if (root->root_ft != ft)
- return 0;
-
- fs_get_parent(prio, ft);
- /*Assuming objs containis only flow tables and
- * flow tables are sorted by level.
- */
- if (!list_is_last(&ft->base.list, &prio->objs)) {
- next_ft = list_next_entry(ft, base.list);
- } else {
- next_ft = find_next_ft(prio);
- put_ft = next_ft;
- }
-
- if (next_ft) {
- err = mlx5_cmd_update_root_ft(root->dev, next_ft->type,
- next_ft->id);
- if (err)
- mlx5_core_warn(root->dev, "Update root flow table of id=%u failed\n",
- ft->id);
- }
- if (!err)
- root->root_ft = next_ft;
-
- if (put_ft)
- fs_put(&put_ft->base);
-
- return err;
-}
-
-/*Objects in the same prio are destroyed in the reverse order they were createrd*/
-int mlx5_destroy_flow_table(struct mlx5_flow_table *ft)
-{
- int err = 0;
- struct fs_prio *prio;
- struct mlx5_flow_root_namespace *root;
- bool is_shared_prio;
- struct mlx5_core_dev *dev;
-
- fs_get_parent(prio, ft);
- root = find_root(&prio->base);
- dev = fs_get_dev(&prio->base);
-
- if (!root) {
- mlx5_core_err(dev,
- "flow steering failed to find root of priority %s",
- prio->base.name);
- return -ENODEV;
- }
-
- is_shared_prio = prio->flags & MLX5_CORE_FS_PRIO_SHARED;
- if (is_shared_prio) {
- mutex_lock(&prio->shared_lock);
- if (ft->shared_refcount > 1) {
- --ft->shared_refcount;
- fs_put(&ft->base);
- mutex_unlock(&prio->shared_lock);
- return 0;
- }
- }
-
- mutex_lock(&prio->base.lock);
- mutex_lock(&ft->base.lock);
-
- err = update_root_ft_destroy(root, ft);
- if (err)
- goto unlock_ft;
-
- /* delete two last entries */
- destroy_star_rule(ft, prio);
-
- mutex_unlock(&ft->base.lock);
- fs_remove_node_parent_locked(&ft->base);
- mutex_unlock(&prio->base.lock);
- if (is_shared_prio)
- mutex_unlock(&prio->shared_lock);
-
- return err;
-
-unlock_ft:
- mutex_unlock(&ft->base.lock);
- mutex_unlock(&prio->base.lock);
- if (is_shared_prio)
- mutex_unlock(&prio->shared_lock);
-
- return err;
-}
-EXPORT_SYMBOL(mlx5_destroy_flow_table);
-
-static struct mlx5_flow_group *fs_create_fg(struct mlx5_core_dev *dev,
- struct mlx5_flow_table *ft,
- struct list_head *prev,
- u32 *fg_in,
- int refcount)
-{
- struct mlx5_flow_group *fg;
- unsigned int group_size;
- int err;
- char name[20];
-
- fg = fs_alloc_fg(fg_in);
- if (IS_ERR(fg))
- return fg;
-
- group_size = MLX5_GET(create_flow_group_in, fg_in, end_flow_index) -
- MLX5_GET(create_flow_group_in, fg_in, start_flow_index) + 1;
- err = mlx5_cmd_fs_create_fg(dev, fg_in,
- ft->vport, ft->type, ft->id,
- &fg->id);
- if (err)
- goto free_fg;
-
- mutex_lock(&ft->base.lock);
-
- if (ft->autogroup.active && group_size == ft->autogroup.group_size)
- ft->autogroup.num_types++;
-
- snprintf(name, sizeof(name), "group_%u", fg->id);
- /*Add node to tree*/
- fs_add_node(&fg->base, &ft->base, name, refcount);
- /*Add node to group list*/
- list_add(&fg->base.list, prev);
- mutex_unlock(&ft->base.lock);
-
- return fg;
-
-free_fg:
- kfree(fg);
- return ERR_PTR(err);
-}
-
-struct mlx5_flow_group *mlx5_create_flow_group(struct mlx5_flow_table *ft,
- u32 *in)
-{
- struct mlx5_flow_group *fg;
- struct mlx5_core_dev *dev = fs_get_dev(&ft->base);
- unsigned int start_index;
-
- start_index = MLX5_GET(create_flow_group_in, in, start_flow_index);
- if (!dev)
- return ERR_PTR(-ENODEV);
-
- if (ft->autogroup.active && start_index < ft->autogroup.max_fte)
- return ERR_PTR(-EPERM);
-
- fg = fs_create_fg(dev, ft, ft->fgs.prev, in, 1);
-
- return fg;
-}
-EXPORT_SYMBOL(mlx5_create_flow_group);
-
-/*Group is destoyed when all the rules in the group were removed*/
-static void fs_del_fg(struct mlx5_flow_group *fg)
-{
- struct mlx5_flow_table *parent_ft;
- struct mlx5_core_dev *dev;
-
- fs_get_parent(parent_ft, fg);
- dev = fs_get_dev(&parent_ft->base);
- WARN_ON(!dev);
-
- if (parent_ft->autogroup.active &&
- fg->max_ftes == parent_ft->autogroup.group_size &&
- fg->start_index < parent_ft->autogroup.max_fte)
- parent_ft->autogroup.num_types--;
-
- if (mlx5_cmd_fs_destroy_fg(dev, parent_ft->vport,
- parent_ft->type,
- parent_ft->id, fg->id))
- mlx5_core_warn(dev, "flow steering can't destroy fg\n");
-}
-
-void mlx5_destroy_flow_group(struct mlx5_flow_group *fg)
-{
- fs_remove_node(&fg->base);
-}
-EXPORT_SYMBOL(mlx5_destroy_flow_group);
-
-static bool _fs_match_exact_val(void *mask, void *val1, void *val2, size_t size)
-{
- unsigned int i;
-
- /* TODO: optimize by comparing 64bits when possible */
- for (i = 0; i < size; i++, mask++, val1++, val2++)
- if ((*((u8 *)val1) & (*(u8 *)mask)) !=
- ((*(u8 *)val2) & (*(u8 *)mask)))
- return false;
-
- return true;
-}
-
-bool fs_match_exact_val(struct mlx5_core_fs_mask *mask,
- void *val1, void *val2)
-{
- if (mask->match_criteria_enable &
- 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS) {
- void *fte_match1 = MLX5_ADDR_OF(fte_match_param,
- val1, outer_headers);
- void *fte_match2 = MLX5_ADDR_OF(fte_match_param,
- val2, outer_headers);
- void *fte_mask = MLX5_ADDR_OF(fte_match_param,
- mask->match_criteria, outer_headers);
-
- if (!_fs_match_exact_val(fte_mask, fte_match1, fte_match2,
- MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4)))
- return false;
- }
-
- if (mask->match_criteria_enable &
- 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS) {
- void *fte_match1 = MLX5_ADDR_OF(fte_match_param,
- val1, misc_parameters);
- void *fte_match2 = MLX5_ADDR_OF(fte_match_param,
- val2, misc_parameters);
- void *fte_mask = MLX5_ADDR_OF(fte_match_param,
- mask->match_criteria, misc_parameters);
-
- if (!_fs_match_exact_val(fte_mask, fte_match1, fte_match2,
- MLX5_ST_SZ_BYTES(fte_match_set_misc)))
- return false;
- }
- if (mask->match_criteria_enable &
- 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_INNER_HEADERS) {
- void *fte_match1 = MLX5_ADDR_OF(fte_match_param,
- val1, inner_headers);
- void *fte_match2 = MLX5_ADDR_OF(fte_match_param,
- val2, inner_headers);
- void *fte_mask = MLX5_ADDR_OF(fte_match_param,
- mask->match_criteria, inner_headers);
-
- if (!_fs_match_exact_val(fte_mask, fte_match1, fte_match2,
- MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4)))
- return false;
- }
- return true;
-}
-
-bool fs_match_exact_mask(u8 match_criteria_enable1,
- u8 match_criteria_enable2,
- void *mask1, void *mask2)
-{
- return match_criteria_enable1 == match_criteria_enable2 &&
- !memcmp(mask1, mask2, MLX5_ST_SZ_BYTES(fte_match_param));
-}
-
-static struct mlx5_flow_table *find_first_ft_in_ns_reverse(struct mlx5_flow_namespace *ns,
- struct list_head *start);
-
-static struct mlx5_flow_table *_find_first_ft_in_prio_reverse(struct fs_prio *prio,
- struct list_head *start)
-{
- struct fs_base *it = container_of(start, struct fs_base, list);
-
- if (!prio)
- return NULL;
-
- fs_for_each_ns_or_ft_continue_reverse(it, prio) {
- struct mlx5_flow_namespace *ns;
- struct mlx5_flow_table *ft;
-
- if (it->type == FS_TYPE_FLOW_TABLE) {
- fs_get_obj(ft, it);
- fs_get(&ft->base);
- return ft;
- }
-
- fs_get_obj(ns, it);
- WARN_ON(ns->base.type != FS_TYPE_NAMESPACE);
-
- ft = find_first_ft_in_ns_reverse(ns, &ns->prios);
- if (ft)
- return ft;
- }
-
- return NULL;
-}
-
-static struct mlx5_flow_table *find_first_ft_in_prio_reverse(struct fs_prio *prio,
- struct list_head *start)
-{
- struct mlx5_flow_table *ft;
-
- if (!prio)
- return NULL;
-
- mutex_lock(&prio->base.lock);
- ft = _find_first_ft_in_prio_reverse(prio, start);
- mutex_unlock(&prio->base.lock);
-
- return ft;
-}
-
-static struct mlx5_flow_table *find_first_ft_in_ns_reverse(struct mlx5_flow_namespace *ns,
- struct list_head *start)
-{
- struct fs_prio *prio;
-
- if (!ns)
- return NULL;
-
- fs_get_obj(prio, container_of(start, struct fs_base, list));
- mutex_lock(&ns->base.lock);
- fs_for_each_prio_continue_reverse(prio, ns) {
- struct mlx5_flow_table *ft;
-
- ft = find_first_ft_in_prio_reverse(prio, &prio->objs);
- if (ft) {
- mutex_unlock(&ns->base.lock);
- return ft;
- }
- }
- mutex_unlock(&ns->base.lock);
-
- return NULL;
-}
-
-/* Returned a held ft, assumed curr is protected, assumed curr's parent is
- * locked
- */
-static struct mlx5_flow_table *find_prev_ft(struct mlx5_flow_table *curr,
- struct fs_prio *prio)
-{
- struct mlx5_flow_table *ft = NULL;
- struct fs_base *curr_base;
-
- if (!curr)
- return NULL;
-
- /* prio has either namespace or flow-tables, but not both */
- if (!list_empty(&prio->objs) &&
- list_first_entry(&prio->objs, struct mlx5_flow_table, base.list) !=
- curr)
- return NULL;
-
- while (!ft && prio) {
- struct mlx5_flow_namespace *ns;
-
- fs_get_parent(ns, prio);
- ft = find_first_ft_in_ns_reverse(ns, &prio->base.list);
- curr_base = &ns->base;
- fs_get_parent(prio, ns);
-
- if (prio && !ft)
- ft = find_first_ft_in_prio_reverse(prio,
- &curr_base->list);
- }
- return ft;
-}
-
-static struct mlx5_flow_table *_find_first_ft_in_prio(struct fs_prio *prio,
- struct list_head *start)
-{
- struct fs_base *it = container_of(start, struct fs_base, list);
-
- if (!prio)
- return NULL;
-
- fs_for_each_ns_or_ft_continue(it, prio) {
- struct mlx5_flow_namespace *ns;
- struct mlx5_flow_table *ft;
-
- if (it->type == FS_TYPE_FLOW_TABLE) {
- fs_get_obj(ft, it);
- fs_get(&ft->base);
- return ft;
- }
-
- fs_get_obj(ns, it);
- WARN_ON(ns->base.type != FS_TYPE_NAMESPACE);
-
- ft = find_first_ft_in_ns(ns, &ns->prios);
- if (ft)
- return ft;
- }
-
- return NULL;
-}
-
-static struct mlx5_flow_table *find_first_ft_in_prio(struct fs_prio *prio,
- struct list_head *start)
-{
- struct mlx5_flow_table *ft;
-
- if (!prio)
- return NULL;
-
- mutex_lock(&prio->base.lock);
- ft = _find_first_ft_in_prio(prio, start);
- mutex_unlock(&prio->base.lock);
-
- return ft;
-}
-
-static struct mlx5_flow_table *find_first_ft_in_ns(struct mlx5_flow_namespace *ns,
- struct list_head *start)
-{
- struct fs_prio *prio;
-
- if (!ns)
- return NULL;
-
- fs_get_obj(prio, container_of(start, struct fs_base, list));
- mutex_lock(&ns->base.lock);
- fs_for_each_prio_continue(prio, ns) {
- struct mlx5_flow_table *ft;
-
- ft = find_first_ft_in_prio(prio, &prio->objs);
- if (ft) {
- mutex_unlock(&ns->base.lock);
- return ft;
- }
- }
- mutex_unlock(&ns->base.lock);
-
- return NULL;
-}
-
-/* returned a held ft, assumed curr is protected, assumed curr's parent is
- * locked
- */
-static struct mlx5_flow_table *find_next_ft(struct fs_prio *prio)
-{
- struct mlx5_flow_table *ft = NULL;
- struct fs_base *curr_base;
-
- while (!ft && prio) {
- struct mlx5_flow_namespace *ns;
-
- fs_get_parent(ns, prio);
- ft = find_first_ft_in_ns(ns, &prio->base.list);
- curr_base = &ns->base;
- fs_get_parent(prio, ns);
-
- if (!ft && prio)
- ft = _find_first_ft_in_prio(prio, &curr_base->list);
- }
- return ft;
-}
-
-
-/* called under ft mutex lock */
-static struct mlx5_flow_group *create_autogroup(struct mlx5_flow_table *ft,
- u8 match_criteria_enable,
- u32 *match_criteria)
-{
- unsigned int group_size;
- unsigned int candidate_index = 0;
- struct mlx5_flow_group *g;
- struct mlx5_flow_group *ret;
- struct list_head *prev = &ft->fgs;
- struct mlx5_core_dev *dev;
- u32 *in;
- int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
- void *match_criteria_addr;
- u32 max_fte = ft->autogroup.max_fte;
-
- if (!ft->autogroup.active)
- return ERR_PTR(-ENOENT);
-
- dev = fs_get_dev(&ft->base);
- if (!dev)
- return ERR_PTR(-ENODEV);
-
- in = mlx5_vzalloc(inlen);
- if (!in) {
- mlx5_core_warn(dev, "failed to allocate inbox\n");
- return ERR_PTR(-ENOMEM);
- }
-
-
- if (ft->autogroup.num_types < ft->autogroup.max_types)
- group_size = ft->autogroup.group_size;
- else
- group_size = 1;
-
- if (group_size == 0) {
- mlx5_core_warn(dev,
- "flow steering can't create group size of 0\n");
- ret = ERR_PTR(-EINVAL);
- goto out;
- }
-
- /* sorted by start_index */
- fs_for_each_fg(g, ft) {
- if (candidate_index + group_size > g->start_index)
- candidate_index = g->start_index + g->max_ftes;
- else
- break;
- prev = &g->base.list;
- }
-
- if (candidate_index + group_size > max_fte) {
- ret = ERR_PTR(-ENOSPC);
- goto out;
- }
-
- MLX5_SET(create_flow_group_in, in, match_criteria_enable,
- match_criteria_enable);
- MLX5_SET(create_flow_group_in, in, start_flow_index, candidate_index);
- MLX5_SET(create_flow_group_in, in, end_flow_index, candidate_index +
- group_size - 1);
- match_criteria_addr = MLX5_ADDR_OF(create_flow_group_in,
- in, match_criteria);
- memcpy(match_criteria_addr, match_criteria,
- MLX5_ST_SZ_BYTES(fte_match_param));
-
- ret = fs_create_fg(dev, ft, prev, in, 0);
-out:
- kvfree(in);
- return ret;
-}
-
-static struct mlx5_flow_namespace *get_ns_with_notifiers(struct fs_base *node)
-{
- struct mlx5_flow_namespace *ns = NULL;
-
- while (node && (node->type != FS_TYPE_NAMESPACE ||
- list_empty(&container_of(node, struct
- mlx5_flow_namespace,
- base)->list_notifiers)))
- node = node->parent;
-
- if (node)
- fs_get_obj(ns, node);
-
- return ns;
-}
-
-
-/*Assumption- fte is locked*/
-static void call_to_add_rule_notifiers(struct mlx5_flow_rule *dst,
- struct fs_fte *fte)
-{
- struct mlx5_flow_namespace *ns;
- struct mlx5_flow_handler *iter_handler;
- struct fs_client_priv_data *iter_client;
- void *data;
- bool is_new_rule = list_first_entry(&fte->dests,
- struct mlx5_flow_rule,
- base.list) == dst;
- int err;
-
- ns = get_ns_with_notifiers(&fte->base);
- if (!ns)
- return;
-
- down_read(&ns->notifiers_rw_sem);
- list_for_each_entry(iter_handler, &ns->list_notifiers,
- list) {
- if (iter_handler->add_dst_cb) {
- data = NULL;
- mutex_lock(&dst->clients_lock);
- list_for_each_entry(
- iter_client, &dst->clients_data, list) {
- if (iter_client->fs_handler == iter_handler) {
- data = iter_client->client_dst_data;
- break;
- }
- }
- mutex_unlock(&dst->clients_lock);
- err = iter_handler->add_dst_cb(dst,
- is_new_rule,
- data,
- iter_handler->client_context);
- if (err)
- break;
- }
- }
- up_read(&ns->notifiers_rw_sem);
-}
-
-static void call_to_del_rule_notifiers(struct mlx5_flow_rule *dst,
- struct fs_fte *fte)
-{
- struct mlx5_flow_namespace *ns;
- struct mlx5_flow_handler *iter_handler;
- struct fs_client_priv_data *iter_client;
- void *data;
- bool ctx_changed = (fte->dests_size == 0);
-
- ns = get_ns_with_notifiers(&fte->base);
- if (!ns)
- return;
- down_read(&ns->notifiers_rw_sem);
- list_for_each_entry(iter_handler, &ns->list_notifiers,
- list) {
- data = NULL;
- mutex_lock(&dst->clients_lock);
- list_for_each_entry(iter_client, &dst->clients_data, list) {
- if (iter_client->fs_handler == iter_handler) {
- data = iter_client->client_dst_data;
- break;
- }
- }
- mutex_unlock(&dst->clients_lock);
- if (iter_handler->del_dst_cb) {
- iter_handler->del_dst_cb(dst, ctx_changed, data,
- iter_handler->client_context);
- }
- }
- up_read(&ns->notifiers_rw_sem);
-}
-
-/* fte should not be deleted while calling this function */
-static struct mlx5_flow_rule *_fs_add_dst_fte(struct fs_fte *fte,
- struct mlx5_flow_group *fg,
- struct mlx5_flow_destination *dest)
-{
- struct mlx5_flow_table *ft;
- struct mlx5_flow_rule *dst;
- int err;
-
- dst = kzalloc(sizeof(*dst), GFP_KERNEL);
- if (!dst)
- return ERR_PTR(-ENOMEM);
-
- memcpy(&dst->dest_attr, dest, sizeof(*dest));
- dst->base.type = FS_TYPE_FLOW_DEST;
- INIT_LIST_HEAD(&dst->clients_data);
- mutex_init(&dst->clients_lock);
- fs_get_parent(ft, fg);
- /*Add dest to dests list- added as first element after the head*/
- list_add_tail(&dst->base.list, &fte->dests);
- fte->dests_size++;
- err = mlx5_cmd_fs_set_fte(fs_get_dev(&ft->base),
- ft->vport,
- &fte->status,
- fte->val, ft->type,
- ft->id, fte->index, fg->id, &fte->flow_act,
- fte->sw_action, fte->dests_size, &fte->dests);
- if (err)
- goto free_dst;
-
- list_del(&dst->base.list);
-
- return dst;
-
-free_dst:
- list_del(&dst->base.list);
- kfree(dst);
- fte->dests_size--;
- return ERR_PTR(err);
-}
-
-static char *get_dest_name(struct mlx5_flow_destination *dest)
-{
- char *name = kzalloc(sizeof(char) * 20, GFP_KERNEL);
-
- switch (dest->type) {
- case MLX5_FLOW_CONTEXT_DEST_TYPE_FLOW_TABLE:
- snprintf(name, 20, "dest_%s_%u", "flow_table",
- dest->ft->id);
- return name;
- case MLX5_FLOW_CONTEXT_DEST_TYPE_VPORT:
- snprintf(name, 20, "dest_%s_%u", "vport",
- dest->vport_num);
- return name;
- case MLX5_FLOW_CONTEXT_DEST_TYPE_TIR:
- snprintf(name, 20, "dest_%s_%u", "tir", dest->tir_num);
- return name;
- default:
- kfree(name);
- return NULL;
- }
-}
-
-/* assumed fg is locked */
-static unsigned int fs_get_free_fg_index(struct mlx5_flow_group *fg,
- struct list_head **prev)
-{
- struct fs_fte *fte;
- unsigned int start = fg->start_index;
-
- if (prev)
- *prev = &fg->ftes;
-
- /* assumed list is sorted by index */
- fs_for_each_fte(fte, fg) {
- if (fte->index != start)
- return start;
- start++;
- if (prev)
- *prev = &fte->base.list;
- }
-
- return start;
-}
-
-
-static struct fs_fte *fs_create_fte(struct mlx5_flow_group *fg,
- u32 *match_value,
- u32 sw_action,
- struct mlx5_flow_act *flow_act,
- struct list_head **prev)
-{
- struct fs_fte *fte;
- int index = 0;
-
- index = fs_get_free_fg_index(fg, prev);
- fte = fs_alloc_fte(sw_action, flow_act, match_value, index);
- if (IS_ERR(fte))
- return fte;
-
- return fte;
-}
-
-static void add_rule_to_tree(struct mlx5_flow_rule *rule,
- struct fs_fte *fte)
-{
- char *dest_name;
-
- dest_name = get_dest_name(&rule->dest_attr);
- fs_add_node(&rule->base, &fte->base, dest_name, 1);
- /* re-add to list, since fs_add_node reset our list */
- list_add_tail(&rule->base.list, &fte->dests);
- kfree(dest_name);
- call_to_add_rule_notifiers(rule, fte);
-}
-
-static void fs_del_dst(struct mlx5_flow_rule *dst)
-{
- struct mlx5_flow_table *ft;
- struct mlx5_flow_group *fg;
- struct fs_fte *fte;
- u32 *match_value;
- struct mlx5_core_dev *dev = fs_get_dev(&dst->base);
- int match_len = MLX5_ST_SZ_BYTES(fte_match_param);
- int err;
-
- WARN_ON(!dev);
-
- match_value = mlx5_vzalloc(match_len);
- if (!match_value) {
- mlx5_core_warn(dev, "failed to allocate inbox\n");
- return;
- }
-
- fs_get_parent(fte, dst);
- fs_get_parent(fg, fte);
- mutex_lock(&fg->base.lock);
- memcpy(match_value, fte->val, sizeof(fte->val));
- /* ft can't be changed as fg is locked */
- fs_get_parent(ft, fg);
- list_del(&dst->base.list);
- fte->dests_size--;
- if (fte->dests_size) {
- err = mlx5_cmd_fs_set_fte(dev, ft->vport,
- &fte->status, match_value, ft->type,
- ft->id, fte->index, fg->id,
- &fte->flow_act, fte->sw_action,
- fte->dests_size, &fte->dests);
- if (err) {
- mlx5_core_warn(dev, "%s can't delete dst %s\n",
- __func__, dst->base.name);
- goto err;
- }
- }
- call_to_del_rule_notifiers(dst, fte);
-err:
- mutex_unlock(&fg->base.lock);
- kvfree(match_value);
-}
-
-static void fs_del_fte(struct fs_fte *fte)
-{
- struct mlx5_flow_table *ft;
- struct mlx5_flow_group *fg;
- int err;
- struct mlx5_core_dev *dev;
-
- fs_get_parent(fg, fte);
- fs_get_parent(ft, fg);
-
- dev = fs_get_dev(&ft->base);
- WARN_ON(!dev);
-
- err = mlx5_cmd_fs_delete_fte(dev, ft->vport, &fte->status,
- ft->type, ft->id, fte->index);
- if (err)
- mlx5_core_warn(dev, "flow steering can't delete fte %s\n",
- fte->base.name);
-
- fg->num_ftes--;
-}
-
-static bool check_conflicting_actions(const struct mlx5_flow_act *act1,
- const struct mlx5_flow_act *act2)
-{
- u32 action1 = act1->actions;
- u32 action2 = act2->actions;
- u32 xored_actions;
-
- xored_actions = action1 ^ action2;
-
- if (xored_actions & (MLX5_FLOW_ACT_ACTIONS_FLOW_TAG))
- return true;
-
- if (action1 & MLX5_FLOW_ACT_ACTIONS_FLOW_TAG &&
- act1->flow_tag != act2->flow_tag)
- return true;
-
- /* Can even have complex actions in merged rules */
- if (action1 & MLX5_FLOW_ACT_ACTIONS_MODIFY_HDR)
- return true;
-
- if (action1 & MLX5_FLOW_ACT_ACTIONS_PACKET_REFORMAT)
- return true;
-
- if (action1 & MLX5_FLOW_ACT_ACTIONS_COUNT)
- return true;
-
- return false;
-}
-
-/* assuming parent fg is locked */
-/* Add dst algorithm */
-static struct mlx5_flow_rule *fs_add_dst_fg(struct mlx5_flow_group *fg,
- u32 *match_value,
- u32 sw_action,
- struct mlx5_flow_act *flow_act,
- struct mlx5_flow_destination *dest)
-{
- struct fs_fte *fte;
- struct mlx5_flow_rule *dst;
- struct mlx5_flow_table *ft;
- struct list_head *prev;
- char fte_name[20];
-
- mutex_lock(&fg->base.lock);
- if (flow_act->flags & MLX5_FLOW_ACT_NO_APPEND)
- goto insert_fte;
-
- fs_for_each_fte(fte, fg) {
- /* TODO: Check of size against PRM max size */
- mutex_lock(&fte->base.lock);
- if (fs_match_exact_val(&fg->mask, match_value, &fte->val) &&
- sw_action == fte->sw_action &&
- !check_conflicting_actions(flow_act, &fte->flow_act)) {
- dst = _fs_add_dst_fte(fte, fg, dest);
- mutex_unlock(&fte->base.lock);
- if (IS_ERR(dst))
- goto unlock_fg;
- goto add_rule;
- }
- mutex_unlock(&fte->base.lock);
- }
-
-insert_fte:
- fs_get_parent(ft, fg);
- if (fg->num_ftes == fg->max_ftes) {
- dst = ERR_PTR(-ENOSPC);
- goto unlock_fg;
- }
-
- fte = fs_create_fte(fg, match_value, sw_action, flow_act, &prev);
- if (IS_ERR(fte)) {
- dst = (void *)fte;
- goto unlock_fg;
- }
- dst = _fs_add_dst_fte(fte, fg, dest);
- if (IS_ERR(dst)) {
- kfree(fte);
- goto unlock_fg;
- }
-
- fg->num_ftes++;
-
- snprintf(fte_name, sizeof(fte_name), "fte%u", fte->index);
- /* Add node to tree */
- fs_add_node(&fte->base, &fg->base, fte_name, 0);
- list_add(&fte->base.list, prev);
-add_rule:
- add_rule_to_tree(dst, fte);
-unlock_fg:
- mutex_unlock(&fg->base.lock);
- return dst;
-}
-
-static struct mlx5_flow_rule *fs_add_dst_ft(struct mlx5_flow_table *ft,
- u8 match_criteria_enable,
- u32 *match_criteria,
- u32 *match_value,
- u32 sw_action,
- struct mlx5_flow_act *flow_act,
- struct mlx5_flow_destination *dest)
-{
- /*? where dst_entry is allocated*/
- struct mlx5_flow_group *g;
- struct mlx5_flow_rule *dst;
-
- fs_get(&ft->base);
- mutex_lock(&ft->base.lock);
- fs_for_each_fg(g, ft)
- if (fs_match_exact_mask(g->mask.match_criteria_enable,
- match_criteria_enable,
- g->mask.match_criteria,
- match_criteria)) {
- mutex_unlock(&ft->base.lock);
-
- dst = fs_add_dst_fg(g, match_value, sw_action, flow_act, dest);
- if (PTR_ERR(dst) && PTR_ERR(dst) != -ENOSPC)
- goto unlock;
- }
- mutex_unlock(&ft->base.lock);
-
- g = create_autogroup(ft, match_criteria_enable, match_criteria);
- if (IS_ERR(g)) {
- dst = (void *)g;
- goto unlock;
- }
-
- dst = fs_add_dst_fg(g, match_value,
- sw_action, flow_act, dest);
- if (IS_ERR(dst)) {
- /* Remove assumes refcount > 0 and autogroup creates a group
- * with a refcount = 0.
- */
- fs_get(&g->base);
- fs_remove_node(&g->base);
- goto unlock;
- }
-
-unlock:
- fs_put(&ft->base);
- return dst;
-}
-
-struct mlx5_flow_rule *
-mlx5_add_flow_rule(struct mlx5_flow_table *ft,
- u8 match_criteria_enable,
- u32 *match_criteria,
- u32 *match_value,
- u32 sw_action,
- struct mlx5_flow_act *flow_act,
- struct mlx5_flow_destination *dest)
-{
- struct mlx5_flow_rule *dst;
- struct mlx5_flow_namespace *ns;
-
- ns = get_ns_with_notifiers(&ft->base);
- if (ns)
- down_read(&ns->dests_rw_sem);
- dst = fs_add_dst_ft(ft, match_criteria_enable, match_criteria,
- match_value, sw_action, flow_act, dest);
- if (ns)
- up_read(&ns->dests_rw_sem);
-
- return dst;
-
-
-}
-EXPORT_SYMBOL(mlx5_add_flow_rule);
-
-void mlx5_del_flow_rule(struct mlx5_flow_rule **pp)
-{
- struct mlx5_flow_namespace *ns;
- struct mlx5_flow_rule *dst;
-
- dst = *pp;
- *pp = NULL;
-
- if (IS_ERR_OR_NULL(dst))
- return;
- ns = get_ns_with_notifiers(&dst->base);
- if (ns)
- down_read(&ns->dests_rw_sem);
- fs_remove_node(&dst->base);
- if (ns)
- up_read(&ns->dests_rw_sem);
-}
-EXPORT_SYMBOL(mlx5_del_flow_rule);
-
-#define MLX5_CORE_FS_ROOT_NS_NAME "root"
-#define MLX5_CORE_FS_ESW_EGRESS_ACL "esw_egress_root"
-#define MLX5_CORE_FS_ESW_INGRESS_ACL "esw_ingress_root"
-#define MLX5_CORE_FS_FDB_ROOT_NS_NAME "fdb_root"
-#define MLX5_CORE_FS_SNIFFER_RX_ROOT_NS_NAME "sniffer_rx_root"
-#define MLX5_CORE_FS_SNIFFER_TX_ROOT_NS_NAME "sniffer_tx_root"
-#define MLX5_CORE_FS_PRIO_MAX_FT 4
-#define MLX5_CORE_FS_PRIO_MAX_NS 1
-
-static struct fs_prio *fs_create_prio(struct mlx5_flow_namespace *ns,
- unsigned prio, int max_ft,
- const char *name, u8 flags)
-{
- struct fs_prio *fs_prio;
-
- fs_prio = kzalloc(sizeof(*fs_prio), GFP_KERNEL);
- if (!fs_prio)
- return ERR_PTR(-ENOMEM);
-
- fs_prio->base.type = FS_TYPE_PRIO;
- fs_add_node(&fs_prio->base, &ns->base, name, 1);
- fs_prio->max_ft = max_ft;
- fs_prio->max_ns = MLX5_CORE_FS_PRIO_MAX_NS;
- fs_prio->prio = prio;
- fs_prio->flags = flags;
- list_add_tail(&fs_prio->base.list, &ns->prios);
- INIT_LIST_HEAD(&fs_prio->objs);
- mutex_init(&fs_prio->shared_lock);
-
- return fs_prio;
-}
-
-static void cleanup_root_ns(struct mlx5_core_dev *dev)
-{
- struct mlx5_flow_root_namespace *root_ns = dev->root_ns;
- struct fs_prio *iter_prio;
-
- if (!root_ns)
- return;
-
- /* stage 1 */
- fs_for_each_prio(iter_prio, &root_ns->ns) {
- struct mlx5_flow_namespace *iter_ns;
-
- fs_for_each_ns(iter_ns, iter_prio) {
- while (!list_empty(&iter_ns->prios)) {
- struct fs_base *iter_prio2 =
- list_first_entry(&iter_ns->prios,
- struct fs_base,
- list);
-
- fs_remove_node(iter_prio2);
- }
- }
- }
-
- /* stage 2 */
- fs_for_each_prio(iter_prio, &root_ns->ns) {
- while (!list_empty(&iter_prio->objs)) {
- struct fs_base *iter_ns =
- list_first_entry(&iter_prio->objs,
- struct fs_base,
- list);
-
- fs_remove_node(iter_ns);
- }
- }
- /* stage 3 */
- while (!list_empty(&root_ns->ns.prios)) {
- struct fs_base *iter_prio =
- list_first_entry(&root_ns->ns.prios,
- struct fs_base,
- list);
-
- fs_remove_node(iter_prio);
- }
-
- fs_remove_node(&root_ns->ns.base);
- dev->root_ns = NULL;
-}
-
-static void cleanup_single_prio_root_ns(struct mlx5_core_dev *dev,
- struct mlx5_flow_root_namespace *root_ns)
-{
- struct fs_base *prio;
-
- if (!root_ns)
- return;
-
- if (!list_empty(&root_ns->ns.prios)) {
- prio = list_first_entry(&root_ns->ns.prios,
- struct fs_base,
- list);
- fs_remove_node(prio);
- }
- fs_remove_node(&root_ns->ns.base);
- root_ns = NULL;
-}
-
-void mlx5_cleanup_fs(struct mlx5_core_dev *dev)
-{
- mlx5_cleanup_fc_stats(dev);
- cleanup_root_ns(dev);
- cleanup_single_prio_root_ns(dev, dev->sniffer_rx_root_ns);
- cleanup_single_prio_root_ns(dev, dev->sniffer_tx_root_ns);
- cleanup_single_prio_root_ns(dev, dev->fdb_root_ns);
- cleanup_single_prio_root_ns(dev, dev->esw_egress_root_ns);
- cleanup_single_prio_root_ns(dev, dev->esw_ingress_root_ns);
-}
-
-static struct mlx5_flow_namespace *fs_init_namespace(struct mlx5_flow_namespace
- *ns)
-{
- ns->base.type = FS_TYPE_NAMESPACE;
- init_rwsem(&ns->dests_rw_sem);
- init_rwsem(&ns->notifiers_rw_sem);
- INIT_LIST_HEAD(&ns->prios);
- INIT_LIST_HEAD(&ns->list_notifiers);
-
- return ns;
-}
-
-static struct mlx5_flow_root_namespace *create_root_ns(struct mlx5_core_dev *dev,
- enum fs_ft_type
- table_type,
- char *name)
-{
- struct mlx5_flow_root_namespace *root_ns;
- struct mlx5_flow_namespace *ns;
-
- /* create the root namespace */
- root_ns = mlx5_vzalloc(sizeof(*root_ns));
- if (!root_ns)
- goto err;
-
- root_ns->dev = dev;
- root_ns->table_type = table_type;
- mutex_init(&root_ns->fs_chain_lock);
-
- ns = &root_ns->ns;
- fs_init_namespace(ns);
- fs_add_node(&ns->base, NULL, name, 1);
-
- return root_ns;
-err:
- return NULL;
-}
-
-static int init_fdb_root_ns(struct mlx5_core_dev *dev)
-{
- struct fs_prio *prio;
-
- dev->fdb_root_ns = create_root_ns(dev, FS_FT_FDB,
- MLX5_CORE_FS_FDB_ROOT_NS_NAME);
- if (!dev->fdb_root_ns)
- return -ENOMEM;
-
- /* create 1 prio*/
- prio = fs_create_prio(&dev->fdb_root_ns->ns, 0, 1, "fdb_prio", 0);
- if (IS_ERR(prio))
- return PTR_ERR(prio);
- else
- return 0;
-}
-
-#define MAX_VPORTS 128
-
-static int init_egress_acl_root_ns(struct mlx5_core_dev *dev)
-{
- struct fs_prio *prio;
-
- dev->esw_egress_root_ns = create_root_ns(dev, FS_FT_ESW_EGRESS_ACL,
- MLX5_CORE_FS_ESW_EGRESS_ACL);
- if (!dev->esw_egress_root_ns)
- return -ENOMEM;
-
- /* create 1 prio*/
- prio = fs_create_prio(&dev->esw_egress_root_ns->ns, 0, MAX_VPORTS,
- "esw_egress_prio", 0);
- if (IS_ERR(prio))
- return PTR_ERR(prio);
- else
- return 0;
-}
-
-static int init_ingress_acl_root_ns(struct mlx5_core_dev *dev)
-{
- struct fs_prio *prio;
-
- dev->esw_ingress_root_ns = create_root_ns(dev, FS_FT_ESW_INGRESS_ACL,
- MLX5_CORE_FS_ESW_INGRESS_ACL);
- if (!dev->esw_ingress_root_ns)
- return -ENOMEM;
-
- /* create 1 prio*/
- prio = fs_create_prio(&dev->esw_ingress_root_ns->ns, 0, MAX_VPORTS,
- "esw_ingress_prio", 0);
- if (IS_ERR(prio))
- return PTR_ERR(prio);
- else
- return 0;
-}
-
-static int init_sniffer_rx_root_ns(struct mlx5_core_dev *dev)
-{
- struct fs_prio *prio;
-
- dev->sniffer_rx_root_ns = create_root_ns(dev, FS_FT_SNIFFER_RX,
- MLX5_CORE_FS_SNIFFER_RX_ROOT_NS_NAME);
- if (!dev->sniffer_rx_root_ns)
- return -ENOMEM;
-
- /* create 1 prio*/
- prio = fs_create_prio(&dev->sniffer_rx_root_ns->ns, 0, 1,
- "sniffer_prio", 0);
- if (IS_ERR(prio))
- return PTR_ERR(prio);
- else
- return 0;
-}
-
-
-static int init_sniffer_tx_root_ns(struct mlx5_core_dev *dev)
-{
- struct fs_prio *prio;
-
- dev->sniffer_tx_root_ns = create_root_ns(dev, FS_FT_SNIFFER_TX,
- MLX5_CORE_FS_SNIFFER_TX_ROOT_NS_NAME);
- if (!dev->sniffer_tx_root_ns)
- return -ENOMEM;
-
- /* create 1 prio*/
- prio = fs_create_prio(&dev->sniffer_tx_root_ns->ns, 0, 1,
- "sniffer_prio", 0);
- if (IS_ERR(prio))
- return PTR_ERR(prio);
- else
- return 0;
-}
-
-static struct mlx5_flow_namespace *fs_create_namespace(struct fs_prio *prio,
- const char *name)
-{
- struct mlx5_flow_namespace *ns;
-
- ns = kzalloc(sizeof(*ns), GFP_KERNEL);
- if (!ns)
- return ERR_PTR(-ENOMEM);
-
- fs_init_namespace(ns);
- fs_add_node(&ns->base, &prio->base, name, 1);
- list_add_tail(&ns->base.list, &prio->objs);
-
- return ns;
-}
-
-#define FLOW_TABLE_BIT_SZ 1
-#define GET_FLOW_TABLE_CAP(dev, offset) \
- ((be32_to_cpu(*((__be32 *)(dev->hca_caps_cur[MLX5_CAP_FLOW_TABLE]) + \
- offset / 32)) >> \
- (32 - FLOW_TABLE_BIT_SZ - (offset & 0x1f))) & FLOW_TABLE_BIT_SZ)
-
-static bool has_required_caps(struct mlx5_core_dev *dev, struct node_caps *caps)
-{
- int i;
-
- for (i = 0; i < caps->arr_sz; i++) {
- if (!GET_FLOW_TABLE_CAP(dev, caps->caps[i]))
- return false;
- }
- return true;
-}
-
-static int _init_root_tree(struct mlx5_core_dev *dev, int max_ft_level,
- struct init_tree_node *node, struct fs_base *base_parent,
- struct init_tree_node *tree_parent)
-{
- struct mlx5_flow_namespace *fs_ns;
- struct fs_prio *fs_prio;
- int priority;
- struct fs_base *base;
- int i;
- int err = 0;
-
- if (node->type == FS_TYPE_PRIO) {
- if ((node->min_ft_level > max_ft_level) ||
- !has_required_caps(dev, &node->caps))
- goto out;
-
- fs_get_obj(fs_ns, base_parent);
- priority = node - tree_parent->children;
- fs_prio = fs_create_prio(fs_ns, priority,
- node->max_ft,
- node->name, node->flags);
- if (IS_ERR(fs_prio)) {
- err = PTR_ERR(fs_prio);
- goto out;
- }
- base = &fs_prio->base;
- } else if (node->type == FS_TYPE_NAMESPACE) {
- fs_get_obj(fs_prio, base_parent);
- fs_ns = fs_create_namespace(fs_prio, node->name);
- if (IS_ERR(fs_ns)) {
- err = PTR_ERR(fs_ns);
- goto out;
- }
- base = &fs_ns->base;
- } else {
- return -EINVAL;
- }
- for (i = 0; i < node->ar_size; i++) {
- err = _init_root_tree(dev, max_ft_level, &node->children[i], base,
- node);
- if (err)
- break;
- }
-out:
- return err;
-}
-
-static int init_root_tree(struct mlx5_core_dev *dev, int max_ft_level,
- struct init_tree_node *node, struct fs_base *parent)
-{
- int i;
- struct mlx5_flow_namespace *fs_ns;
- int err = 0;
-
- fs_get_obj(fs_ns, parent);
- for (i = 0; i < node->ar_size; i++) {
- err = _init_root_tree(dev, max_ft_level,
- &node->children[i], &fs_ns->base, node);
- if (err)
- break;
- }
- return err;
-}
-
-static int sum_max_ft_in_prio(struct fs_prio *prio);
-static int sum_max_ft_in_ns(struct mlx5_flow_namespace *ns)
-{
- struct fs_prio *prio;
- int sum = 0;
-
- fs_for_each_prio(prio, ns) {
- sum += sum_max_ft_in_prio(prio);
- }
- return sum;
-}
-
-static int sum_max_ft_in_prio(struct fs_prio *prio)
-{
- int sum = 0;
- struct fs_base *it;
- struct mlx5_flow_namespace *ns;
-
- if (prio->max_ft)
- return prio->max_ft;
-
- fs_for_each_ns_or_ft(it, prio) {
- if (it->type == FS_TYPE_FLOW_TABLE)
- continue;
-
- fs_get_obj(ns, it);
- sum += sum_max_ft_in_ns(ns);
- }
- prio->max_ft = sum;
- return sum;
-}
-
-static void set_max_ft(struct mlx5_flow_namespace *ns)
-{
- struct fs_prio *prio;
-
- if (!ns)
- return;
-
- fs_for_each_prio(prio, ns)
- sum_max_ft_in_prio(prio);
-}
-
-static int init_root_ns(struct mlx5_core_dev *dev)
-{
- int max_ft_level = MLX5_CAP_FLOWTABLE(dev,
- flow_table_properties_nic_receive.
- max_ft_level);
-
- dev->root_ns = create_root_ns(dev, FS_FT_NIC_RX,
- MLX5_CORE_FS_ROOT_NS_NAME);
- if (IS_ERR_OR_NULL(dev->root_ns))
- goto err;
-
-
- if (init_root_tree(dev, max_ft_level, &root_fs, &dev->root_ns->ns.base))
- goto err;
-
- set_max_ft(&dev->root_ns->ns);
-
- return 0;
-err:
- return -ENOMEM;
-}
-
-u8 mlx5_get_match_criteria_enable(struct mlx5_flow_rule *rule)
-{
- struct fs_base *pbase;
- struct mlx5_flow_group *fg;
-
- pbase = rule->base.parent;
- WARN_ON(!pbase);
- pbase = pbase->parent;
- WARN_ON(!pbase);
-
- fs_get_obj(fg, pbase);
- return fg->mask.match_criteria_enable;
-}
-
-void mlx5_get_match_value(u32 *match_value,
- struct mlx5_flow_rule *rule)
-{
- struct fs_base *pbase;
- struct fs_fte *fte;
-
- pbase = rule->base.parent;
- WARN_ON(!pbase);
- fs_get_obj(fte, pbase);
-
- memcpy(match_value, fte->val, sizeof(fte->val));
-}
-
-void mlx5_get_match_criteria(u32 *match_criteria,
- struct mlx5_flow_rule *rule)
-{
- struct fs_base *pbase;
- struct mlx5_flow_group *fg;
-
- pbase = rule->base.parent;
- WARN_ON(!pbase);
- pbase = pbase->parent;
- WARN_ON(!pbase);
-
- fs_get_obj(fg, pbase);
- memcpy(match_criteria, &fg->mask.match_criteria,
- sizeof(fg->mask.match_criteria));
-}
-
-int mlx5_init_fs(struct mlx5_core_dev *dev)
-{
- int err;
-
- if (MLX5_CAP_GEN(dev, nic_flow_table)) {
- err = init_root_ns(dev);
- if (err)
- goto err;
- }
-
- err = init_fdb_root_ns(dev);
- if (err)
- goto err;
-
- err = init_egress_acl_root_ns(dev);
- if (err)
- goto err;
-
- err = init_ingress_acl_root_ns(dev);
- if (err)
- goto err;
-
- err = init_sniffer_tx_root_ns(dev);
- if (err)
- goto err;
-
- err = init_sniffer_rx_root_ns(dev);
- if (err)
- goto err;
-
- err = mlx5_init_fc_stats(dev);
- if (err)
- goto err;
-
- return 0;
-err:
- mlx5_cleanup_fs(dev);
- return err;
-}
-
-struct mlx5_flow_namespace *mlx5_get_flow_namespace(struct mlx5_core_dev *dev,
- enum mlx5_flow_namespace_type type)
-{
- struct mlx5_flow_root_namespace *root_ns = dev->root_ns;
- int prio;
- static struct fs_prio *fs_prio;
- struct mlx5_flow_namespace *ns;
-
- switch (type) {
- case MLX5_FLOW_NAMESPACE_BYPASS:
- prio = 0;
- break;
- case MLX5_FLOW_NAMESPACE_OFFLOADS:
- prio = 1;
- break;
- case MLX5_FLOW_NAMESPACE_KERNEL:
- prio = 2;
- break;
- case MLX5_FLOW_NAMESPACE_LEFTOVERS:
- prio = 3;
- break;
- case MLX5_FLOW_NAMESPACE_FDB:
- if (dev->fdb_root_ns)
- return &dev->fdb_root_ns->ns;
- else
- return NULL;
- case MLX5_FLOW_NAMESPACE_ESW_EGRESS:
- if (dev->esw_egress_root_ns)
- return &dev->esw_egress_root_ns->ns;
- else
- return NULL;
- case MLX5_FLOW_NAMESPACE_ESW_INGRESS:
- if (dev->esw_ingress_root_ns)
- return &dev->esw_ingress_root_ns->ns;
- else
- return NULL;
- case MLX5_FLOW_NAMESPACE_SNIFFER_RX:
- if (dev->sniffer_rx_root_ns)
- return &dev->sniffer_rx_root_ns->ns;
- else
- return NULL;
- case MLX5_FLOW_NAMESPACE_SNIFFER_TX:
- if (dev->sniffer_tx_root_ns)
- return &dev->sniffer_tx_root_ns->ns;
- else
- return NULL;
- default:
- return NULL;
- }
-
- if (!root_ns)
- return NULL;
-
- fs_prio = find_prio(&root_ns->ns, prio);
- if (!fs_prio)
- return NULL;
-
- ns = list_first_entry(&fs_prio->objs,
- typeof(*ns),
- base.list);
-
- return ns;
-}
-EXPORT_SYMBOL(mlx5_get_flow_namespace);
-
-
-int mlx5_set_rule_private_data(struct mlx5_flow_rule *rule,
- struct mlx5_flow_handler *fs_handler,
- void *client_data)
-{
- struct fs_client_priv_data *priv_data;
-
- mutex_lock(&rule->clients_lock);
- /*Check that hanlder isn't exists in the list already*/
- list_for_each_entry(priv_data, &rule->clients_data, list) {
- if (priv_data->fs_handler == fs_handler) {
- priv_data->client_dst_data = client_data;
- goto unlock;
- }
- }
- priv_data = kzalloc(sizeof(*priv_data), GFP_KERNEL);
- if (!priv_data) {
- mutex_unlock(&rule->clients_lock);
- return -ENOMEM;
- }
-
- priv_data->client_dst_data = client_data;
- priv_data->fs_handler = fs_handler;
- list_add(&priv_data->list, &rule->clients_data);
-
-unlock:
- mutex_unlock(&rule->clients_lock);
-
- return 0;
-}
-
-static int remove_from_clients(struct mlx5_flow_rule *rule,
- bool ctx_changed,
- void *client_data,
- void *context)
-{
- struct fs_client_priv_data *iter_client;
- struct fs_client_priv_data *temp_client;
- struct mlx5_flow_handler *handler = (struct
- mlx5_flow_handler*)context;
-
- mutex_lock(&rule->clients_lock);
- list_for_each_entry_safe(iter_client, temp_client,
- &rule->clients_data, list) {
- if (iter_client->fs_handler == handler) {
- list_del(&iter_client->list);
- kfree(iter_client);
- break;
- }
- }
- mutex_unlock(&rule->clients_lock);
-
- return 0;
-}
-
-struct mlx5_flow_handler *mlx5_register_rule_notifier(struct mlx5_core_dev *dev,
- enum mlx5_flow_namespace_type ns_type,
- rule_event_fn add_cb,
- rule_event_fn del_cb,
- void *context)
-{
- struct mlx5_flow_namespace *ns;
- struct mlx5_flow_handler *handler;
-
- ns = mlx5_get_flow_namespace(dev, ns_type);
- if (!ns)
- return ERR_PTR(-EINVAL);
-
- handler = kzalloc(sizeof(*handler), GFP_KERNEL);
- if (!handler)
- return ERR_PTR(-ENOMEM);
-
- handler->add_dst_cb = add_cb;
- handler->del_dst_cb = del_cb;
- handler->client_context = context;
- handler->ns = ns;
- down_write(&ns->notifiers_rw_sem);
- list_add_tail(&handler->list, &ns->list_notifiers);
- up_write(&ns->notifiers_rw_sem);
-
- return handler;
-}
-
-static void iterate_rules_in_ns(struct mlx5_flow_namespace *ns,
- rule_event_fn add_rule_cb,
- void *context);
-
-void mlx5_unregister_rule_notifier(struct mlx5_flow_handler *handler)
-{
- struct mlx5_flow_namespace *ns = handler->ns;
-
- /*Remove from dst's clients*/
- down_write(&ns->dests_rw_sem);
- down_write(&ns->notifiers_rw_sem);
- iterate_rules_in_ns(ns, remove_from_clients, handler);
- list_del(&handler->list);
- up_write(&ns->notifiers_rw_sem);
- up_write(&ns->dests_rw_sem);
- kfree(handler);
-}
-
-static void iterate_rules_in_ft(struct mlx5_flow_table *ft,
- rule_event_fn add_rule_cb,
- void *context)
-{
- struct mlx5_flow_group *iter_fg;
- struct fs_fte *iter_fte;
- struct mlx5_flow_rule *iter_rule;
- int err = 0;
- bool is_new_rule;
-
- mutex_lock(&ft->base.lock);
- fs_for_each_fg(iter_fg, ft) {
- mutex_lock(&iter_fg->base.lock);
- fs_for_each_fte(iter_fte, iter_fg) {
- mutex_lock(&iter_fte->base.lock);
- is_new_rule = true;
- fs_for_each_dst(iter_rule, iter_fte) {
- fs_get(&iter_rule->base);
- err = add_rule_cb(iter_rule,
- is_new_rule,
- NULL,
- context);
- fs_put_parent_locked(&iter_rule->base);
- if (err)
- break;
- is_new_rule = false;
- }
- mutex_unlock(&iter_fte->base.lock);
- if (err)
- break;
- }
- mutex_unlock(&iter_fg->base.lock);
- if (err)
- break;
- }
- mutex_unlock(&ft->base.lock);
-}
-
-static void iterate_rules_in_prio(struct fs_prio *prio,
- rule_event_fn add_rule_cb,
- void *context)
-{
- struct fs_base *it;
-
- mutex_lock(&prio->base.lock);
- fs_for_each_ns_or_ft(it, prio) {
- if (it->type == FS_TYPE_FLOW_TABLE) {
- struct mlx5_flow_table *ft;
-
- fs_get_obj(ft, it);
- iterate_rules_in_ft(ft, add_rule_cb, context);
- } else {
- struct mlx5_flow_namespace *ns;
-
- fs_get_obj(ns, it);
- iterate_rules_in_ns(ns, add_rule_cb, context);
- }
- }
- mutex_unlock(&prio->base.lock);
-}
-
-static void iterate_rules_in_ns(struct mlx5_flow_namespace *ns,
- rule_event_fn add_rule_cb,
- void *context)
-{
- struct fs_prio *iter_prio;
-
- mutex_lock(&ns->base.lock);
- fs_for_each_prio(iter_prio, ns) {
- iterate_rules_in_prio(iter_prio, add_rule_cb, context);
- }
- mutex_unlock(&ns->base.lock);
-}
-
-void mlx5_flow_iterate_existing_rules(struct mlx5_flow_namespace *ns,
- rule_event_fn add_rule_cb,
- void *context)
-{
- down_write(&ns->dests_rw_sem);
- down_read(&ns->notifiers_rw_sem);
- iterate_rules_in_ns(ns, add_rule_cb, context);
- up_read(&ns->notifiers_rw_sem);
- up_write(&ns->dests_rw_sem);
-}
-
-
-void mlx5_del_flow_rules_list(struct mlx5_flow_rules_list *rules_list)
-{
- struct mlx5_flow_rule_node *iter_node;
- struct mlx5_flow_rule_node *temp_node;
-
- list_for_each_entry_safe(iter_node, temp_node, &rules_list->head, list) {
- list_del(&iter_node->list);
- kfree(iter_node);
- }
-
- kfree(rules_list);
-}
-
-#define ROCEV1_ETHERTYPE 0x8915
-static int set_rocev1_rules(struct list_head *rules_list)
-{
- struct mlx5_flow_rule_node *rocev1_rule;
-
- rocev1_rule = kzalloc(sizeof(*rocev1_rule), GFP_KERNEL);
- if (!rocev1_rule)
- return -ENOMEM;
-
- rocev1_rule->match_criteria_enable =
- 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS;
- MLX5_SET(fte_match_set_lyr_2_4, rocev1_rule->match_criteria, ethertype,
- 0xffff);
- MLX5_SET(fte_match_set_lyr_2_4, rocev1_rule->match_value, ethertype,
- ROCEV1_ETHERTYPE);
-
- list_add_tail(&rocev1_rule->list, rules_list);
-
- return 0;
-}
-
-#define ROCEV2_UDP_PORT 4791
-static int set_rocev2_rules(struct list_head *rules_list)
-{
- struct mlx5_flow_rule_node *ipv4_rule;
- struct mlx5_flow_rule_node *ipv6_rule;
-
- ipv4_rule = kzalloc(sizeof(*ipv4_rule), GFP_KERNEL);
- if (!ipv4_rule)
- return -ENOMEM;
-
- ipv6_rule = kzalloc(sizeof(*ipv6_rule), GFP_KERNEL);
- if (!ipv6_rule) {
- kfree(ipv4_rule);
- return -ENOMEM;
- }
-
- ipv4_rule->match_criteria_enable =
- 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS;
- MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_criteria, ethertype,
- 0xffff);
- MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_value, ethertype,
- 0x0800);
- MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_criteria, ip_protocol,
- 0xff);
- MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_value, ip_protocol,
- IPPROTO_UDP);
- MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_criteria, udp_dport,
- 0xffff);
- MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_value, udp_dport,
- ROCEV2_UDP_PORT);
-
- ipv6_rule->match_criteria_enable =
- 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS;
- MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_criteria, ethertype,
- 0xffff);
- MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_value, ethertype,
- 0x86dd);
- MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_criteria, ip_protocol,
- 0xff);
- MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_value, ip_protocol,
- IPPROTO_UDP);
- MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_criteria, udp_dport,
- 0xffff);
- MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_value, udp_dport,
- ROCEV2_UDP_PORT);
-
- list_add_tail(&ipv4_rule->list, rules_list);
- list_add_tail(&ipv6_rule->list, rules_list);
-
- return 0;
-}
-
-
-struct mlx5_flow_rules_list *get_roce_flow_rules(u8 roce_mode)
-{
- int err = 0;
- struct mlx5_flow_rules_list *rules_list =
- kzalloc(sizeof(*rules_list), GFP_KERNEL);
-
- if (!rules_list)
- return NULL;
-
- INIT_LIST_HEAD(&rules_list->head);
-
- if (roce_mode & MLX5_ROCE_VERSION_1_CAP) {
- err = set_rocev1_rules(&rules_list->head);
- if (err)
- goto free_list;
- }
- if (roce_mode & MLX5_ROCE_VERSION_2_CAP)
- err = set_rocev2_rules(&rules_list->head);
- if (err)
- goto free_list;
-
- return rules_list;
-
-free_list:
- mlx5_del_flow_rules_list(rules_list);
- return NULL;
-}
-
-struct mlx5_modify_hdr *mlx5_modify_header_alloc(struct mlx5_core_dev *dev,
- enum mlx5_flow_namespace_type ns_type,
- u8 num_actions,
- void *modify_actions)
-{
- struct mlx5_modify_hdr *modify_hdr;
- int err;
-
- modify_hdr = kzalloc(sizeof(*modify_hdr), GFP_KERNEL);
- if (!modify_hdr)
- return ERR_PTR(-ENOMEM);
-
- modify_hdr->ns_type = ns_type;
- err = mlx5_cmd_modify_header_alloc(dev, ns_type, num_actions,
- modify_actions, modify_hdr);
- if (err) {
- kfree(modify_hdr);
- return ERR_PTR(err);
- }
-
- return modify_hdr;
-}
-EXPORT_SYMBOL(mlx5_modify_header_alloc);
-
-void mlx5_modify_header_dealloc(struct mlx5_core_dev *dev,
- struct mlx5_modify_hdr *modify_hdr)
-{
- mlx5_cmd_modify_header_dealloc(dev, modify_hdr);
- kfree(modify_hdr);
-}
-EXPORT_SYMBOL(mlx5_modify_header_dealloc);
-
-struct mlx5_pkt_reformat *mlx5_packet_reformat_alloc(struct mlx5_core_dev *dev,
- struct mlx5_pkt_reformat_params *params,
- enum mlx5_flow_namespace_type ns_type)
-{
- struct mlx5_pkt_reformat *pkt_reformat;
- int err;
-
- pkt_reformat = kzalloc(sizeof(*pkt_reformat), GFP_KERNEL);
- if (!pkt_reformat)
- return ERR_PTR(-ENOMEM);
-
- pkt_reformat->ns_type = ns_type;
- pkt_reformat->reformat_type = params->type;
- err = mlx5_cmd_packet_reformat_alloc(dev, params, ns_type,
- pkt_reformat);
- if (err) {
- kfree(pkt_reformat);
- return ERR_PTR(err);
- }
-
- return pkt_reformat;
-}
-EXPORT_SYMBOL(mlx5_packet_reformat_alloc);
-
-void mlx5_packet_reformat_dealloc(struct mlx5_core_dev *dev,
- struct mlx5_pkt_reformat *pkt_reformat)
-{
- mlx5_cmd_packet_reformat_dealloc(dev, pkt_reformat);
- kfree(pkt_reformat);
-}
-EXPORT_SYMBOL(mlx5_packet_reformat_dealloc);
-
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fw.c b/sys/dev/mlx5/mlx5_core/mlx5_fw.c
--- a/sys/dev/mlx5/mlx5_core/mlx5_fw.c
+++ b/sys/dev/mlx5/mlx5_core/mlx5_fw.c
@@ -241,6 +241,12 @@
return err;
}
+ if (MLX5_CAP_GEN(dev, ipsec_offload)) {
+ err = mlx5_core_get_caps(dev, MLX5_CAP_IPSEC);
+ if (err)
+ return err;
+ }
+
err = mlx5_core_query_special_contexts(dev);
if (err)
return err;
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_main.c b/sys/dev/mlx5/mlx5_core/mlx5_main.c
--- a/sys/dev/mlx5/mlx5_core/mlx5_main.c
+++ b/sys/dev/mlx5/mlx5_core/mlx5_main.c
@@ -28,6 +28,7 @@
#include "opt_rss.h"
#include "opt_ratelimit.h"
+#include "opt_ipsec.h"
#include <linux/kmod.h>
#include <linux/module.h>
@@ -64,6 +65,9 @@
MODULE_DEPEND(mlx5, linuxkpi, 1, 1, 1);
MODULE_DEPEND(mlx5, mlxfw, 1, 1, 1);
MODULE_DEPEND(mlx5, firmware, 1, 1, 1);
+#ifdef IPSEC_ACCEL
+MODULE_DEPEND(mlx5, ipsec, 1, 1, 1);
+#endif
MODULE_VERSION(mlx5, 1);
SYSCTL_NODE(_hw, OID_AUTO, mlx5, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
@@ -1203,7 +1207,7 @@
goto err_stop_eqs;
}
- err = mlx5_init_fs(dev);
+ err = mlx5_fs_core_init(dev);
if (err) {
mlx5_core_err(dev, "flow steering init %d\n", err);
goto err_free_comp_eqs;
@@ -1317,7 +1321,7 @@
mlx5_eswitch_cleanup(dev->priv.eswitch);
mlx5_diag_cnt_cleanup(dev);
mlx5_mpfs_destroy(dev);
- mlx5_cleanup_fs(dev);
+ mlx5_fs_core_cleanup(dev);
mlx5_wait_for_reclaim_vfs_pages(dev);
free_comp_eqs(dev);
mlx5_stop_eqs(dev);
@@ -1680,10 +1684,17 @@
mlx5_pagealloc_init(dev);
+ pr_info("%s - MARK BLOCH WAS HERE\n", __func__);
+ err = mlx5_fs_core_alloc(dev);
+ if (err) {
+ mlx5_core_err(dev, "Failed to alloc flow steering\n");
+ goto clean_health;
+ }
+
err = mlx5_load_one(dev, priv, true);
if (err) {
mlx5_core_err(dev, "mlx5_load_one failed %d\n", err);
- goto clean_health;
+ goto clean_fs;
}
mlx5_fwdump_prep(dev);
@@ -1727,6 +1738,8 @@
pci_save_state(pdev);
return 0;
+clean_fs:
+ mlx5_fs_core_free(dev);
clean_health:
mlx5_pagealloc_cleanup(dev);
mlx5_health_cleanup(dev);
@@ -1758,6 +1771,7 @@
(long long)(dev->priv.fw_pages * MLX5_ADAPTER_PAGE_SIZE));
}
+ mlx5_fs_core_free(dev);
mlx5_pagealloc_cleanup(dev);
mlx5_health_cleanup(dev);
mlx5_fwdump_clean(dev);
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_tls.c b/sys/dev/mlx5/mlx5_core/mlx5_tls.c
--- a/sys/dev/mlx5/mlx5_core/mlx5_tls.c
+++ b/sys/dev/mlx5/mlx5_core/mlx5_tls.c
@@ -34,66 +34,6 @@
#include <dev/mlx5/mlx5_core/mlx5_core.h>
#include <dev/mlx5/mlx5_core/transobj.h>
-int mlx5_encryption_key_create(struct mlx5_core_dev *mdev, u32 pdn,
- const void *p_key, u32 key_len, u32 *p_obj_id)
-{
- u32 in[MLX5_ST_SZ_DW(create_encryption_key_in)] = {};
- u32 out[MLX5_ST_SZ_DW(create_encryption_key_out)] = {};
- u64 general_obj_types;
- int err;
-
- general_obj_types = MLX5_CAP_GEN_64(mdev, general_obj_types);
- if (!(general_obj_types & MLX5_HCA_CAP_GENERAL_OBJ_TYPES_ENCRYPTION_KEY))
- return -EINVAL;
-
- switch (key_len) {
- case 128 / 8:
- memcpy(MLX5_ADDR_OF(create_encryption_key_in, in,
- encryption_key_object.key[4]), p_key, 128 / 8);
- MLX5_SET(create_encryption_key_in, in, encryption_key_object.pd, pdn);
- MLX5_SET(create_encryption_key_in, in, encryption_key_object.key_size,
- MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_KEY_SIZE_128);
- MLX5_SET(create_encryption_key_in, in, encryption_key_object.key_type,
- MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_DEK);
- break;
- case 256 / 8:
- memcpy(MLX5_ADDR_OF(create_encryption_key_in, in,
- encryption_key_object.key[0]), p_key, 256 / 8);
- MLX5_SET(create_encryption_key_in, in, encryption_key_object.pd, pdn);
- MLX5_SET(create_encryption_key_in, in, encryption_key_object.key_size,
- MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_KEY_SIZE_256);
- MLX5_SET(create_encryption_key_in, in, encryption_key_object.key_type,
- MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_DEK);
- break;
- default:
- return -EINVAL;
- }
-
- MLX5_SET(create_encryption_key_in, in, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJ);
- MLX5_SET(create_encryption_key_in, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY);
-
- err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
- if (err == 0)
- *p_obj_id = MLX5_GET(create_encryption_key_out, out, obj_id);
-
- /* avoid leaking key on the stack */
- memset(in, 0, sizeof(in));
-
- return err;
-}
-
-int mlx5_encryption_key_destroy(struct mlx5_core_dev *mdev, u32 oid)
-{
- u32 in[MLX5_ST_SZ_DW(destroy_encryption_key_in)] = {};
- u32 out[MLX5_ST_SZ_DW(destroy_encryption_key_out)] = {};
-
- MLX5_SET(destroy_encryption_key_in, in, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJ);
- MLX5_SET(destroy_encryption_key_in, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY);
- MLX5_SET(destroy_encryption_key_in, in, obj_id, oid);
-
- return mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
-}
-
int mlx5_tls_open_tis(struct mlx5_core_dev *mdev, int tc, int tdn, int pdn, u32 *p_tisn)
{
u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {};
diff --git a/sys/dev/mlx5/mlx5_core/wq.h b/sys/dev/mlx5/mlx5_core/wq.h
--- a/sys/dev/mlx5/mlx5_core/wq.h
+++ b/sys/dev/mlx5/mlx5_core/wq.h
@@ -29,6 +29,7 @@
#define __MLX5_WQ_H__
#include <dev/mlx5/mlx5_ifc.h>
+#include <dev/mlx5/cq.h>
struct mlx5_wq_param {
int linear;
@@ -138,6 +139,22 @@
*wq->db = cpu_to_be32(wq->cc & 0xffffff);
}
+static inline struct mlx5_cqe64 *mlx5_cqwq_get_cqe(struct mlx5_cqwq *wq)
+{
+ u32 ci = mlx5_cqwq_get_ci(wq);
+ struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(wq, ci);
+ u8 cqe_ownership_bit = cqe->op_own & MLX5_CQE_OWNER_MASK;
+ u8 sw_ownership_val = mlx5_cqwq_get_wrap_cnt(wq) & 1;
+
+ if (cqe_ownership_bit != sw_ownership_val)
+ return NULL;
+
+ /* ensure cqe content is read after cqe ownership bit */
+ atomic_thread_fence_acq();
+
+ return cqe;
+}
+
static inline int mlx5_wq_ll_is_full(struct mlx5_wq_ll *wq)
{
return wq->cur_sz == wq->sz_m1;
diff --git a/sys/dev/mlx5/mlx5_en/en.h b/sys/dev/mlx5/mlx5_en/en.h
--- a/sys/dev/mlx5/mlx5_en/en.h
+++ b/sys/dev/mlx5/mlx5_en/en.h
@@ -72,6 +72,7 @@
#include <dev/mlx5/mlx5_core/wq.h>
#include <dev/mlx5/mlx5_core/transobj.h>
#include <dev/mlx5/mlx5_core/mlx5_core.h>
+#include <dev/mlx5/mlx5_accel/ipsec.h>
#define MLX5_SET_CFG(p, f, v) MLX5_SET(create_flow_group_in, p, f, v)
@@ -963,7 +964,7 @@
struct mlx5e_eth_addr_info {
u8 addr [ETH_ALEN + 2];
/* flow table rule per traffic type */
- struct mlx5_flow_rule *ft_rule[MLX5E_NUM_TT];
+ struct mlx5_flow_handle *ft_rule[MLX5E_NUM_TT];
};
#define MLX5E_ETH_ADDR_HASH_SIZE (1 << BITS_PER_BYTE)
@@ -999,10 +1000,10 @@
struct mlx5e_vlan_db {
unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)];
- struct mlx5_flow_rule *active_vlans_ft_rule[VLAN_N_VID];
- struct mlx5_flow_rule *untagged_ft_rule;
- struct mlx5_flow_rule *any_cvlan_ft_rule;
- struct mlx5_flow_rule *any_svlan_ft_rule;
+ struct mlx5_flow_handle *active_vlans_ft_rule[VLAN_N_VID];
+ struct mlx5_flow_handle *untagged_ft_rule;
+ struct mlx5_flow_handle *any_cvlan_ft_rule;
+ struct mlx5_flow_handle *any_svlan_ft_rule;
bool filter_disabled;
};
@@ -1011,7 +1012,7 @@
u_int proto;
u_int port;
bool installed;
- struct mlx5_flow_rule *vxlan_ft_rule;
+ struct mlx5_flow_handle *vxlan_ft_rule;
TAILQ_ENTRY(mlx5e_vxlan_db_el) link;
};
@@ -1034,19 +1035,20 @@
struct mlx5e_accel_fs_tcp {
struct mlx5_flow_namespace *ns;
struct mlx5e_flow_table tables[MLX5E_ACCEL_FS_TCP_NUM_TYPES];
- struct mlx5_flow_rule *default_rules[MLX5E_ACCEL_FS_TCP_NUM_TYPES];
+ struct mlx5_flow_handle *default_rules[MLX5E_ACCEL_FS_TCP_NUM_TYPES];
};
struct mlx5e_flow_tables {
struct mlx5_flow_namespace *ns;
struct mlx5e_flow_table vlan;
struct mlx5e_flow_table vxlan;
- struct mlx5_flow_rule *vxlan_catchall_ft_rule;
+ struct mlx5_flow_handle *vxlan_catchall_ft_rule;
struct mlx5e_flow_table main;
struct mlx5e_flow_table main_vxlan;
- struct mlx5_flow_rule *main_vxlan_rule[MLX5E_NUM_TT];
+ struct mlx5_flow_handle *main_vxlan_rule[MLX5E_NUM_TT];
struct mlx5e_flow_table inner_rss;
struct mlx5e_accel_fs_tcp accel_tcp;
+ struct mlx5_flow_table *ipsec_ft;
};
struct mlx5e_xmit_args {
@@ -1074,6 +1076,7 @@
u32 xoff;
};
+struct mlx5e_ipsec;
struct mlx5e_priv {
struct mlx5_core_dev *mdev; /* must be first */
@@ -1152,6 +1155,7 @@
bool sw_is_port_buf_owner;
struct pfil_head *pfil;
+ struct mlx5e_ipsec *ipsec;
struct mlx5e_channel channel[];
};
diff --git a/sys/dev/mlx5/mlx5_en/en_hw_tls_rx.h b/sys/dev/mlx5/mlx5_en/en_hw_tls_rx.h
--- a/sys/dev/mlx5/mlx5_en/en_hw_tls_rx.h
+++ b/sys/dev/mlx5/mlx5_en/en_hw_tls_rx.h
@@ -63,7 +63,7 @@
uint32_t tirn; /* HW TIR context number */
uint32_t dek_index; /* HW TLS context number */
struct mlx5e_tls_rx *tls_rx; /* parent pointer */
- struct mlx5_flow_rule *flow_rule;
+ struct mlx5_flow_handle *flow_rule;
struct mtx mtx;
struct completion progress_complete;
uint32_t state; /* see MLX5E_TLS_RX_ST_XXX */
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_flow_table.c b/sys/dev/mlx5/mlx5_en/mlx5_en_flow_table.c
--- a/sys/dev/mlx5/mlx5_en/mlx5_en_flow_table.c
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_flow_table.c
@@ -145,17 +145,17 @@
mlx5e_del_eth_addr_from_flow_table(struct mlx5e_priv *priv,
struct mlx5e_eth_addr_info *ai)
{
- mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV6_IPSEC_ESP]);
- mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV4_IPSEC_ESP]);
- mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV6_IPSEC_AH]);
- mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV4_IPSEC_AH]);
- mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV6_TCP]);
- mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV4_TCP]);
- mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV6_UDP]);
- mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV4_UDP]);
- mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV6]);
- mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_IPV4]);
- mlx5_del_flow_rule(&ai->ft_rule[MLX5E_TT_ANY]);
+ mlx5_del_flow_rules(&ai->ft_rule[MLX5E_TT_IPV6_IPSEC_ESP]);
+ mlx5_del_flow_rules(&ai->ft_rule[MLX5E_TT_IPV4_IPSEC_ESP]);
+ mlx5_del_flow_rules(&ai->ft_rule[MLX5E_TT_IPV6_IPSEC_AH]);
+ mlx5_del_flow_rules(&ai->ft_rule[MLX5E_TT_IPV4_IPSEC_AH]);
+ mlx5_del_flow_rules(&ai->ft_rule[MLX5E_TT_IPV6_TCP]);
+ mlx5_del_flow_rules(&ai->ft_rule[MLX5E_TT_IPV4_TCP]);
+ mlx5_del_flow_rules(&ai->ft_rule[MLX5E_TT_IPV6_UDP]);
+ mlx5_del_flow_rules(&ai->ft_rule[MLX5E_TT_IPV4_UDP]);
+ mlx5_del_flow_rules(&ai->ft_rule[MLX5E_TT_IPV6]);
+ mlx5_del_flow_rules(&ai->ft_rule[MLX5E_TT_IPV4]);
+ mlx5_del_flow_rules(&ai->ft_rule[MLX5E_TT_ANY]);
}
static int
@@ -250,24 +250,30 @@
static int
mlx5e_add_eth_addr_rule_sub(struct mlx5e_priv *priv,
struct mlx5e_eth_addr_info *ai, int type,
- u32 *mc, u32 *mv)
+ struct mlx5_flow_spec *spec)
{
struct mlx5_flow_destination dest = {};
u8 mc_enable = 0;
- struct mlx5_flow_rule **rule_p;
+ struct mlx5_flow_handle **rule_p;
struct mlx5_flow_table *ft = priv->fts.main.t;
- u8 *mc_dmac = MLX5_ADDR_OF(fte_match_param, mc,
+ u8 *mc_dmac = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
outer_headers.dmac_47_16);
- u8 *mv_dmac = MLX5_ADDR_OF(fte_match_param, mv,
+ u8 *mv_dmac = MLX5_ADDR_OF(fte_match_param, spec->match_value,
outer_headers.dmac_47_16);
u32 *tirn = priv->tirn;
u32 tt_vec;
int err = 0;
struct mlx5_flow_act flow_act = {
- .actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG,
- .flow_tag = MLX5_FS_ETH_FLOW_TAG,
+ .action = MLX5_FLOW_RULE_FWD_ACTION_DEST,
};
+ u8 *mc;
+ u8 *mv;
+ mv = (u8 *)spec->match_value;
+ mc = (u8 *)spec->match_criteria;
+
+ spec->flow_context.flow_tag = MLX5_FS_ETH_FLOW_TAG;
+ spec->flow_context.flags = FLOW_CONTEXT_HAS_TAG;
dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
switch (type) {
@@ -291,12 +297,11 @@
tt_vec = mlx5e_get_tt_vec(ai, type);
+ spec->match_criteria_enable = mc_enable;
if (tt_vec & BIT(MLX5E_TT_ANY)) {
rule_p = &ai->ft_rule[MLX5E_TT_ANY];
dest.tir_num = tirn[MLX5E_TT_ANY];
- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
- MLX5_FLOW_RULE_FWD_ACTION_DEST,
- &flow_act, &dest);
+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
if (IS_ERR_OR_NULL(*rule_p))
goto err_del_ai;
}
@@ -304,14 +309,13 @@
mc_enable = MLX5_MATCH_OUTER_HEADERS;
MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);
+ spec->match_criteria_enable = mc_enable;
if (tt_vec & BIT(MLX5E_TT_IPV4)) {
rule_p = &ai->ft_rule[MLX5E_TT_IPV4];
dest.tir_num = tirn[MLX5E_TT_IPV4];
MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
ETHERTYPE_IP);
- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
- MLX5_FLOW_RULE_FWD_ACTION_DEST,
- &flow_act, &dest);
+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
if (IS_ERR_OR_NULL(*rule_p))
goto err_del_ai;
}
@@ -321,9 +325,7 @@
dest.tir_num = tirn[MLX5E_TT_IPV6];
MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
ETHERTYPE_IPV6);
- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
- MLX5_FLOW_RULE_FWD_ACTION_DEST,
- &flow_act, &dest);
+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
if (IS_ERR_OR_NULL(*rule_p))
goto err_del_ai;
}
@@ -336,9 +338,7 @@
dest.tir_num = tirn[MLX5E_TT_IPV4_UDP];
MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
ETHERTYPE_IP);
- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
- MLX5_FLOW_RULE_FWD_ACTION_DEST,
- &flow_act, &dest);
+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
if (IS_ERR_OR_NULL(*rule_p))
goto err_del_ai;
}
@@ -348,9 +348,7 @@
dest.tir_num = tirn[MLX5E_TT_IPV6_UDP];
MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
ETHERTYPE_IPV6);
- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
- MLX5_FLOW_RULE_FWD_ACTION_DEST,
- &flow_act, &dest);
+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
if (IS_ERR_OR_NULL(*rule_p))
goto err_del_ai;
}
@@ -362,9 +360,7 @@
dest.tir_num = tirn[MLX5E_TT_IPV4_TCP];
MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
ETHERTYPE_IP);
- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
- MLX5_FLOW_RULE_FWD_ACTION_DEST,
- &flow_act, &dest);
+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
if (IS_ERR_OR_NULL(*rule_p))
goto err_del_ai;
}
@@ -374,9 +370,7 @@
dest.tir_num = tirn[MLX5E_TT_IPV6_TCP];
MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
ETHERTYPE_IPV6);
- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
- MLX5_FLOW_RULE_FWD_ACTION_DEST,
- &flow_act, &dest);
+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
if (IS_ERR_OR_NULL(*rule_p))
goto err_del_ai;
}
@@ -388,9 +382,7 @@
dest.tir_num = tirn[MLX5E_TT_IPV4_IPSEC_AH];
MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
ETHERTYPE_IP);
- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
- MLX5_FLOW_RULE_FWD_ACTION_DEST,
- &flow_act, &dest);
+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
if (IS_ERR_OR_NULL(*rule_p))
goto err_del_ai;
}
@@ -400,9 +392,7 @@
dest.tir_num = tirn[MLX5E_TT_IPV6_IPSEC_AH];
MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
ETHERTYPE_IPV6);
- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
- MLX5_FLOW_RULE_FWD_ACTION_DEST,
- &flow_act, &dest);
+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
if (IS_ERR_OR_NULL(*rule_p))
goto err_del_ai;
}
@@ -414,9 +404,7 @@
dest.tir_num = tirn[MLX5E_TT_IPV4_IPSEC_ESP];
MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
ETHERTYPE_IP);
- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
- MLX5_FLOW_RULE_FWD_ACTION_DEST,
- &flow_act, &dest);
+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
if (IS_ERR_OR_NULL(*rule_p))
goto err_del_ai;
}
@@ -426,9 +414,7 @@
dest.tir_num = tirn[MLX5E_TT_IPV6_IPSEC_ESP];
MLX5_SET(fte_match_param, mv, outer_headers.ethertype,
ETHERTYPE_IPV6);
- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
- MLX5_FLOW_RULE_FWD_ACTION_DEST,
- &flow_act, &dest);
+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
if (IS_ERR_OR_NULL(*rule_p))
goto err_del_ai;
}
@@ -447,23 +433,19 @@
mlx5e_add_eth_addr_rule(struct mlx5e_priv *priv,
struct mlx5e_eth_addr_info *ai, int type)
{
- u32 *match_criteria;
- u32 *match_value;
+ struct mlx5_flow_spec *spec;
int err = 0;
- match_value = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
- match_criteria = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
- if (!match_value || !match_criteria) {
+ spec = mlx5_vzalloc(sizeof(*spec));
+ if (!spec) {
mlx5_en_err(priv->ifp, "alloc failed\n");
err = -ENOMEM;
goto add_eth_addr_rule_out;
}
- err = mlx5e_add_eth_addr_rule_sub(priv, ai, type, match_criteria,
- match_value);
+ err = mlx5e_add_eth_addr_rule_sub(priv, ai, type, spec);
add_eth_addr_rule_out:
- kvfree(match_criteria);
- kvfree(match_value);
+ kvfree(spec);
return (err);
}
@@ -471,51 +453,56 @@
static void
mlx5e_del_main_vxlan_rules(struct mlx5e_priv *priv)
{
- mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_IPSEC_ESP]);
- mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_IPSEC_ESP]);
- mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_IPSEC_AH]);
- mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_IPSEC_AH]);
- mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_TCP]);
- mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_TCP]);
- mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_UDP]);
- mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_UDP]);
- mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6]);
- mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4]);
- mlx5_del_flow_rule(&priv->fts.main_vxlan_rule[MLX5E_TT_ANY]);
+ mlx5_del_flow_rules(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_IPSEC_ESP]);
+ mlx5_del_flow_rules(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_IPSEC_ESP]);
+ mlx5_del_flow_rules(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_IPSEC_AH]);
+ mlx5_del_flow_rules(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_IPSEC_AH]);
+ mlx5_del_flow_rules(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_TCP]);
+ mlx5_del_flow_rules(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_TCP]);
+ mlx5_del_flow_rules(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_UDP]);
+ mlx5_del_flow_rules(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_UDP]);
+ mlx5_del_flow_rules(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV6]);
+ mlx5_del_flow_rules(&priv->fts.main_vxlan_rule[MLX5E_TT_IPV4]);
+ mlx5_del_flow_rules(&priv->fts.main_vxlan_rule[MLX5E_TT_ANY]);
}
static int
-mlx5e_add_main_vxlan_rules_sub(struct mlx5e_priv *priv, u32 *mc, u32 *mv)
+mlx5e_add_main_vxlan_rules_sub(struct mlx5e_priv *priv,
+ struct mlx5_flow_spec *spec)
{
struct mlx5_flow_destination dest = {};
- u8 mc_enable = 0;
- struct mlx5_flow_rule **rule_p;
+ struct mlx5_flow_handle **rule_p;
struct mlx5_flow_table *ft = priv->fts.main_vxlan.t;
u32 *tirn = priv->tirn_inner_vxlan;
struct mlx5_flow_act flow_act = {
- .actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG,
- .flow_tag = MLX5_FS_ETH_FLOW_TAG,
+ .action = MLX5_FLOW_RULE_FWD_ACTION_DEST,
};
int err = 0;
+ u8 *mc;
+ u8 *mv;
+
+ spec->flow_context.flags = FLOW_CONTEXT_HAS_TAG;
+ spec->flow_context.flow_tag = MLX5_FS_ETH_FLOW_TAG;
+
+ mc = (u8 *)spec->match_criteria;
+ mv = (u8 *)spec->match_value;
dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
- mc_enable = MLX5_MATCH_INNER_HEADERS;
+ spec->match_criteria_enable = MLX5_MATCH_INNER_HEADERS;
MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.ethertype);
rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV4];
dest.tir_num = tirn[MLX5E_TT_IPV4];
MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IP);
- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
- MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
if (IS_ERR_OR_NULL(*rule_p))
goto err_del_ai;
rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV6];
dest.tir_num = tirn[MLX5E_TT_IPV6];
MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IPV6);
- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
- MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
if (IS_ERR_OR_NULL(*rule_p))
goto err_del_ai;
@@ -525,16 +512,14 @@
rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_UDP];
dest.tir_num = tirn[MLX5E_TT_IPV4_UDP];
MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IP);
- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
- MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
if (IS_ERR_OR_NULL(*rule_p))
goto err_del_ai;
rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_UDP];
dest.tir_num = tirn[MLX5E_TT_IPV6_UDP];
MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IPV6);
- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
- MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
if (IS_ERR_OR_NULL(*rule_p))
goto err_del_ai;
@@ -543,16 +528,14 @@
rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_TCP];
dest.tir_num = tirn[MLX5E_TT_IPV4_TCP];
MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IP);
- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
- MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
if (IS_ERR_OR_NULL(*rule_p))
goto err_del_ai;
rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_TCP];
dest.tir_num = tirn[MLX5E_TT_IPV6_TCP];
MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IPV6);
- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
- MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
if (IS_ERR_OR_NULL(*rule_p))
goto err_del_ai;
@@ -561,16 +544,14 @@
rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_IPSEC_AH];
dest.tir_num = tirn[MLX5E_TT_IPV4_IPSEC_AH];
MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IP);
- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
- MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
if (IS_ERR_OR_NULL(*rule_p))
goto err_del_ai;
rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV6_IPSEC_AH];
dest.tir_num = tirn[MLX5E_TT_IPV6_IPSEC_AH];
MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IPV6);
- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
- MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
if (IS_ERR_OR_NULL(*rule_p))
goto err_del_ai;
@@ -579,8 +560,7 @@
rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_IPV4_IPSEC_ESP];
dest.tir_num = tirn[MLX5E_TT_IPV4_IPSEC_ESP];
MLX5_SET(fte_match_param, mv, inner_headers.ethertype, ETHERTYPE_IP);
- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
- MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
if (IS_ERR_OR_NULL(*rule_p))
goto err_del_ai;
@@ -588,18 +568,16 @@
dest.tir_num = tirn[MLX5E_TT_IPV6_IPSEC_ESP];
MLX5_SET(fte_match_param, mv, inner_headers.ethertype,
ETHERTYPE_IPV6);
- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
- MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
if (IS_ERR_OR_NULL(*rule_p))
goto err_del_ai;
- mc_enable = 0;
+ spec->match_criteria_enable = 0;
memset(mv, 0, MLX5_ST_SZ_BYTES(fte_match_param));
memset(mc, 0, MLX5_ST_SZ_BYTES(fte_match_param));
rule_p = &priv->fts.main_vxlan_rule[MLX5E_TT_ANY];
dest.tir_num = tirn[MLX5E_TT_ANY];
- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
- MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
+ *rule_p = mlx5_add_flow_rules(ft, NULL, &flow_act, &dest, 1);
if (IS_ERR_OR_NULL(*rule_p))
goto err_del_ai;
@@ -616,22 +594,19 @@
static int
mlx5e_add_main_vxlan_rules(struct mlx5e_priv *priv)
{
- u32 *match_criteria;
- u32 *match_value;
+ struct mlx5_flow_spec *spec;
int err = 0;
- match_value = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
- match_criteria = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
- if (match_value == NULL || match_criteria == NULL) {
+ spec = mlx5_vzalloc(sizeof(*spec));
+ if (!spec) {
mlx5_en_err(priv->ifp, "alloc failed\n");
err = -ENOMEM;
goto add_main_vxlan_rules_out;
}
- err = mlx5e_add_main_vxlan_rules_sub(priv, match_criteria, match_value);
+ err = mlx5e_add_main_vxlan_rules_sub(priv, spec);
add_main_vxlan_rules_out:
- kvfree(match_criteria);
- kvfree(match_value);
+ kvfree(spec);
return (err);
}
@@ -689,22 +664,27 @@
static int
mlx5e_add_vlan_rule_sub(struct mlx5e_priv *priv,
enum mlx5e_vlan_rule_type rule_type, u16 vid,
- u32 *mc, u32 *mv)
+ struct mlx5_flow_spec *spec)
{
struct mlx5_flow_table *ft = priv->fts.vlan.t;
struct mlx5_flow_destination dest = {};
- u8 mc_enable = 0;
- struct mlx5_flow_rule **rule_p;
+ struct mlx5_flow_handle **rule_p;
int err = 0;
struct mlx5_flow_act flow_act = {
- .actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG,
- .flow_tag = MLX5_FS_ETH_FLOW_TAG,
+ .action = MLX5_FLOW_RULE_FWD_ACTION_DEST,
};
+ u8 *mv;
+ u8 *mc;
+ mv = (u8 *)spec->match_value;
+ mc = (u8 *)spec->match_criteria;
+
+ spec->flow_context.flow_tag = MLX5_FS_ETH_FLOW_TAG;
+ spec->flow_context.flags = FLOW_CONTEXT_HAS_TAG;
dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
dest.ft = priv->fts.vxlan.t;
- mc_enable = MLX5_MATCH_OUTER_HEADERS;
+ spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
switch (rule_type) {
case MLX5E_VLAN_RULE_TYPE_UNTAGGED:
@@ -731,11 +711,7 @@
break;
}
- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
- MLX5_FLOW_RULE_FWD_ACTION_DEST,
- &flow_act,
- &dest);
-
+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
if (IS_ERR(*rule_p)) {
err = PTR_ERR(*rule_p);
*rule_p = NULL;
@@ -749,24 +725,20 @@
mlx5e_add_vlan_rule(struct mlx5e_priv *priv,
enum mlx5e_vlan_rule_type rule_type, u16 vid)
{
- u32 *match_criteria;
- u32 *match_value;
+ struct mlx5_flow_spec *spec;
int err = 0;
- match_value = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
- match_criteria = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
- if (!match_value || !match_criteria) {
+ spec = mlx5_vzalloc(sizeof(*spec));
+ if (!spec) {
mlx5_en_err(priv->ifp, "alloc failed\n");
err = -ENOMEM;
goto add_vlan_rule_out;
}
- err = mlx5e_add_vlan_rule_sub(priv, rule_type, vid, match_criteria,
- match_value);
+ err = mlx5e_add_vlan_rule_sub(priv, rule_type, vid, spec);
add_vlan_rule_out:
- kvfree(match_criteria);
- kvfree(match_value);
+ kvfree(spec);
return (err);
}
@@ -777,16 +749,16 @@
{
switch (rule_type) {
case MLX5E_VLAN_RULE_TYPE_UNTAGGED:
- mlx5_del_flow_rule(&priv->vlan.untagged_ft_rule);
+ mlx5_del_flow_rules(&priv->vlan.untagged_ft_rule);
break;
case MLX5E_VLAN_RULE_TYPE_ANY_CTAG_VID:
- mlx5_del_flow_rule(&priv->vlan.any_cvlan_ft_rule);
+ mlx5_del_flow_rules(&priv->vlan.any_cvlan_ft_rule);
break;
case MLX5E_VLAN_RULE_TYPE_ANY_STAG_VID:
- mlx5_del_flow_rule(&priv->vlan.any_svlan_ft_rule);
+ mlx5_del_flow_rules(&priv->vlan.any_svlan_ft_rule);
break;
case MLX5E_VLAN_RULE_TYPE_MATCH_VID:
- mlx5_del_flow_rule(&priv->vlan.active_vlans_ft_rule[vid]);
+ mlx5_del_flow_rules(&priv->vlan.active_vlans_ft_rule[vid]);
mlx5e_vport_context_update_vlans(priv);
break;
default:
@@ -1520,11 +1492,16 @@
{
struct mlx5e_flow_table *ft = inner_vxlan ? &priv->fts.main_vxlan :
&priv->fts.main;
+ struct mlx5_flow_table_attr ft_attr = {};
int err;
ft->num_groups = 0;
- ft->t = mlx5_create_flow_table(priv->fts.ns, 0,
- inner_vxlan ? "vxlan_main" : "main", MLX5E_MAIN_TABLE_SIZE);
+ ft_attr.max_fte = MLX5E_MAIN_TABLE_SIZE;
+ if (priv->ipsec)
+ ft_attr.level = inner_vxlan ? 10 : 12;
+ else
+ ft_attr.level = inner_vxlan ? 2 : 4;
+ ft->t = mlx5_create_flow_table(priv->fts.ns, &ft_attr);
if (IS_ERR(ft->t)) {
err = PTR_ERR(ft->t);
@@ -1645,11 +1622,13 @@
mlx5e_create_vlan_flow_table(struct mlx5e_priv *priv)
{
struct mlx5e_flow_table *ft = &priv->fts.vlan;
+ struct mlx5_flow_table_attr ft_attr = {};
int err;
ft->num_groups = 0;
- ft->t = mlx5_create_flow_table(priv->fts.ns, 0, "vlan",
- MLX5E_VLAN_TABLE_SIZE);
+ ft_attr.max_fte = MLX5E_VLAN_TABLE_SIZE;
+ ft_attr.level = (priv->ipsec) ? 8 : 0;
+ ft->t = mlx5_create_flow_table(priv->fts.ns, &ft_attr);
if (IS_ERR(ft->t)) {
err = PTR_ERR(ft->t);
@@ -1685,23 +1664,29 @@
}
static int
-mlx5e_add_vxlan_rule_sub(struct mlx5e_priv *priv, u32 *mc, u32 *mv,
+mlx5e_add_vxlan_rule_sub(struct mlx5e_priv *priv, struct mlx5_flow_spec *spec,
struct mlx5e_vxlan_db_el *el)
{
struct mlx5_flow_table *ft = priv->fts.vxlan.t;
struct mlx5_flow_destination dest = {};
- u8 mc_enable;
- struct mlx5_flow_rule **rule_p;
+ struct mlx5_flow_handle **rule_p;
int err = 0;
struct mlx5_flow_act flow_act = {
- .actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG,
- .flow_tag = MLX5_FS_ETH_FLOW_TAG,
+ .action = MLX5_FLOW_RULE_FWD_ACTION_DEST,
};
+ u8 *mc;
+ u8 *mv;
+
+ mv = (u8 *)spec->match_value;
+ mc = (u8 *)spec->match_criteria;
+
+ spec->flow_context.flow_tag = MLX5_FS_ETH_FLOW_TAG;
+ spec->flow_context.flags = FLOW_CONTEXT_HAS_TAG;
dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
dest.ft = priv->fts.main_vxlan.t;
- mc_enable = MLX5_MATCH_OUTER_HEADERS;
+ spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
rule_p = &el->vxlan_ft_rule;
MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);
MLX5_SET(fte_match_param, mv, outer_headers.ethertype, el->proto);
@@ -1710,8 +1695,7 @@
MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.udp_dport);
MLX5_SET(fte_match_param, mv, outer_headers.udp_dport, el->port);
- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
- MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
if (IS_ERR(*rule_p)) {
err = PTR_ERR(*rule_p);
@@ -1766,23 +1750,20 @@
mlx5e_add_vxlan_rule_from_db(struct mlx5e_priv *priv,
struct mlx5e_vxlan_db_el *el)
{
- u32 *match_criteria;
- u32 *match_value;
+ struct mlx5_flow_spec *spec;
int err;
- match_value = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
- match_criteria = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
- if (match_value == NULL || match_criteria == NULL) {
+ spec = mlx5_vzalloc(sizeof(*spec));
+ if (!spec) {
mlx5_en_err(priv->ifp, "alloc failed\n");
err = -ENOMEM;
goto add_vxlan_rule_out;
}
- err = mlx5e_add_vxlan_rule_sub(priv, match_criteria, match_value, el);
+ err = mlx5e_add_vxlan_rule_sub(priv, spec, el);
add_vxlan_rule_out:
- kvfree(match_criteria);
- kvfree(match_value);
+ kvfree(spec);
return (err);
}
@@ -1820,24 +1801,25 @@
}
static int
-mlx5e_add_vxlan_catchall_rule_sub(struct mlx5e_priv *priv, u32 *mc, u32 *mv)
+mlx5e_add_vxlan_catchall_rule_sub(struct mlx5e_priv *priv,
+ struct mlx5_flow_spec *spec)
{
struct mlx5_flow_table *ft = priv->fts.vxlan.t;
struct mlx5_flow_destination dest = {};
- u8 mc_enable = 0;
- struct mlx5_flow_rule **rule_p;
+ struct mlx5_flow_handle **rule_p;
int err = 0;
struct mlx5_flow_act flow_act = {
- .actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG,
- .flow_tag = MLX5_FS_ETH_FLOW_TAG,
+ .action = MLX5_FLOW_RULE_FWD_ACTION_DEST,
};
+ spec->flow_context.flow_tag = MLX5_FS_ETH_FLOW_TAG;
+ spec->flow_context.flags = FLOW_CONTEXT_HAS_TAG;
+
dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
dest.ft = priv->fts.main.t;
rule_p = &priv->fts.vxlan_catchall_ft_rule;
- *rule_p = mlx5_add_flow_rule(ft, mc_enable, mc, mv,
- MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
+ *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
if (IS_ERR(*rule_p)) {
err = PTR_ERR(*rule_p);
@@ -1852,24 +1834,20 @@
static int
mlx5e_add_vxlan_catchall_rule(struct mlx5e_priv *priv)
{
- u32 *match_criteria;
- u32 *match_value;
+ struct mlx5_flow_spec *spec;
int err;
- match_value = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
- match_criteria = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
- if (match_value == NULL || match_criteria == NULL) {
+ spec = mlx5_vzalloc(sizeof(*spec));
+ if (!spec) {
mlx5_en_err(priv->ifp, "alloc failed\n");
err = -ENOMEM;
goto add_vxlan_rule_out;
}
- err = mlx5e_add_vxlan_catchall_rule_sub(priv, match_criteria,
- match_value);
+ err = mlx5e_add_vxlan_catchall_rule_sub(priv, spec);
add_vxlan_rule_out:
- kvfree(match_criteria);
- kvfree(match_value);
+ kvfree(spec);
return (err);
}
@@ -1913,7 +1891,7 @@
}
if (el->installed)
- mlx5_del_flow_rule(&el->vxlan_ft_rule);
+ mlx5_del_flow_rules(&el->vxlan_ft_rule);
TAILQ_REMOVE(&priv->vxlan.head, el, link);
kvfree(el);
return (0);
@@ -1927,7 +1905,7 @@
TAILQ_FOREACH(el, &priv->vxlan.head, link) {
if (!el->installed)
continue;
- mlx5_del_flow_rule(&el->vxlan_ft_rule);
+ mlx5_del_flow_rules(&el->vxlan_ft_rule);
el->installed = false;
}
}
@@ -1935,7 +1913,7 @@
static void
mlx5e_del_vxlan_catchall_rule(struct mlx5e_priv *priv)
{
- mlx5_del_flow_rule(&priv->fts.vxlan_catchall_ft_rule);
+ mlx5_del_flow_rules(&priv->fts.vxlan_catchall_ft_rule);
}
void
@@ -2032,11 +2010,13 @@
mlx5e_create_vxlan_flow_table(struct mlx5e_priv *priv)
{
struct mlx5e_flow_table *ft = &priv->fts.vxlan;
+ struct mlx5_flow_table_attr ft_attr = {};
int err;
ft->num_groups = 0;
- ft->t = mlx5_create_flow_table(priv->fts.ns, 0, "vxlan",
- MLX5E_VXLAN_TABLE_SIZE);
+ ft_attr.max_fte = MLX5E_VXLAN_TABLE_SIZE;
+ ft_attr.level = (priv->ipsec) ? 9 : 1;
+ ft->t = mlx5_create_flow_table(priv->fts.ns, &ft_attr);
if (IS_ERR(ft->t)) {
err = PTR_ERR(ft->t);
@@ -2146,11 +2126,13 @@
mlx5e_create_inner_rss_flow_table(struct mlx5e_priv *priv)
{
struct mlx5e_flow_table *ft = &priv->fts.inner_rss;
+ struct mlx5_flow_table_attr ft_attr = {};
int err;
ft->num_groups = 0;
- ft->t = mlx5_create_flow_table(priv->fts.ns, 0, "inner_rss",
- MLX5E_INNER_RSS_TABLE_SIZE);
+ ft_attr.max_fte = MLX5E_INNER_RSS_TABLE_SIZE;
+ ft_attr.level = (priv->ipsec) ? 11 : 3;
+ ft->t = mlx5_create_flow_table(priv->fts.ns, &ft_attr);
if (IS_ERR(ft->t)) {
err = PTR_ERR(ft->t);
@@ -2200,9 +2182,13 @@
priv->fts.ns = mlx5_get_flow_namespace(
priv->mdev, MLX5_FLOW_NAMESPACE_KERNEL);
+ err = mlx5e_accel_ipsec_fs_rx_tables_create(priv);
+ if (err)
+ return err;
+
err = mlx5e_create_vlan_flow_table(priv);
if (err)
- return (err);
+ goto err_destroy_ipsec_flow_table;
err = mlx5e_create_vxlan_flow_table(priv);
if (err)
@@ -2224,13 +2210,19 @@
if (err)
goto err_destroy_main_flow_table_false;
+ err = mlx5e_accel_ipsec_fs_rx_catchall_rules(priv);
+ if (err)
+ goto err_destroy_vxlan_catchall_rule;
+
err = mlx5e_accel_fs_tcp_create(priv);
if (err)
- goto err_del_vxlan_catchall_rule;
+ goto err_destroy_ipsec_catchall_rules;
return (0);
-err_del_vxlan_catchall_rule:
+err_destroy_ipsec_catchall_rules:
+ mlx5e_accel_ipsec_fs_rx_catchall_rules_destroy(priv);
+err_destroy_vxlan_catchall_rule:
mlx5e_del_vxlan_catchall_rule(priv);
err_destroy_main_flow_table_false:
mlx5e_destroy_main_flow_table(priv);
@@ -2242,6 +2234,8 @@
mlx5e_destroy_vxlan_flow_table(priv);
err_destroy_vlan_flow_table:
mlx5e_destroy_vlan_flow_table(priv);
+err_destroy_ipsec_flow_table:
+ mlx5e_accel_ipsec_fs_rx_tables_destroy(priv);
return (err);
}
@@ -2250,12 +2244,14 @@
mlx5e_close_flow_tables(struct mlx5e_priv *priv)
{
mlx5e_accel_fs_tcp_destroy(priv);
+ mlx5e_accel_ipsec_fs_rx_catchall_rules_destroy(priv);
mlx5e_del_vxlan_catchall_rule(priv);
mlx5e_destroy_main_flow_table(priv);
mlx5e_destroy_inner_rss_flow_table(priv);
mlx5e_destroy_main_vxlan_flow_table(priv);
mlx5e_destroy_vxlan_flow_table(priv);
mlx5e_destroy_vlan_flow_table(priv);
+ mlx5e_accel_ipsec_fs_rx_tables_destroy(priv);
}
int
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_hw_tls.c b/sys/dev/mlx5/mlx5_en/mlx5_en_hw_tls.c
--- a/sys/dev/mlx5/mlx5_en/mlx5_en_hw_tls.c
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_hw_tls.c
@@ -33,6 +33,7 @@
#include <dev/mlx5/mlx5_en/en.h>
#include <dev/mlx5/tls.h>
+#include <dev/mlx5/crypto.h>
#include <linux/delay.h>
#include <sys/ktls.h>
@@ -239,6 +240,7 @@
/* try to allocate a DEK context ID */
err = mlx5_encryption_key_create(priv->mdev, priv->pdn,
+ MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_TLS,
MLX5_ADDR_OF(sw_tls_cntx, ptag->crypto_params, key.key_data),
MLX5_GET(sw_tls_cntx, ptag->crypto_params, key.key_len),
&ptag->dek_index);
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_hw_tls_rx.c b/sys/dev/mlx5/mlx5_en/mlx5_en_hw_tls_rx.c
--- a/sys/dev/mlx5/mlx5_en/mlx5_en_hw_tls_rx.c
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_hw_tls_rx.c
@@ -31,6 +31,7 @@
#include <dev/mlx5/mlx5_en/en.h>
+#include <dev/mlx5/crypto.h>
#include <dev/mlx5/tls.h>
#include <dev/mlx5/fs.h>
@@ -553,6 +554,7 @@
/* try to allocate a DEK context ID */
err = mlx5_encryption_key_create(priv->mdev, priv->pdn,
+ MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_TLS,
MLX5_ADDR_OF(sw_tls_rx_cntx, ptag->crypto_params, key.key_data),
MLX5_GET(sw_tls_rx_cntx, ptag->crypto_params, key.key_len),
&ptag->dek_index);
@@ -661,7 +663,7 @@
struct mlx5e_iq *iq;
struct mlx5e_priv *priv;
struct mlx5e_tls_rx_tag *ptag;
- struct mlx5_flow_rule *flow_rule;
+ struct mlx5_flow_handle *flow_rule;
const struct tls_session_params *en;
uint32_t value;
int error;
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_main.c b/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
--- a/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
@@ -26,6 +26,7 @@
* $FreeBSD$
*/
+#include "opt_ipsec.h"
#include "opt_kern_tls.h"
#include "opt_rss.h"
#include "opt_ratelimit.h"
@@ -37,6 +38,8 @@
#include <machine/atomic.h>
#include <net/debugnet.h>
+#include <netipsec/keydb.h>
+#include <netipsec/ipsec_accel.h>
static int mlx5e_get_wqe_sz(struct mlx5e_priv *priv, u32 *wqe_sz, u32 *nsegs);
static if_snd_tag_query_t mlx5e_ul_snd_tag_query;
@@ -4449,6 +4452,10 @@
if_setcapabilitiesbit(ifp, IFCAP_VXLAN_HWCSUM | IFCAP_VXLAN_HWTSO, 0);
if_setcapabilities2bit(ifp, IFCAP2_BIT(IFCAP2_RXTLS4) |
IFCAP2_BIT(IFCAP2_RXTLS6), 0);
+
+ if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PACKET_OFFLOAD)
+ if_setcapabilities2bit(ifp, IFCAP2_BIT(IFCAP2_IPSEC_ACCEL) , 0);
+
if_setsndtagallocfn(ifp, mlx5e_snd_tag_alloc);
#ifdef RATELIMIT
if_setratelimitqueryfn(ifp, mlx5e_rl_query);
@@ -4548,10 +4555,16 @@
goto err_rl_init;
}
+ err = mlx5e_ipsec_init(priv);
+ if (err) {
+ if_printf(ifp, "%s: mlx5e_tls_init failed\n", __func__);
+ goto err_tls_init;
+ }
+
err = mlx5e_open_drop_rq(priv, &priv->drop_rq);
if (err) {
if_printf(ifp, "%s: mlx5e_open_drop_rq failed (%d)\n", __func__, err);
- goto err_tls_init;
+ goto err_ipsec_init;
}
err = mlx5e_open_rqts(priv);
@@ -4727,6 +4740,9 @@
err_open_drop_rq:
mlx5e_close_drop_rq(&priv->drop_rq);
+err_ipsec_init:
+ mlx5e_ipsec_cleanup(priv);
+
err_tls_init:
mlx5e_tls_cleanup(priv);
@@ -4835,10 +4851,14 @@
ether_ifdetach(ifp);
mlx5e_tls_rx_cleanup(priv);
+#ifdef IPSEC_ACCEL
+ ipsec_accel_on_ifdown(priv->ifp);
+#endif
mlx5e_close_flow_tables(priv);
mlx5e_close_tirs(priv);
mlx5e_close_rqts(priv);
mlx5e_close_drop_rq(&priv->drop_rq);
+ mlx5e_ipsec_cleanup(priv);
mlx5e_tls_cleanup(priv);
mlx5e_rl_cleanup(priv);
@@ -4953,6 +4973,7 @@
module_init_order(mlx5e_init, SI_ORDER_SIXTH);
module_exit_order(mlx5e_cleanup, SI_ORDER_SIXTH);
+MODULE_DEPEND(mlx5en, ipsec, 1, 1, 1);
MODULE_DEPEND(mlx5en, linuxkpi, 1, 1, 1);
MODULE_DEPEND(mlx5en, mlx5, 1, 1, 1);
MODULE_VERSION(mlx5en, 1);
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c b/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
--- a/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
@@ -30,6 +30,7 @@
#include <dev/mlx5/mlx5_en/en.h>
#include <machine/in_cksum.h>
+#include <dev/mlx5/mlx5_accel/ipsec.h>
static inline int
mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq,
@@ -71,6 +72,9 @@
/* get IP header aligned */
m_adj(mb, MLX5E_NET_IP_ALIGN);
+ err = mlx5_accel_ipsec_rx_tag_add(rq->ifp, mb);
+ if (err)
+ goto err_free_mbuf;
err = -bus_dmamap_load_mbuf_sg(rq->dma_tag, rq->mbuf[ix].dma_map,
mb, segs, &nsegs, BUS_DMA_NOWAIT);
if (err != 0)
@@ -419,6 +423,8 @@
default:
break;
}
+
+ mlx5e_accel_ipsec_handle_rx(mb, cqe);
}
static inline void
@@ -564,7 +570,9 @@
("Filter returned %d!\n", rv));
}
}
- if ((MHLEN - MLX5E_NET_IP_ALIGN) >= byte_cnt &&
+ if (!mlx5e_accel_ipsec_flow(cqe) /* tag is already assigned
+ to rq->mbuf */ &&
+ MHLEN - MLX5E_NET_IP_ALIGN >= byte_cnt &&
(mb = m_gethdr(M_NOWAIT, MT_DATA)) != NULL) {
/* set maximum mbuf length */
mb->m_len = MHLEN - MLX5E_NET_IP_ALIGN;
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c b/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c
--- a/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c
@@ -32,6 +32,7 @@
#include <dev/mlx5/mlx5_en/en.h>
#include <machine/atomic.h>
+#include <dev/mlx5/mlx5_accel/ipsec.h>
static inline bool
mlx5e_do_send_cqe_inline(struct mlx5e_sq *sq)
@@ -746,6 +747,8 @@
/* get pointer to mbuf */
mb = *mbp;
+ mlx5e_accel_ipsec_handle_tx(mb, wqe);
+
/* Send a copy of the frame to the BPF listener, if any */
if (ifp != NULL && if_getbpf(ifp) != NULL)
ETHER_BPF_MTAP(ifp, mb);
diff --git a/sys/dev/mlx5/mlx5_ib/mlx5_ib.h b/sys/dev/mlx5/mlx5_ib/mlx5_ib.h
--- a/sys/dev/mlx5/mlx5_ib/mlx5_ib.h
+++ b/sys/dev/mlx5/mlx5_ib/mlx5_ib.h
@@ -172,7 +172,7 @@
struct list_head list;
struct ib_flow ibflow;
struct mlx5_ib_flow_prio *prio;
- struct mlx5_flow_rule *rule;
+ struct mlx5_flow_handle *rule;
};
struct mlx5_ib_flow_db {
diff --git a/sys/dev/mlx5/mlx5_ib/mlx5_ib_cq.c b/sys/dev/mlx5/mlx5_ib/mlx5_ib_cq.c
--- a/sys/dev/mlx5/mlx5_ib/mlx5_ib_cq.c
+++ b/sys/dev/mlx5/mlx5_ib/mlx5_ib_cq.c
@@ -202,7 +202,7 @@
case MLX5_CQE_RESP_WR_IMM:
wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
wc->wc_flags = IB_WC_WITH_IMM;
- wc->ex.imm_data = cqe->imm_inval_pkey;
+ wc->ex.imm_data = cqe->immediate;
break;
case MLX5_CQE_RESP_SEND:
wc->opcode = IB_WC_RECV;
@@ -214,12 +214,12 @@
case MLX5_CQE_RESP_SEND_IMM:
wc->opcode = IB_WC_RECV;
wc->wc_flags = IB_WC_WITH_IMM;
- wc->ex.imm_data = cqe->imm_inval_pkey;
+ wc->ex.imm_data = cqe->immediate;
break;
case MLX5_CQE_RESP_SEND_INV:
wc->opcode = IB_WC_RECV;
wc->wc_flags = IB_WC_WITH_INVALIDATE;
- wc->ex.invalidate_rkey = be32_to_cpu(cqe->imm_inval_pkey);
+ wc->ex.invalidate_rkey = be32_to_cpu(cqe->inval_rkey);
break;
}
wc->src_qp = be32_to_cpu(cqe->flags_rqpn) & 0xffffff;
@@ -227,7 +227,7 @@
g = (be32_to_cpu(cqe->flags_rqpn) >> 28) & 3;
wc->wc_flags |= g ? IB_WC_GRH : 0;
if (unlikely(is_qp1(qp->ibqp.qp_type))) {
- u16 pkey = be32_to_cpu(cqe->imm_inval_pkey) & 0xffff;
+ u16 pkey = be32_to_cpu(cqe->pkey) & 0xffff;
ib_find_cached_pkey(&dev->ib_dev, qp->port, pkey,
&wc->pkey_index);
diff --git a/sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c b/sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c
--- a/sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c
+++ b/sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c
@@ -2073,13 +2073,13 @@
mutex_lock(&dev->flow_db.lock);
list_for_each_entry_safe(iter, tmp, &handler->list, list) {
- mlx5_del_flow_rule(&iter->rule);
+ mlx5_del_flow_rules(&iter->rule);
put_flow_table(dev, iter->prio, true);
list_del(&iter->list);
kfree(iter);
}
- mlx5_del_flow_rule(&handler->rule);
+ mlx5_del_flow_rules(&handler->rule);
put_flow_table(dev, handler->prio, true);
mutex_unlock(&dev->flow_db.lock);
@@ -2108,6 +2108,7 @@
enum flow_table_type ft_type)
{
bool dont_trap = flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP;
+ struct mlx5_flow_table_attr ft_attr = {};
struct mlx5_flow_namespace *ns = NULL;
struct mlx5_ib_flow_prio *prio;
struct mlx5_flow_table *ft;
@@ -2156,10 +2157,11 @@
ft = prio->flow_table;
if (!ft) {
- ft = mlx5_create_auto_grouped_flow_table(ns, priority, "bypass",
- num_entries,
- num_groups,
- 0);
+ ft_attr.prio = priority;
+ ft_attr.max_fte = num_entries;
+ ft_attr.autogroup.max_num_groups = num_groups;
+
+ ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr);
if (!IS_ERR(ft)) {
prio->refcount = 0;
@@ -2182,10 +2184,8 @@
struct mlx5_flow_spec *spec;
const void *ib_flow = (const void *)flow_attr + sizeof(*flow_attr);
unsigned int spec_index;
- struct mlx5_flow_act flow_act = {
- .actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG,
- .flow_tag = MLX5_FS_DEFAULT_FLOW_TAG,
- };
+ struct mlx5_flow_act flow_act = {};
+
u32 action;
int err = 0;
@@ -2199,6 +2199,9 @@
goto free;
}
+ spec->flow_context.flags = FLOW_CONTEXT_HAS_TAG;
+ spec->flow_context.flow_tag = MLX5_FS_DEFAULT_FLOW_TAG;
+
INIT_LIST_HEAD(&handler->list);
for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) {
@@ -2212,12 +2215,8 @@
spec->match_criteria_enable = get_match_criteria_enable(spec->match_criteria);
action = dst ? MLX5_FLOW_RULE_FWD_ACTION_DEST : 0;
- handler->rule = mlx5_add_flow_rule(ft, spec->match_criteria_enable,
- spec->match_criteria,
- spec->match_value,
- action,
- &flow_act,
- dst);
+ flow_act.action = action;
+ handler->rule = mlx5_add_flow_rules(ft, spec, &flow_act, dst, 1);
if (IS_ERR(handler->rule)) {
err = PTR_ERR(handler->rule);
@@ -2248,7 +2247,7 @@
handler_dst = create_flow_rule(dev, ft_prio,
flow_attr, dst);
if (IS_ERR(handler_dst)) {
- mlx5_del_flow_rule(&handler->rule);
+ mlx5_del_flow_rules(&handler->rule);
ft_prio->refcount--;
kfree(handler);
handler = handler_dst;
@@ -2311,7 +2310,7 @@
&leftovers_specs[LEFTOVERS_UC].flow_attr,
dst);
if (IS_ERR(handler_ucast)) {
- mlx5_del_flow_rule(&handler->rule);
+ mlx5_del_flow_rules(&handler->rule);
ft_prio->refcount--;
kfree(handler);
handler = handler_ucast;
@@ -2354,7 +2353,7 @@
return handler_rx;
err_tx:
- mlx5_del_flow_rule(&handler_rx->rule);
+ mlx5_del_flow_rules(&handler_rx->rule);
ft_rx->refcount--;
kfree(handler_rx);
err:
diff --git a/sys/dev/mlx5/mlx5_ifc.h b/sys/dev/mlx5/mlx5_ifc.h
--- a/sys/dev/mlx5/mlx5_ifc.h
+++ b/sys/dev/mlx5/mlx5_ifc.h
@@ -64,7 +64,7 @@
MLX5_EVENT_TYPE_FPGA_ERROR = 0x20,
MLX5_EVENT_TYPE_FPGA_QP_ERROR = 0x21,
MLX5_EVENT_TYPE_DEVICE_TRACER = 0x26,
- MLX5_EVENT_TYPE_CODING_GENERAL_OBJ_EVENT = 0x27,
+ MLX5_EVENT_TYPE_OBJECT_CHANGE = 0x27,
};
enum {
@@ -323,8 +323,13 @@
MLX5_ICMD_CMDS_OPCODE_ICMD_OPCODE_INIT_OCSD = 0xf004
};
+enum {
+ MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_IPSEC = 1ULL << 0x13,
+};
+
enum {
MLX5_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY = 0xc,
+ MLX5_GENERAL_OBJECT_TYPES_IPSEC = 0x13,
};
enum {
@@ -337,7 +342,8 @@
};
enum {
- MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_DEK = 0x1,
+ MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_TLS = 0x1,
+ MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_IPSEC = 0x2,
};
struct mlx5_ifc_flow_table_fields_supported_bits {
@@ -464,39 +470,70 @@
u8 reserved_at_340[0x440];
};
+
struct mlx5_ifc_flow_table_prop_layout_bits {
u8 ft_support[0x1];
- u8 flow_tag[0x1];
+ u8 reserved_at_1[0x1];
u8 flow_counter[0x1];
u8 flow_modify_en[0x1];
u8 modify_root[0x1];
- u8 identified_miss_table[0x1];
+ u8 identified_miss_table_mode[0x1];
u8 flow_table_modify[0x1];
- u8 encap[0x1];
+ u8 reformat[0x1];
u8 decap[0x1];
- u8 reset_root_to_default[0x1];
- u8 reserved_at_a[0x16];
-
- u8 reserved_at_20[0x2];
+ u8 reserved_at_9[0x1];
+ u8 pop_vlan[0x1];
+ u8 push_vlan[0x1];
+ u8 reserved_at_c[0x1];
+ u8 pop_vlan_2[0x1];
+ u8 push_vlan_2[0x1];
+ u8 reformat_and_vlan_action[0x1];
+ u8 reserved_at_10[0x1];
+ u8 sw_owner[0x1];
+ u8 reformat_l3_tunnel_to_l2[0x1];
+ u8 reformat_l2_to_l3_tunnel[0x1];
+ u8 reformat_and_modify_action[0x1];
+ u8 ignore_flow_level[0x1];
+ u8 reserved_at_16[0x1];
+ u8 table_miss_action_domain[0x1];
+ u8 termination_table[0x1];
+ u8 reformat_and_fwd_to_table[0x1];
+ u8 reserved_at_1a[0x2];
+ u8 ipsec_encrypt[0x1];
+ u8 ipsec_decrypt[0x1];
+ u8 sw_owner_v2[0x1];
+ u8 reserved_at_1f[0x1];
+ u8 termination_table_raw_traffic[0x1];
+ u8 reserved_at_21[0x1];
u8 log_max_ft_size[0x6];
u8 log_max_modify_header_context[0x8];
u8 max_modify_header_actions[0x8];
u8 max_ft_level[0x8];
- u8 reserved_at_40[0x20];
-
- u8 reserved_at_60[0x18];
+ u8 reformat_add_esp_trasport[0x1];
+ u8 reformat_l2_to_l3_esp_tunnel[0x1];
+ u8 reformat_add_esp_transport_over_udp[0x1];
+ u8 reformat_del_esp_trasport[0x1];
+ u8 reformat_l3_esp_tunnel_to_l2[0x1];
+ u8 reformat_del_esp_transport_over_udp[0x1];
+ u8 execute_aso[0x1];
+ u8 reserved_at_47[0x19];
+ u8 reserved_at_60[0x2];
+ u8 reformat_insert[0x1];
+ u8 reformat_remove[0x1];
+ u8 macsec_encrypt[0x1];
+ u8 macsec_decrypt[0x1];
+ u8 reserved_at_66[0x2];
+ u8 reformat_add_macsec[0x1];
+ u8 reformat_remove_macsec[0x1];
+ u8 reserved_at_6a[0xe];
u8 log_max_ft_num[0x8];
-
u8 reserved_at_80[0x10];
u8 log_max_flow_counter[0x8];
u8 log_max_destination[0x8];
-
u8 reserved_at_a0[0x18];
u8 log_max_flow[0x8];
-
u8 reserved_at_c0[0x40];
-
struct mlx5_ifc_flow_table_fields_supported_bits ft_field_support;
struct mlx5_ifc_flow_table_fields_supported_bits ft_field_bitmask_support;
@@ -524,15 +561,18 @@
MLX5_FLOW_CONTEXT_DEST_TYPE_FLOW_TABLE = 0x1,
MLX5_FLOW_CONTEXT_DEST_TYPE_TIR = 0x2,
MLX5_FLOW_CONTEXT_DEST_TYPE_QP = 0x3,
+ MLX5_FLOW_CONTEXT_DEST_TYPE_TABLE_TYPE = 0xA,
};
struct mlx5_ifc_dest_format_struct_bits {
- u8 destination_type[0x8];
- u8 destination_id[0x18];
+ u8 destination_type[0x8];
+ u8 destination_id[0x18];
- u8 reserved_0[0x8];
- u8 destination_table_type[0x8];
- u8 reserved_at_1[0x10];
+ u8 destination_eswitch_owner_vhca_id_valid[0x1];
+ u8 packet_reformat[0x1];
+ u8 reserved_at_22[0x6];
+ u8 destination_table_type[0x8];
+ u8 destination_eswitch_owner_vhca_id[0x10];
};
struct mlx5_ifc_ipv4_layout_bits {
@@ -586,11 +626,25 @@
union mlx5_ifc_ipv6_layout_ipv4_layout_auto_bits dst_ipv4_dst_ipv6;
};
+struct mlx5_ifc_nvgre_key_bits {
+ u8 hi[0x18];
+ u8 lo[0x8];
+};
+
+union mlx5_ifc_gre_key_bits {
+ struct mlx5_ifc_nvgre_key_bits nvgre;
+ u8 key[0x20];
+};
+
struct mlx5_ifc_fte_match_set_misc_bits {
- u8 reserved_0[0x8];
+ u8 gre_c_present[0x1];
+ u8 reserved_at_1[0x1];
+ u8 gre_k_present[0x1];
+ u8 gre_s_present[0x1];
+ u8 source_vhca_port[0x4];
u8 source_sqn[0x18];
- u8 reserved_1[0x10];
+ u8 source_eswitch_owner_vhca_id[0x10];
u8 source_port[0x10];
u8 outer_second_prio[0x3];
@@ -600,35 +654,163 @@
u8 inner_second_cfi[0x1];
u8 inner_second_vid[0xc];
- u8 outer_second_vlan_tag[0x1];
- u8 inner_second_vlan_tag[0x1];
- u8 reserved_2[0xe];
+ u8 outer_second_cvlan_tag[0x1];
+ u8 inner_second_cvlan_tag[0x1];
+ u8 outer_second_svlan_tag[0x1];
+ u8 inner_second_svlan_tag[0x1];
+ u8 reserved_at_64[0xc];
u8 gre_protocol[0x10];
- u8 gre_key_h[0x18];
- u8 gre_key_l[0x8];
+ union mlx5_ifc_gre_key_bits gre_key;
u8 vxlan_vni[0x18];
- u8 reserved_3[0x8];
+ u8 bth_opcode[0x8];
u8 geneve_vni[0x18];
- u8 reserved4[0x7];
+ u8 reserved_at_d8[0x6];
+ u8 geneve_tlv_option_0_exist[0x1];
u8 geneve_oam[0x1];
- u8 reserved_5[0xc];
+ u8 reserved_at_e0[0xc];
u8 outer_ipv6_flow_label[0x14];
- u8 reserved_6[0xc];
+ u8 reserved_at_100[0xc];
u8 inner_ipv6_flow_label[0x14];
- u8 reserved_7[0xa];
+ u8 reserved_at_120[0xa];
u8 geneve_opt_len[0x6];
u8 geneve_protocol_type[0x10];
- u8 reserved_8[0x8];
+ u8 reserved_at_140[0x8];
u8 bth_dst_qp[0x18];
+ u8 inner_esp_spi[0x20];
+ u8 outer_esp_spi[0x20];
+ u8 reserved_at_1a0[0x60];
+};
- u8 reserved_9[0xa0];
+struct mlx5_ifc_fte_match_mpls_bits {
+ u8 mpls_label[0x14];
+ u8 mpls_exp[0x3];
+ u8 mpls_s_bos[0x1];
+ u8 mpls_ttl[0x8];
+};
+
+struct mlx5_ifc_fte_match_set_misc2_bits {
+ struct mlx5_ifc_fte_match_mpls_bits outer_first_mpls;
+
+ struct mlx5_ifc_fte_match_mpls_bits inner_first_mpls;
+
+ struct mlx5_ifc_fte_match_mpls_bits outer_first_mpls_over_gre;
+
+ struct mlx5_ifc_fte_match_mpls_bits outer_first_mpls_over_udp;
+
+ u8 metadata_reg_c_7[0x20];
+
+ u8 metadata_reg_c_6[0x20];
+
+ u8 metadata_reg_c_5[0x20];
+
+ u8 metadata_reg_c_4[0x20];
+
+ u8 metadata_reg_c_3[0x20];
+
+ u8 metadata_reg_c_2[0x20];
+
+ u8 metadata_reg_c_1[0x20];
+
+ u8 metadata_reg_c_0[0x20];
+
+ u8 metadata_reg_a[0x20];
+
+ u8 reserved_at_1a0[0x8];
+
+ u8 macsec_syndrome[0x8];
+ u8 ipsec_syndrome[0x8];
+ u8 reserved_at_1b8[0x8];
+
+ u8 reserved_at_1c0[0x40];
+};
+
+struct mlx5_ifc_fte_match_set_misc3_bits {
+ u8 inner_tcp_seq_num[0x20];
+
+ u8 outer_tcp_seq_num[0x20];
+
+ u8 inner_tcp_ack_num[0x20];
+
+ u8 outer_tcp_ack_num[0x20];
+
+ u8 reserved_at_80[0x8];
+ u8 outer_vxlan_gpe_vni[0x18];
+
+ u8 outer_vxlan_gpe_next_protocol[0x8];
+ u8 outer_vxlan_gpe_flags[0x8];
+ u8 reserved_at_b0[0x10];
+
+ u8 icmp_header_data[0x20];
+
+ u8 icmpv6_header_data[0x20];
+
+ u8 icmp_type[0x8];
+ u8 icmp_code[0x8];
+ u8 icmpv6_type[0x8];
+ u8 icmpv6_code[0x8];
+
+ u8 geneve_tlv_option_0_data[0x20];
+
+ u8 gtpu_teid[0x20];
+
+ u8 gtpu_msg_type[0x8];
+ u8 gtpu_msg_flags[0x8];
+ u8 reserved_at_170[0x10];
+
+ u8 gtpu_dw_2[0x20];
+
+ u8 gtpu_first_ext_dw_0[0x20];
+
+ u8 gtpu_dw_0[0x20];
+
+ u8 reserved_at_1e0[0x20];
+};
+
+struct mlx5_ifc_fte_match_set_misc4_bits {
+ u8 prog_sample_field_value_0[0x20];
+
+ u8 prog_sample_field_id_0[0x20];
+
+ u8 prog_sample_field_value_1[0x20];
+
+ u8 prog_sample_field_id_1[0x20];
+
+ u8 prog_sample_field_value_2[0x20];
+
+ u8 prog_sample_field_id_2[0x20];
+
+ u8 prog_sample_field_value_3[0x20];
+
+ u8 prog_sample_field_id_3[0x20];
+
+ u8 reserved_at_100[0x100];
+};
+
+struct mlx5_ifc_fte_match_set_misc5_bits {
+ u8 macsec_tag_0[0x20];
+
+ u8 macsec_tag_1[0x20];
+
+ u8 macsec_tag_2[0x20];
+
+ u8 macsec_tag_3[0x20];
+
+ u8 tunnel_header_0[0x20];
+
+ u8 tunnel_header_1[0x20];
+
+ u8 tunnel_header_2[0x20];
+
+ u8 tunnel_header_3[0x20];
+
+ u8 reserved_at_100[0x100];
};
struct mlx5_ifc_cmd_pas_bits {
@@ -866,6 +1048,20 @@
u8 reserved_1[0x7200];
};
+struct mlx5_ifc_port_selection_cap_bits {
+ u8 reserved_at_0[0x10];
+ u8 port_select_flow_table[0x1];
+ u8 reserved_at_11[0x1];
+ u8 port_select_flow_table_bypass[0x1];
+ u8 reserved_at_13[0xd];
+
+ u8 reserved_at_20[0x1e0];
+
+ struct mlx5_ifc_flow_table_prop_layout_bits flow_table_properties_port_selection;
+
+ u8 reserved_at_400[0x7c00];
+};
+
struct mlx5_ifc_pddr_module_info_bits {
u8 cable_technology[0x8];
u8 cable_breakout[0x8];
@@ -1157,7 +1353,15 @@
u8 reserved_0[0x20];
u8 hca_cap_2[0x1];
- u8 reserved_at_21[0x1f];
+ u8 create_lag_when_not_master_up[0x1];
+ u8 dtor[0x1];
+ u8 event_on_vhca_state_teardown_request[0x1];
+ u8 event_on_vhca_state_in_use[0x1];
+ u8 event_on_vhca_state_active[0x1];
+ u8 event_on_vhca_state_allocated[0x1];
+ u8 event_on_vhca_state_invalid[0x1];
+ u8 reserved_at_28[0x8];
+ u8 vhca_id[0x10];
u8 reserved_at_40[0x40];
@@ -1407,7 +1611,8 @@
u8 reserved_at_460[0x3];
u8 log_max_uctx[0x5];
- u8 reserved_at_468[0x3];
+ u8 reserved_at_468[0x2];
+ u8 ipsec_offload[0x1];
u8 log_max_umem[0x5];
u8 max_num_eqs[0x10];
@@ -1491,11 +1696,27 @@
u8 reserved_at_260[0x5a0];
};
-enum mlx5_flow_destination_type {
- MLX5_FLOW_DESTINATION_TYPE_VPORT = 0x0,
- MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE = 0x1,
- MLX5_FLOW_DESTINATION_TYPE_TIR = 0x2,
- MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE = 0xA,
+enum mlx5_ifc_flow_destination_type {
+ MLX5_IFC_FLOW_DESTINATION_TYPE_VPORT = 0x0,
+ MLX5_IFC_FLOW_DESTINATION_TYPE_FLOW_TABLE = 0x1,
+ MLX5_IFC_FLOW_DESTINATION_TYPE_TIR = 0x2,
+ MLX5_IFC_FLOW_DESTINATION_TYPE_FLOW_SAMPLER = 0x6,
+ MLX5_IFC_FLOW_DESTINATION_TYPE_UPLINK = 0x8,
+ MLX5_IFC_FLOW_DESTINATION_TYPE_TABLE_TYPE = 0xA,
+};
+
+enum mlx5_flow_table_miss_action {
+ MLX5_FLOW_TABLE_MISS_ACTION_DEF,
+ MLX5_FLOW_TABLE_MISS_ACTION_FWD,
+ MLX5_FLOW_TABLE_MISS_ACTION_SWITCH_DOMAIN,
+};
+
+struct mlx5_ifc_extended_dest_format_bits {
+ struct mlx5_ifc_dest_format_struct_bits destination_entry;
+
+ u8 packet_reformat_id[0x20];
+
+ u8 reserved_at_60[0x20];
};
union mlx5_ifc_dest_format_struct_flow_counter_list_auto_bits {
@@ -1505,13 +1726,21 @@
};
struct mlx5_ifc_fte_match_param_bits {
- struct mlx5_ifc_fte_match_set_lyr_2_4_bits outer_headers;
+ struct mlx5_ifc_fte_match_set_lyr_2_4_bits outer_headers;
+
+ struct mlx5_ifc_fte_match_set_misc_bits misc_parameters;
+
+ struct mlx5_ifc_fte_match_set_lyr_2_4_bits inner_headers;
+
+ struct mlx5_ifc_fte_match_set_misc2_bits misc_parameters_2;
- struct mlx5_ifc_fte_match_set_misc_bits misc_parameters;
+ struct mlx5_ifc_fte_match_set_misc3_bits misc_parameters_3;
- struct mlx5_ifc_fte_match_set_lyr_2_4_bits inner_headers;
+ struct mlx5_ifc_fte_match_set_misc4_bits misc_parameters_4;
- u8 reserved_0[0xa00];
+ struct mlx5_ifc_fte_match_set_misc5_bits misc_parameters_5;
+
+ u8 reserved_at_e00[0x200];
};
enum {
@@ -2313,43 +2542,85 @@
u8 atomic_resp[32][0x8];
};
+struct mlx5_ifc_vlan_bits {
+ u8 ethtype[0x10];
+ u8 prio[0x3];
+ u8 cfi[0x1];
+ u8 vid[0xc];
+};
+
enum {
- MLX5_FLOW_CONTEXT_ACTION_ALLOW = 0x1,
- MLX5_FLOW_CONTEXT_ACTION_DROP = 0x2,
- MLX5_FLOW_CONTEXT_ACTION_FWD_DEST = 0x4,
- MLX5_FLOW_CONTEXT_ACTION_COUNT = 0x8,
- MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT = 0x10,
- MLX5_FLOW_CONTEXT_ACTION_MOD_HDR = 0x40,
+ MLX5_FLOW_METER_COLOR_RED = 0x0,
+ MLX5_FLOW_METER_COLOR_YELLOW = 0x1,
+ MLX5_FLOW_METER_COLOR_GREEN = 0x2,
+ MLX5_FLOW_METER_COLOR_UNDEFINED = 0x3,
+};
+
+enum {
+ MLX5_EXE_ASO_FLOW_METER = 0x2,
+};
+
+struct mlx5_ifc_exe_aso_ctrl_flow_meter_bits {
+ u8 return_reg_id[0x4];
+ u8 aso_type[0x4];
+ u8 reserved_at_8[0x14];
+ u8 action[0x1];
+ u8 init_color[0x2];
+ u8 meter_id[0x1];
+};
+
+union mlx5_ifc_exe_aso_ctrl {
+ struct mlx5_ifc_exe_aso_ctrl_flow_meter_bits exe_aso_ctrl_flow_meter;
+};
+
+struct mlx5_ifc_execute_aso_bits {
+ u8 valid[0x1];
+ u8 reserved_at_1[0x7];
+ u8 aso_object_id[0x18];
+
+ union mlx5_ifc_exe_aso_ctrl exe_aso_ctrl;
+};
+
+enum {
+ MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_IPSEC = 0x0,
};
struct mlx5_ifc_flow_context_bits {
- u8 reserved_0[0x20];
+ struct mlx5_ifc_vlan_bits push_vlan;
u8 group_id[0x20];
- u8 reserved_1[0x8];
+ u8 reserved_at_40[0x8];
u8 flow_tag[0x18];
- u8 reserved_2[0x10];
+ u8 reserved_at_60[0x10];
u8 action[0x10];
- u8 reserved_3[0x8];
+ u8 extended_destination[0x1];
+ u8 reserved_at_81[0x1];
+ u8 flow_source[0x2];
+ u8 encrypt_decrypt_type[0x4];
u8 destination_list_size[0x18];
- u8 reserved_4[0x8];
+ u8 reserved_at_a0[0x8];
u8 flow_counter_list_size[0x18];
u8 packet_reformat_id[0x20];
- u8 modify_header_id[0x20];
+ u8 modify_header_id[0x20];
- u8 reserved_6[0x100];
+ struct mlx5_ifc_vlan_bits push_vlan_2;
+
+ u8 encrypt_decrypt_obj_id[0x20];
+ u8 reserved_at_140[0xc0];
struct mlx5_ifc_fte_match_param_bits match_value;
- u8 reserved_7[0x600];
+ struct mlx5_ifc_execute_aso_bits execute_aso[4];
+
+ u8 reserved_at_1300[0x500];
- union mlx5_ifc_dest_format_struct_flow_counter_list_auto_bits destination[0];
+ union mlx5_ifc_dest_format_struct_flow_counter_list_auto_bits destination[];
};
enum {
@@ -3011,21 +3282,27 @@
};
struct mlx5_ifc_flow_table_context_bits {
- u8 reformat_en[0x1];
- u8 decap_en[0x1];
- u8 reserved_at_2[0x2];
- u8 table_miss_action[0x4];
- u8 level[0x8];
- u8 reserved_at_10[0x8];
- u8 log_size[0x8];
+ u8 reformat_en[0x1];
+ u8 decap_en[0x1];
+ u8 sw_owner[0x1];
+ u8 termination_table[0x1];
+ u8 table_miss_action[0x4];
+ u8 level[0x8];
+ u8 reserved_at_10[0x8];
+ u8 log_size[0x8];
- u8 reserved_at_20[0x8];
- u8 table_miss_id[0x18];
+ u8 reserved_at_20[0x8];
+ u8 table_miss_id[0x18];
- u8 reserved_at_40[0x8];
- u8 lag_master_next_table_id[0x18];
+ u8 reserved_at_40[0x8];
+ u8 lag_master_next_table_id[0x18];
+
+ u8 reserved_at_60[0x60];
+
+ u8 sw_owner_icm_root_1[0x40];
+
+ u8 sw_owner_icm_root_0[0x40];
- u8 reserved_at_60[0xe0];
};
struct mlx5_ifc_esw_vport_context_bits {
@@ -3983,28 +4260,32 @@
};
struct mlx5_ifc_set_flow_table_root_in_bits {
- u8 opcode[0x10];
- u8 reserved_0[0x10];
-
- u8 reserved_1[0x10];
- u8 op_mod[0x10];
+ u8 opcode[0x10];
+ u8 reserved_at_10[0x10];
- u8 other_vport[0x1];
- u8 reserved_2[0xf];
- u8 vport_number[0x10];
+ u8 reserved_at_20[0x10];
+ u8 op_mod[0x10];
- u8 reserved_3[0x20];
+ u8 other_vport[0x1];
+ u8 reserved_at_41[0xf];
+ u8 vport_number[0x10];
- u8 table_type[0x8];
- u8 reserved_4[0x18];
+ u8 reserved_at_60[0x20];
- u8 reserved_5[0x8];
- u8 table_id[0x18];
+ u8 table_type[0x8];
+ u8 reserved_at_88[0x7];
+ u8 table_of_other_vport[0x1];
+ u8 table_vport_number[0x10];
- u8 reserved_6[0x8];
- u8 underlay_qpn[0x18];
+ u8 reserved_at_a0[0x8];
+ u8 table_id[0x18];
- u8 reserved_7[0x120];
+ u8 reserved_at_c0[0x8];
+ u8 underlay_qpn[0x18];
+ u8 table_eswitch_owner_vhca_id_valid[0x1];
+ u8 reserved_at_e1[0xf];
+ u8 table_eswitch_owner_vhca_id[0x10];
+ u8 reserved_at_100[0x100];
};
struct mlx5_ifc_set_fte_out_bits {
@@ -4017,34 +4298,35 @@
};
struct mlx5_ifc_set_fte_in_bits {
- u8 opcode[0x10];
- u8 reserved_0[0x10];
+ u8 opcode[0x10];
+ u8 reserved_at_10[0x10];
- u8 reserved_1[0x10];
- u8 op_mod[0x10];
+ u8 reserved_at_20[0x10];
+ u8 op_mod[0x10];
- u8 other_vport[0x1];
- u8 reserved_2[0xf];
- u8 vport_number[0x10];
+ u8 other_vport[0x1];
+ u8 reserved_at_41[0xf];
+ u8 vport_number[0x10];
- u8 reserved_3[0x20];
+ u8 reserved_at_60[0x20];
- u8 table_type[0x8];
- u8 reserved_4[0x18];
+ u8 table_type[0x8];
+ u8 reserved_at_88[0x18];
- u8 reserved_5[0x8];
- u8 table_id[0x18];
+ u8 reserved_at_a0[0x8];
+ u8 table_id[0x18];
- u8 reserved_6[0x18];
- u8 modify_enable_mask[0x8];
+ u8 ignore_flow_level[0x1];
+ u8 reserved_at_c1[0x17];
+ u8 modify_enable_mask[0x8];
- u8 reserved_7[0x20];
+ u8 reserved_at_e0[0x20];
- u8 flow_index[0x20];
+ u8 flow_index[0x20];
- u8 reserved_8[0xe0];
+ u8 reserved_at_120[0xe0];
- struct mlx5_ifc_flow_context_bits flow_context;
+ struct mlx5_ifc_flow_context_bits flow_context;
};
struct mlx5_ifc_set_driver_version_out_bits {
@@ -4325,6 +4607,23 @@
MLX5_QUERY_VPORT_STATE_IN_OP_MOD_UPLINK = 0x2,
};
+enum {
+ MLX5_FLOW_CONTEXT_ACTION_ALLOW = 0x1,
+ MLX5_FLOW_CONTEXT_ACTION_DROP = 0x2,
+ MLX5_FLOW_CONTEXT_ACTION_FWD_DEST = 0x4,
+ MLX5_FLOW_CONTEXT_ACTION_COUNT = 0x8,
+ MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT = 0x10,
+ MLX5_FLOW_CONTEXT_ACTION_DECAP = 0x20,
+ MLX5_FLOW_CONTEXT_ACTION_MOD_HDR = 0x40,
+ MLX5_FLOW_CONTEXT_ACTION_VLAN_POP = 0x80,
+ MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH = 0x100,
+ MLX5_FLOW_CONTEXT_ACTION_VLAN_POP_2 = 0x400,
+ MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2 = 0x800,
+ MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT = 0x1000,
+ MLX5_FLOW_CONTEXT_ACTION_CRYPTO_ENCRYPT = 0x2000,
+ MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO = 0x4000,
+};
+
struct mlx5_ifc_query_vport_state_in_bits {
u8 opcode[0x10];
u8 reserved_0[0x10];
@@ -5545,6 +5844,12 @@
MLX5_REFORMAT_TYPE_L2_TO_L2_TUNNEL = 0x2,
MLX5_REFORMAT_TYPE_L3_TUNNEL_TO_L2 = 0x3,
MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL = 0x4,
+ MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV4 = 0x5,
+ MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_UDPV4 = 0x7,
+ MLX5_REFORMAT_TYPE_DEL_ESP_TRANSPORT = 0x8,
+ MLX5_REFORMAT_TYPE_DEL_ESP_TRANSPORT_OVER_UDP = 0xa,
+ MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV6 = 0xb,
+ MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_UDPV6 = 0xc,
};
struct mlx5_ifc_alloc_packet_reformat_context_in_bits {
@@ -6255,6 +6560,11 @@
struct mlx5_ifc_hca_vport_context_bits hca_vport_context;
};
+enum {
+ MLX5_MODIFY_FLOW_TABLE_MISS_TABLE_ID = (1UL << 0),
+ MLX5_MODIFY_FLOW_TABLE_LAG_NEXT_TABLE_ID = (1UL << 15),
+};
+
struct mlx5_ifc_modify_flow_table_out_bits {
u8 status[0x8];
u8 reserved_at_8[0x18];
@@ -7898,24 +8208,24 @@
};
struct mlx5_ifc_create_flow_table_in_bits {
- u8 opcode[0x10];
- u8 reserved_at_10[0x10];
+ u8 opcode[0x10];
+ u8 uid[0x10];
- u8 reserved_at_20[0x10];
- u8 op_mod[0x10];
+ u8 reserved_at_20[0x10];
+ u8 op_mod[0x10];
- u8 other_vport[0x1];
- u8 reserved_at_41[0xf];
- u8 vport_number[0x10];
+ u8 other_vport[0x1];
+ u8 reserved_at_41[0xf];
+ u8 vport_number[0x10];
- u8 reserved_at_60[0x20];
+ u8 reserved_at_60[0x20];
- u8 table_type[0x8];
- u8 reserved_at_88[0x18];
+ u8 table_type[0x8];
+ u8 reserved_at_88[0x18];
- u8 reserved_at_a0[0x20];
+ u8 reserved_at_a0[0x20];
- struct mlx5_ifc_flow_table_context_bits flow_table_context;
+ struct mlx5_ifc_flow_table_context_bits flow_table_context;
};
struct mlx5_ifc_create_flow_group_out_bits {
@@ -7931,46 +8241,54 @@
};
enum {
- MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS = 0x0,
- MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS = 0x1,
- MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_INNER_HEADERS = 0x2,
+ MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS = 0x0,
+ MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS = 0x1,
+ MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_INNER_HEADERS = 0x2,
+ MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS_2 = 0x3,
};
struct mlx5_ifc_create_flow_group_in_bits {
- u8 opcode[0x10];
- u8 reserved_0[0x10];
+ u8 opcode[0x10];
+ u8 reserved_at_10[0x10];
- u8 reserved_1[0x10];
- u8 op_mod[0x10];
+ u8 reserved_at_20[0x10];
+ u8 op_mod[0x10];
- u8 other_vport[0x1];
- u8 reserved_2[0xf];
- u8 vport_number[0x10];
+ u8 other_vport[0x1];
+ u8 reserved_at_41[0xf];
+ u8 vport_number[0x10];
- u8 reserved_3[0x20];
+ u8 reserved_at_60[0x20];
- u8 table_type[0x8];
- u8 reserved_4[0x18];
+ u8 table_type[0x8];
+ u8 reserved_at_88[0x4];
+ u8 group_type[0x4];
+ u8 reserved_at_90[0x10];
- u8 reserved_5[0x8];
- u8 table_id[0x18];
+ u8 reserved_at_a0[0x8];
+ u8 table_id[0x18];
- u8 reserved_6[0x20];
+ u8 source_eswitch_owner_vhca_id_valid[0x1];
- u8 start_flow_index[0x20];
+ u8 reserved_at_c1[0x1f];
- u8 reserved_7[0x20];
+ u8 start_flow_index[0x20];
- u8 end_flow_index[0x20];
+ u8 reserved_at_100[0x20];
- u8 reserved_8[0xa0];
+ u8 end_flow_index[0x20];
- u8 reserved_9[0x18];
- u8 match_criteria_enable[0x8];
+ u8 reserved_at_140[0x10];
+ u8 match_definer_id[0x10];
- struct mlx5_ifc_fte_match_param_bits match_criteria;
+ u8 reserved_at_160[0x80];
+
+ u8 reserved_at_1e0[0x18];
+ u8 match_criteria_enable[0x8];
+
+ struct mlx5_ifc_fte_match_param_bits match_criteria;
- u8 reserved_10[0xe00];
+ u8 reserved_at_1200[0xe00];
};
struct mlx5_ifc_create_encryption_key_out_bits {
@@ -11637,5 +11955,120 @@
#define MLX5_FC_BULK_NUM_FCS(fc_enum) (MLX5_FC_BULK_SIZE_FACTOR * (fc_enum))
+struct mlx5_ifc_ipsec_cap_bits {
+ u8 ipsec_full_offload[0x1];
+ u8 ipsec_crypto_offload[0x1];
+ u8 ipsec_esn[0x1];
+ u8 ipsec_crypto_esp_aes_gcm_256_encrypt[0x1];
+ u8 ipsec_crypto_esp_aes_gcm_128_encrypt[0x1];
+ u8 ipsec_crypto_esp_aes_gcm_256_decrypt[0x1];
+ u8 ipsec_crypto_esp_aes_gcm_128_decrypt[0x1];
+ u8 reserved_at_7[0x4];
+ u8 log_max_ipsec_offload[0x5];
+ u8 reserved_at_10[0x10];
+
+ u8 min_log_ipsec_full_replay_window[0x8];
+ u8 max_log_ipsec_full_replay_window[0x8];
+ u8 reserved_at_30[0x7d0];
+};
+
+enum {
+ MLX5_IPSEC_OBJECT_ICV_LEN_16B,
+};
+
+enum {
+ MLX5_IPSEC_ASO_REG_C_0_1 = 0x0,
+ MLX5_IPSEC_ASO_REG_C_2_3 = 0x1,
+ MLX5_IPSEC_ASO_REG_C_4_5 = 0x2,
+ MLX5_IPSEC_ASO_REG_C_6_7 = 0x3,
+};
+
+enum {
+ MLX5_IPSEC_ASO_MODE = 0x0,
+ MLX5_IPSEC_ASO_REPLAY_PROTECTION = 0x1,
+ MLX5_IPSEC_ASO_INC_SN = 0x2,
+};
+
+enum {
+ MLX5_IPSEC_ASO_REPLAY_WIN_32BIT = 0x0,
+ MLX5_IPSEC_ASO_REPLAY_WIN_64BIT = 0x1,
+ MLX5_IPSEC_ASO_REPLAY_WIN_128BIT = 0x2,
+ MLX5_IPSEC_ASO_REPLAY_WIN_256BIT = 0x3,
+};
+
+struct mlx5_ifc_ipsec_aso_bits {
+ u8 valid[0x1];
+ u8 reserved_at_201[0x1];
+ u8 mode[0x2];
+ u8 window_sz[0x2];
+ u8 soft_lft_arm[0x1];
+ u8 hard_lft_arm[0x1];
+ u8 remove_flow_enable[0x1];
+ u8 esn_event_arm[0x1];
+ u8 reserved_at_20a[0x16];
+
+ u8 remove_flow_pkt_cnt[0x20];
+
+ u8 remove_flow_soft_lft[0x20];
+
+ u8 reserved_at_260[0x80];
+
+ u8 mode_parameter[0x20];
+
+ u8 replay_protection_window[0x100];
+};
+
+struct mlx5_ifc_ipsec_obj_bits {
+ u8 modify_field_select[0x40];
+ u8 full_offload[0x1];
+ u8 reserved_at_41[0x1];
+ u8 esn_en[0x1];
+ u8 esn_overlap[0x1];
+ u8 reserved_at_44[0x2];
+ u8 icv_length[0x2];
+ u8 reserved_at_48[0x4];
+ u8 aso_return_reg[0x4];
+ u8 reserved_at_50[0x10];
+
+ u8 esn_msb[0x20];
+
+ u8 reserved_at_80[0x8];
+ u8 dekn[0x18];
+
+ u8 salt[0x20];
+
+ u8 implicit_iv[0x40];
+ u8 reserved_at_100[0x8];
+ u8 ipsec_aso_access_pd[0x18];
+ u8 reserved_at_120[0xe0];
+
+ struct mlx5_ifc_ipsec_aso_bits ipsec_aso;
+};
+
+struct mlx5_ifc_create_ipsec_obj_in_bits {
+ struct mlx5_ifc_general_obj_in_cmd_hdr_bits general_obj_in_cmd_hdr;
+ struct mlx5_ifc_ipsec_obj_bits ipsec_object;
+};
+
+enum {
+ MLX5_MODIFY_IPSEC_BITMASK_ESN_OVERLAP = 1 << 0,
+ MLX5_MODIFY_IPSEC_BITMASK_ESN_MSB = 1 << 1,
+};
+
+struct mlx5_ifc_query_ipsec_obj_out_bits {
+ struct mlx5_ifc_general_obj_out_cmd_hdr_bits general_obj_out_cmd_hdr;
+ struct mlx5_ifc_ipsec_obj_bits ipsec_object;
+};
+
+struct mlx5_ifc_modify_ipsec_obj_in_bits {
+ struct mlx5_ifc_general_obj_in_cmd_hdr_bits general_obj_in_cmd_hdr;
+ struct mlx5_ifc_ipsec_obj_bits ipsec_object;
+};
+
+enum {
+ MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_PURPOSE_TLS = 0x1,
+ MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_PURPOSE_IPSEC = 0x2,
+ MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_PURPOSE_MACSEC = 0x4,
+};
#endif /* MLX5_IFC_H */
diff --git a/sys/dev/mlx5/mlx5_lib/aso.h b/sys/dev/mlx5/mlx5_lib/aso.h
new file mode 100644
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_lib/aso.h
@@ -0,0 +1,92 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
+/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */
+
+#ifndef __MLX5_LIB_ASO_H__
+#define __MLX5_LIB_ASO_H__
+
+#include <dev/mlx5/qp.h>
+#include <dev/mlx5/mlx5_core/mlx5_core.h>
+
+#define MLX5_ASO_WQEBBS \
+ (DIV_ROUND_UP(sizeof(struct mlx5_aso_wqe), MLX5_SEND_WQE_BB))
+#define MLX5_ASO_WQEBBS_DATA \
+ (DIV_ROUND_UP(sizeof(struct mlx5_aso_wqe_data), MLX5_SEND_WQE_BB))
+#define ASO_CTRL_READ_EN BIT(0)
+#define MLX5_WQE_CTRL_WQE_OPC_MOD_SHIFT 24
+#define MLX5_MACSEC_ASO_DS_CNT (DIV_ROUND_UP(sizeof(struct mlx5_aso_wqe), MLX5_SEND_WQE_DS))
+
+#define ASO_CTRL_READ_EN BIT(0)
+struct mlx5_wqe_aso_ctrl_seg {
+ __be32 va_h;
+ __be32 va_l; /* include read_enable */
+ __be32 l_key;
+ u8 data_mask_mode;
+ u8 condition_1_0_operand;
+ u8 condition_1_0_offset;
+ u8 data_offset_condition_operand;
+ __be32 condition_0_data;
+ __be32 condition_0_mask;
+ __be32 condition_1_data;
+ __be32 condition_1_mask;
+ __be64 bitwise_data;
+ __be64 data_mask;
+};
+
+struct mlx5_wqe_aso_data_seg {
+ __be32 bytewise_data[16];
+};
+
+struct mlx5_aso_wqe {
+ struct mlx5_wqe_ctrl_seg ctrl;
+ struct mlx5_wqe_aso_ctrl_seg aso_ctrl;
+};
+
+struct mlx5_aso_wqe_data {
+ struct mlx5_wqe_ctrl_seg ctrl;
+ struct mlx5_wqe_aso_ctrl_seg aso_ctrl;
+ struct mlx5_wqe_aso_data_seg aso_data;
+};
+
+enum {
+ MLX5_ASO_LOGICAL_AND,
+ MLX5_ASO_LOGICAL_OR,
+};
+
+enum {
+ MLX5_ASO_ALWAYS_FALSE,
+ MLX5_ASO_ALWAYS_TRUE,
+ MLX5_ASO_EQUAL,
+ MLX5_ASO_NOT_EQUAL,
+ MLX5_ASO_GREATER_OR_EQUAL,
+ MLX5_ASO_LESSER_OR_EQUAL,
+ MLX5_ASO_LESSER,
+ MLX5_ASO_GREATER,
+ MLX5_ASO_CYCLIC_GREATER,
+ MLX5_ASO_CYCLIC_LESSER,
+};
+
+enum {
+ MLX5_ASO_DATA_MASK_MODE_BITWISE_64BIT,
+ MLX5_ASO_DATA_MASK_MODE_BYTEWISE_64BYTE,
+ MLX5_ASO_DATA_MASK_MODE_CALCULATED_64BYTE,
+};
+
+enum {
+ MLX5_ACCESS_ASO_OPC_MOD_IPSEC = 0x0,
+ MLX5_ACCESS_ASO_OPC_MOD_FLOW_METER = 0x2,
+ MLX5_ACCESS_ASO_OPC_MOD_MACSEC = 0x5,
+};
+
+struct mlx5_aso;
+
+struct mlx5_aso_wqe *mlx5_aso_get_wqe(struct mlx5_aso *aso);
+void mlx5_aso_build_wqe(struct mlx5_aso *aso, u8 ds_cnt,
+ struct mlx5_aso_wqe *aso_wqe,
+ u32 obj_id, u32 opc_mode);
+void mlx5_aso_post_wqe(struct mlx5_aso *aso, bool with_data,
+ struct mlx5_wqe_ctrl_seg *doorbell_cseg);
+int mlx5_aso_poll_cq(struct mlx5_aso *aso, bool with_data);
+
+struct mlx5_aso *mlx5_aso_create(struct mlx5_core_dev *mdev, u32 pdn);
+void mlx5_aso_destroy(struct mlx5_aso *aso);
+#endif /* __MLX5_LIB_ASO_H__ */
diff --git a/sys/dev/mlx5/mlx5_lib/mlx5_aso.c b/sys/dev/mlx5/mlx5_lib/mlx5_aso.c
new file mode 100644
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_lib/mlx5_aso.c
@@ -0,0 +1,428 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+// Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
+#include <linux/printk.h>
+#include <dev/mlx5/driver.h>
+#include <dev/mlx5/mlx5_core/transobj.h>
+#include "aso.h"
+#include <dev/mlx5/mlx5_core/wq.h>
+#include <dev/mlx5/cq.h>
+
+struct mlx5_aso_cq {
+ /* data path - accessed per cqe */
+ struct mlx5_cqwq wq;
+
+ /* data path - accessed per napi poll */
+ struct mlx5_core_cq mcq;
+
+ /* control */
+ struct mlx5_core_dev *mdev;
+ struct mlx5_wq_ctrl wq_ctrl;
+} ____cacheline_aligned_in_smp;
+
+struct mlx5_aso {
+ /* data path */
+ u16 cc;
+ u16 pc;
+
+ struct mlx5_wqe_ctrl_seg *doorbell_cseg;
+ struct mlx5_aso_cq cq;
+
+ /* read only */
+ struct mlx5_wq_cyc wq;
+ void __iomem *uar_map;
+ u32 sqn;
+
+ /* control path */
+ struct mlx5_wq_ctrl wq_ctrl;
+
+} ____cacheline_aligned_in_smp;
+
+static void mlx5_aso_free_cq(struct mlx5_aso_cq *cq)
+{
+ mlx5_wq_destroy(&cq->wq_ctrl);
+}
+
+static int mlx5_aso_alloc_cq(struct mlx5_core_dev *mdev, int numa_node,
+ void *cqc_data, struct mlx5_aso_cq *cq)
+{
+ struct mlx5_core_cq *mcq = &cq->mcq;
+ struct mlx5_wq_param param;
+ int err;
+ u32 i;
+
+ param.linear = 1;
+ err = mlx5_cqwq_create(mdev, &param, cqc_data, &cq->wq, &cq->wq_ctrl);
+ if (err)
+ return err;
+
+ mcq->cqe_sz = 64;
+ mcq->set_ci_db = cq->wq_ctrl.db.db;
+ mcq->arm_db = cq->wq_ctrl.db.db + 1;
+
+ for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) {
+ struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i);
+
+ cqe->op_own = 0xf1;
+ }
+
+ cq->mdev = mdev;
+
+ return 0;
+}
+
+static int create_aso_cq(struct mlx5_aso_cq *cq, void *cqc_data)
+{
+ u32 out[MLX5_ST_SZ_DW(create_cq_out)];
+ struct mlx5_core_dev *mdev = cq->mdev;
+ struct mlx5_core_cq *mcq = &cq->mcq;
+ int inlen, eqn, irqn_not_used;
+ void *in, *cqc;
+ int err;
+
+ err = mlx5_vector2eqn(mdev, 0, &eqn, &irqn_not_used);
+ if (err)
+ return err;
+
+ inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
+ sizeof(u64) * cq->wq_ctrl.buf.npages;
+ in = kvzalloc(inlen, GFP_KERNEL);
+ if (!in)
+ return -ENOMEM;
+
+ cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
+
+ memcpy(cqc, cqc_data, MLX5_ST_SZ_BYTES(cqc));
+
+ mlx5_fill_page_array(&cq->wq_ctrl.buf, (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas));
+
+ MLX5_SET(cqc, cqc, cq_period_mode, 0);
+ MLX5_SET(cqc, cqc, c_eqn, eqn);
+ MLX5_SET(cqc, cqc, uar_page, mdev->priv.uar->index);
+ MLX5_SET(cqc, cqc, log_page_size, cq->wq_ctrl.buf.page_shift -
+ MLX5_ADAPTER_PAGE_SHIFT);
+ MLX5_SET64(cqc, cqc, dbr_addr, cq->wq_ctrl.db.dma);
+
+ err = mlx5_core_create_cq(mdev, mcq, in, inlen, out, sizeof(out));
+
+ kvfree(in);
+
+ return err;
+}
+
+static void mlx5_aso_destroy_cq(struct mlx5_aso_cq *cq)
+{
+ mlx5_core_destroy_cq(cq->mdev, &cq->mcq);
+ mlx5_wq_destroy(&cq->wq_ctrl);
+}
+
+static int mlx5_aso_create_cq(struct mlx5_core_dev *mdev, int numa_node,
+ struct mlx5_aso_cq *cq)
+{
+ void *cqc_data;
+ int err;
+
+ cqc_data = kvzalloc(MLX5_ST_SZ_BYTES(cqc), GFP_KERNEL);
+ if (!cqc_data)
+ return -ENOMEM;
+
+ MLX5_SET(cqc, cqc_data, log_cq_size, 1);
+ MLX5_SET(cqc, cqc_data, uar_page, mdev->priv.uar->index);
+ if (MLX5_CAP_GEN(mdev, cqe_128_always) && cache_line_size() >= 128)
+ MLX5_SET(cqc, cqc_data, cqe_sz, CQE_STRIDE_128_PAD);
+
+ err = mlx5_aso_alloc_cq(mdev, numa_node, cqc_data, cq);
+ if (err) {
+ mlx5_core_err(mdev, "Failed to alloc aso wq cq, err=%d\n", err);
+ goto err_out;
+ }
+
+ err = create_aso_cq(cq, cqc_data);
+ if (err) {
+ mlx5_core_err(mdev, "Failed to create aso wq cq, err=%d\n", err);
+ goto err_free_cq;
+ }
+
+ kvfree(cqc_data);
+ return 0;
+
+err_free_cq:
+ mlx5_aso_free_cq(cq);
+err_out:
+ kvfree(cqc_data);
+ return err;
+}
+
+static int mlx5_aso_alloc_sq(struct mlx5_core_dev *mdev, int numa_node,
+ void *sqc_data, struct mlx5_aso *sq)
+{
+ void *sqc_wq = MLX5_ADDR_OF(sqc, sqc_data, wq);
+ struct mlx5_wq_cyc *wq = &sq->wq;
+ struct mlx5_wq_param param;
+ int err;
+
+ sq->uar_map = mdev->priv.uar->map;
+
+ param.linear = 1;
+ err = mlx5_wq_cyc_create(mdev, &param, sqc_wq, wq, &sq->wq_ctrl);
+ if (err)
+ return err;
+
+ wq->db = &wq->db[MLX5_SND_DBR];
+
+ return 0;
+}
+
+static int create_aso_sq(struct mlx5_core_dev *mdev, int pdn,
+ void *sqc_data, struct mlx5_aso *sq)
+{
+ void *in, *sqc, *wq;
+ int inlen, err;
+ u8 ts_format;
+
+ inlen = MLX5_ST_SZ_BYTES(create_sq_in) +
+ sizeof(u64) * sq->wq_ctrl.buf.npages;
+ in = kvzalloc(inlen, GFP_KERNEL);
+ if (!in)
+ return -ENOMEM;
+
+ sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
+ wq = MLX5_ADDR_OF(sqc, sqc, wq);
+
+ memcpy(sqc, sqc_data, MLX5_ST_SZ_BYTES(sqc));
+ MLX5_SET(sqc, sqc, cqn, sq->cq.mcq.cqn);
+
+ MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST);
+ MLX5_SET(sqc, sqc, flush_in_error_en, 1);
+
+ ts_format = mlx5_get_sq_default_ts(mdev);
+ MLX5_SET(sqc, sqc, ts_format, ts_format);
+
+ MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
+ MLX5_SET(wq, wq, uar_page, mdev->priv.uar->index);
+ MLX5_SET(wq, wq, log_wq_pg_sz, sq->wq_ctrl.buf.page_shift -
+ MLX5_ADAPTER_PAGE_SHIFT);
+ MLX5_SET64(wq, wq, dbr_addr, sq->wq_ctrl.db.dma);
+
+ mlx5_fill_page_array(&sq->wq_ctrl.buf, (__be64 *)MLX5_ADDR_OF(wq, wq, pas));
+
+ err = mlx5_core_create_sq(mdev, in, inlen, &sq->sqn);
+
+ kvfree(in);
+
+ return err;
+}
+
+static int mlx5_aso_set_sq_rdy(struct mlx5_core_dev *mdev, u32 sqn)
+{
+ void *in, *sqc;
+ int inlen, err;
+
+ inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
+ in = kvzalloc(inlen, GFP_KERNEL);
+ if (!in)
+ return -ENOMEM;
+
+ MLX5_SET(modify_sq_in, in, sq_state, MLX5_SQC_STATE_RST);
+ MLX5_SET(modify_sq_in, in, sqn, sqn);
+ sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
+ MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RDY);
+
+ err = mlx5_core_modify_sq(mdev, in, inlen);
+
+ kvfree(in);
+
+ return err;
+}
+
+static int mlx5_aso_create_sq_rdy(struct mlx5_core_dev *mdev, u32 pdn,
+ void *sqc_data, struct mlx5_aso *sq)
+{
+ int err;
+
+ err = create_aso_sq(mdev, pdn, sqc_data, sq);
+ if (err)
+ return err;
+
+ err = mlx5_aso_set_sq_rdy(mdev, sq->sqn);
+ if (err)
+ mlx5_core_destroy_sq(mdev, sq->sqn);
+
+ return err;
+}
+
+static void mlx5_aso_free_sq(struct mlx5_aso *sq)
+{
+ mlx5_wq_destroy(&sq->wq_ctrl);
+}
+
+static void mlx5_aso_destroy_sq(struct mlx5_aso *sq)
+{
+ mlx5_core_destroy_sq(sq->cq.mdev, sq->sqn);
+ mlx5_aso_free_sq(sq);
+}
+
+static int mlx5_aso_create_sq(struct mlx5_core_dev *mdev, int numa_node,
+ u32 pdn, struct mlx5_aso *sq)
+{
+ void *sqc_data, *wq;
+ int err;
+
+ sqc_data = kvzalloc(MLX5_ST_SZ_BYTES(sqc), GFP_KERNEL);
+ if (!sqc_data)
+ return -ENOMEM;
+
+ wq = MLX5_ADDR_OF(sqc, sqc_data, wq);
+ MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
+ MLX5_SET(wq, wq, pd, pdn);
+ MLX5_SET(wq, wq, log_wq_sz, 1);
+
+ err = mlx5_aso_alloc_sq(mdev, numa_node, sqc_data, sq);
+ if (err) {
+ mlx5_core_err(mdev, "Failed to alloc aso wq sq, err=%d\n", err);
+ goto err_out;
+ }
+
+ err = mlx5_aso_create_sq_rdy(mdev, pdn, sqc_data, sq);
+ if (err) {
+ mlx5_core_err(mdev, "Failed to open aso wq sq, err=%d\n", err);
+ goto err_free_asosq;
+ }
+
+ mlx5_core_dbg(mdev, "aso sq->sqn = 0x%x\n", sq->sqn);
+
+ kvfree(sqc_data);
+ return 0;
+
+err_free_asosq:
+ mlx5_aso_free_sq(sq);
+err_out:
+ kvfree(sqc_data);
+ return err;
+}
+
+struct mlx5_aso *mlx5_aso_create(struct mlx5_core_dev *mdev, u32 pdn)
+{
+ int numa_node = dev_to_node(&mdev->pdev->dev);
+ struct mlx5_aso *aso;
+ int err;
+
+ aso = kzalloc(sizeof(*aso), GFP_KERNEL);
+ if (!aso)
+ return ERR_PTR(-ENOMEM);
+
+ err = mlx5_aso_create_cq(mdev, numa_node, &aso->cq);
+ if (err)
+ goto err_cq;
+
+ err = mlx5_aso_create_sq(mdev, numa_node, pdn, aso);
+ if (err)
+ goto err_sq;
+
+ return aso;
+
+err_sq:
+ mlx5_aso_destroy_cq(&aso->cq);
+err_cq:
+ kfree(aso);
+ return ERR_PTR(err);
+}
+
+void mlx5_aso_destroy(struct mlx5_aso *aso)
+{
+ mlx5_aso_destroy_sq(aso);
+ mlx5_aso_destroy_cq(&aso->cq);
+ kfree(aso);
+}
+
+void mlx5_aso_build_wqe(struct mlx5_aso *aso, u8 ds_cnt,
+ struct mlx5_aso_wqe *aso_wqe,
+ u32 obj_id, u32 opc_mode)
+{
+ struct mlx5_wqe_ctrl_seg *cseg = &aso_wqe->ctrl;
+
+ cseg->opmod_idx_opcode = cpu_to_be32((opc_mode << MLX5_WQE_CTRL_WQE_OPC_MOD_SHIFT) |
+ (aso->pc << MLX5_WQE_CTRL_WQE_INDEX_SHIFT) |
+ MLX5_OPCODE_ACCESS_ASO);
+ cseg->qpn_ds = cpu_to_be32((aso->sqn << MLX5_WQE_CTRL_QPN_SHIFT) | ds_cnt);
+ cseg->fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
+ cseg->general_id = cpu_to_be32(obj_id);
+}
+
+struct mlx5_aso_wqe *mlx5_aso_get_wqe(struct mlx5_aso *aso)
+{
+ struct mlx5_aso_wqe *wqe;
+ u16 pi;
+
+ pi = mlx5_wq_cyc_ctr2ix(&aso->wq, aso->pc);
+ wqe = mlx5_wq_cyc_get_wqe(&aso->wq, pi);
+ memset(wqe, 0, sizeof(*wqe));
+ return wqe;
+}
+
+void mlx5_aso_post_wqe(struct mlx5_aso *aso, bool with_data,
+ struct mlx5_wqe_ctrl_seg *doorbell_cseg)
+{
+ doorbell_cseg->fm_ce_se |= MLX5_WQE_CTRL_CQ_UPDATE;
+ /* ensure wqe is visible to device before updating doorbell record */
+ wmb();
+
+ if (with_data)
+ aso->pc += MLX5_ASO_WQEBBS_DATA;
+ else
+ aso->pc += MLX5_ASO_WQEBBS;
+ *aso->wq.db = cpu_to_be32(aso->pc);
+
+ /* ensure doorbell record is visible to device before ringing the
+ * doorbell
+ */
+ wmb();
+
+ mlx5_write64((__be32 *)doorbell_cseg, aso->uar_map + MLX5_BF_OFFSET, NULL);
+
+ /* Ensure doorbell is written on uar_page before poll_cq */
+ WRITE_ONCE(doorbell_cseg, NULL);
+}
+
+int mlx5_aso_poll_cq(struct mlx5_aso *aso, bool with_data)
+{
+ struct mlx5_aso_cq *cq = &aso->cq;
+ struct mlx5_cqe64 *cqe;
+
+ cqe = mlx5_cqwq_get_cqe(&cq->wq);
+ if (!cqe)
+ return -ETIMEDOUT;
+
+ /* sq->cc must be updated only after mlx5_cqwq_update_db_record(),
+ * otherwise a cq overrun may occur
+ */
+ mlx5_cqwq_pop(&cq->wq);
+
+ if (unlikely(get_cqe_opcode(cqe) != MLX5_CQE_REQ)) {
+ struct mlx5_err_cqe *err_cqe;
+
+ mlx5_core_err(cq->mdev, "Bad OP in ASOSQ CQE: 0x%x\n",
+ get_cqe_opcode(cqe));
+
+ err_cqe = (struct mlx5_err_cqe *)cqe;
+ mlx5_core_err(cq->mdev, "vendor_err_synd=%x\n",
+ err_cqe->vendor_err_synd);
+ mlx5_core_err(cq->mdev, "syndrome=%x\n",
+ err_cqe->syndrome);
+ print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET,
+ 16, 1, err_cqe,
+ sizeof(*err_cqe), false);
+ }
+
+ mlx5_cqwq_update_db_record(&cq->wq);
+
+ /* ensure cq space is freed before enabling more cqes */
+ wmb();
+
+ if (with_data)
+ aso->cc += MLX5_ASO_WQEBBS_DATA;
+ else
+ aso->cc += MLX5_ASO_WQEBBS;
+
+ return 0;
+}
diff --git a/sys/dev/mlx5/qp.h b/sys/dev/mlx5/qp.h
--- a/sys/dev/mlx5/qp.h
+++ b/sys/dev/mlx5/qp.h
@@ -146,6 +146,8 @@
#define MLX5_SEND_WQE_DS 16
#define MLX5_SEND_WQE_BB 64
#define MLX5_SEND_WQEBB_NUM_DS (MLX5_SEND_WQE_BB / MLX5_SEND_WQE_DS)
+#define MLX5_WQE_CTRL_QPN_SHIFT 8
+#define MLX5_WQE_CTRL_WQE_INDEX_SHIFT 8
enum {
MLX5_SEND_WQE_MAX_WQEBBS = 16,
@@ -194,7 +196,10 @@
u8 signature;
u8 rsvd[2];
u8 fm_ce_se;
- __be32 imm;
+ union {
+ __be32 imm;
+ __be32 general_id;
+ };
};
#define MLX5_WQE_CTRL_DS_MASK 0x3f
@@ -228,6 +233,10 @@
MLX5_ETH_WQE_SWP_OUTER_L4_TYPE = 1 << 5,
};
+enum {
+ MLX5_ETH_WQE_FT_META_IPSEC = BIT(0),
+};
+
struct mlx5_wqe_eth_seg {
u8 swp_outer_l4_offset;
u8 swp_outer_l3_offset;
@@ -236,7 +245,7 @@
u8 cs_flags;
u8 swp_flags;
__be16 mss;
- __be32 rsvd2;
+ __be32 flow_table_metadata;
union {
struct {
__be16 inline_hdr_sz;
diff --git a/sys/dev/mlx5/tls.h b/sys/dev/mlx5/tls.h
--- a/sys/dev/mlx5/tls.h
+++ b/sys/dev/mlx5/tls.h
@@ -30,9 +30,6 @@
struct mlx5_core_dev;
-int mlx5_encryption_key_create(struct mlx5_core_dev *mdev, u32 pdn,
- const void *p_key, u32 key_len, u32 * p_obj_id);
-int mlx5_encryption_key_destroy(struct mlx5_core_dev *mdev, u32 oid);
int mlx5_tls_open_tis(struct mlx5_core_dev *mdev, int tc, int tdn, int pdn, u32 *p_tisn);
void mlx5_tls_close_tis(struct mlx5_core_dev *mdev, u32 tisn);
int mlx5_tls_open_tir(struct mlx5_core_dev *mdev, int tdn, int rqtn, u32 *p_tirn);
diff --git a/sys/modules/mlx5/Makefile b/sys/modules/mlx5/Makefile
--- a/sys/modules/mlx5/Makefile
+++ b/sys/modules/mlx5/Makefile
@@ -1,21 +1,25 @@
# $FreeBSD$
.PATH: ${.CURDIR}/../../dev/mlx5/mlx5_core \
${.CURDIR}/../../dev/mlx5/mlx5_lib \
+ ${.CURDIR}/../../dev/mlx5/mlx5_accel
KMOD=mlx5
SRCS= \
mlx5_alloc.c \
+mlx5_aso.c \
mlx5_cmd.c \
+mlx5_crypto.c \
mlx5_cq.c \
mlx5_diag_cnt.c \
mlx5_diagnostics.c \
mlx5_eq.c \
mlx5_eswitch.c \
-mlx5_fc_cmd.c \
+mlx5_fs_chains.c \
mlx5_fs_cmd.c \
-mlx5_fs_tcp.c \
-mlx5_fs_tree.c \
+mlx5_fs_core.c \
mlx5_fs_counters.c \
+mlx5_fs_ft_pool.c \
+mlx5_fs_tcp.c \
mlx5_fw.c \
mlx5_fwdump.c \
mlx5_health.c \
@@ -37,6 +41,10 @@
mlx5_vsc.c \
mlx5_wq.c \
mlx5_gid.c \
+mlx5_ipsec_fs.c \
+mlx5_ipsec_offload.c \
+mlx5_ipsec.c \
+mlx5_ipsec_rxtx.c \
device_if.h bus_if.h vnode_if.h pci_if.h pci_iov_if.h \
opt_inet.h opt_inet6.h opt_rss.h opt_ratelimit.h

File Metadata

Mime Type
text/plain
Expires
Sun, May 31, 3:46 AM (9 h, 45 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
33645079
Default Alt Text
D44220.id135396.diff (512 KB)

Event Timeline