diff --git a/sys/dev/bnxt/bnxt_re/bnxt_re-abi.h b/sys/dev/bnxt/bnxt_re/bnxt_re-abi.h
new file mode 100644
index 000000000000..8f48609e7f6f
--- /dev/null
+++ b/sys/dev/bnxt/bnxt_re/bnxt_re-abi.h
@@ -0,0 +1,177 @@
+/*
+ * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term
+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Description: Uverbs ABI header file
+ */
+
+#ifndef __BNXT_RE_UVERBS_ABI_H__
+#define __BNXT_RE_UVERBS_ABI_H__
+
+#include <asm/types.h>
+#include <linux/types.h>
+
+#define BNXT_RE_ABI_VERSION 6
+
+enum {
+ BNXT_RE_COMP_MASK_UCNTX_WC_DPI_ENABLED = 0x01,
+ BNXT_RE_COMP_MASK_UCNTX_POW2_DISABLED = 0x02,
+ BNXT_RE_COMP_MASK_UCNTX_RSVD_WQE_DISABLED = 0x04,
+ BNXT_RE_COMP_MASK_UCNTX_MQP_EX_SUPPORTED = 0x08,
+ BNXT_RE_COMP_MASK_UCNTX_DBR_PACING_ENABLED = 0x10,
+ BNXT_RE_COMP_MASK_UCNTX_DBR_RECOVERY_ENABLED = 0x20,
+ BNXT_RE_COMP_MASK_UCNTX_HW_RETX_ENABLED = 0x40
+};
+
+enum {
+ BNXT_RE_COMP_MASK_REQ_UCNTX_POW2_SUPPORT = 0x01,
+ BNXT_RE_COMP_MASK_REQ_UCNTX_RSVD_WQE = 0x02
+};
+
+struct bnxt_re_uctx_req {
+ __aligned_u64 comp_mask;
+};
+
+#define BNXT_RE_CHIP_ID0_CHIP_NUM_SFT 0x00
+#define BNXT_RE_CHIP_ID0_CHIP_REV_SFT 0x10
+#define BNXT_RE_CHIP_ID0_CHIP_MET_SFT 0x18
+struct bnxt_re_uctx_resp {
+ __u32 dev_id;
+ __u32 max_qp;
+ __u32 pg_size;
+ __u32 cqe_sz;
+ __u32 max_cqd;
+ __u32 chip_id0;
+ __u32 chip_id1;
+ __u32 modes;
+ __aligned_u64 comp_mask;
+} __attribute__((packed));
+
+enum {
+ BNXT_RE_COMP_MASK_PD_HAS_WC_DPI = 0x01,
+ BNXT_RE_COMP_MASK_PD_HAS_DBR_BAR_ADDR = 0x02,
+};
+
+struct bnxt_re_pd_resp {
+ __u32 pdid;
+ __u32 dpi;
+ __u64 dbr;
+ __u64 comp_mask;
+ __u32 wcdpi;
+ __u64 dbr_bar_addr;
+} __attribute__((packed));
+
+enum {
+ BNXT_RE_COMP_MASK_CQ_HAS_DB_INFO = 0x01,
+ BNXT_RE_COMP_MASK_CQ_HAS_WC_DPI = 0x02,
+ BNXT_RE_COMP_MASK_CQ_HAS_CQ_PAGE = 0x04,
+};
+
+enum {
+ BNXT_RE_COMP_MASK_CQ_REQ_HAS_CAP_MASK = 0x1
+};
+
+enum {
+ BNXT_RE_COMP_MASK_CQ_REQ_CAP_DBR_RECOVERY = 0x1,
+ BNXT_RE_COMP_MASK_CQ_REQ_CAP_DBR_PACING_NOTIFY = 0x2
+};
+
+#define BNXT_RE_IS_DBR_PACING_NOTIFY_CQ(_req) \
+ (_req.comp_mask & BNXT_RE_COMP_MASK_CQ_REQ_HAS_CAP_MASK && \
+ _req.cq_capability & BNXT_RE_COMP_MASK_CQ_REQ_CAP_DBR_PACING_NOTIFY)
+
+#define BNXT_RE_IS_DBR_RECOV_CQ(_req) \
+ (_req.comp_mask & BNXT_RE_COMP_MASK_CQ_REQ_HAS_CAP_MASK && \
+ _req.cq_capability & BNXT_RE_COMP_MASK_CQ_REQ_CAP_DBR_RECOVERY)
+
+struct bnxt_re_cq_req {
+ __u64 cq_va;
+ __u64 cq_handle;
+ __aligned_u64 comp_mask;
+ __u16 cq_capability;
+} __attribute__((packed));
+
+struct bnxt_re_cq_resp {
+ __u32 cqid;
+ __u32 tail;
+ __u32 phase;
+ __u32 rsvd;
+ __aligned_u64 comp_mask;
+ __u32 dpi;
+ __u64 dbr;
+ __u32 wcdpi;
+ __u64 uctx_cq_page;
+} __attribute__((packed));
+
+struct bnxt_re_resize_cq_req {
+ __u64 cq_va;
+} __attribute__((packed));
+
+struct bnxt_re_qp_req {
+ __u64 qpsva;
+ __u64 qprva;
+ __u64 qp_handle;
+} __attribute__((packed));
+
+struct bnxt_re_qp_resp {
+ __u32 qpid;
+} __attribute__((packed));
+
+struct bnxt_re_srq_req {
+ __u64 srqva;
+ __u64 srq_handle;
+} __attribute__((packed));
+
+struct bnxt_re_srq_resp {
+ __u32 srqid;
+} __attribute__((packed));
+
+/* Modify QP */
+enum {
+ BNXT_RE_COMP_MASK_MQP_EX_PPP_REQ_EN_MASK = 0x1,
+ BNXT_RE_COMP_MASK_MQP_EX_PPP_REQ_EN = 0x1,
+ BNXT_RE_COMP_MASK_MQP_EX_PATH_MTU_MASK = 0x2
+};
+
+struct bnxt_re_modify_qp_ex_req {
+ __aligned_u64 comp_mask;
+ __u32 dpi;
+ __u32 rsvd;
+} __packed;
+
+struct bnxt_re_modify_qp_ex_resp {
+ __aligned_u64 comp_mask;
+ __u32 ppp_st_idx;
+ __u32 path_mtu;
+} __packed;
+
+enum bnxt_re_shpg_offt {
+ BNXT_RE_BEG_RESV_OFFT = 0x00,
+ BNXT_RE_AVID_OFFT = 0x10,
+ BNXT_RE_AVID_SIZE = 0x04,
+ BNXT_RE_END_RESV_OFFT = 0xFF0
+};
+#endif
diff --git a/sys/dev/bnxt/bnxt_re/bnxt_re.h b/sys/dev/bnxt/bnxt_re/bnxt_re.h
new file mode 100644
index 000000000000..3bf0bbeb1061
--- /dev/null
+++ b/sys/dev/bnxt/bnxt_re/bnxt_re.h
@@ -0,0 +1,1075 @@
+/*
+ * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term
+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Description: main (header)
+ */
+
+#ifndef __BNXT_RE_H__
+#define __BNXT_RE_H__
+
+#include <linux/module.h>
+#include <linux/netdevice.h>
+#include <linux/mutex.h>
+#include <linux/list.h>
+#include <linux/rculist.h>
+#include <linux/spinlock.h>
+#include <net/ipv6.h>
+#include <linux/if_ether.h>
+#include <linux/debugfs.h>
+#include <linux/seq_file.h>
+#include <linux/interrupt.h>
+#include <linux/vmalloc.h>
+#include <linux/delay.h>
+#include <rdma/ib_verbs.h>
+#include <rdma/ib_user_verbs.h>
+#include <rdma/ib_umem.h>
+#include <rdma/ib_addr.h>
+#include <rdma/ib_mad.h>
+#include <rdma/ib_cache.h>
+#include <linux/pci.h>
+
+#include "bnxt.h"
+#include "bnxt_ulp.h"
+#include "hsi_struct_def.h"
+#include "qplib_res.h"
+#include "qplib_sp.h"
+#include "qplib_fp.h"
+#include "qplib_rcfw.h"
+#include "ib_verbs.h"
+#include "stats.h"
+
+#define ROCE_DRV_MODULE_NAME "bnxt_re"
+#define ROCE_DRV_MODULE_VERSION "230.0.133.0"
+#define ROCE_DRV_MODULE_RELDATE "April 22, 2024"
+
+#define BNXT_RE_REF_WAIT_COUNT 20
+#define BNXT_RE_ROCE_V1_ETH_TYPE 0x8915
+#define BNXT_RE_ROCE_V2_PORT_NO 4791
+#define BNXT_RE_RES_FREE_WAIT_COUNT 1000
+
+#define BNXT_RE_PAGE_SHIFT_4K (12)
+#define BNXT_RE_PAGE_SHIFT_8K (13)
+#define BNXT_RE_PAGE_SHIFT_64K (16)
+#define BNXT_RE_PAGE_SHIFT_2M (21)
+#define BNXT_RE_PAGE_SHIFT_8M (23)
+#define BNXT_RE_PAGE_SHIFT_1G (30)
+
+#define BNXT_RE_PAGE_SIZE_4K BIT(BNXT_RE_PAGE_SHIFT_4K)
+#define BNXT_RE_PAGE_SIZE_8K BIT(BNXT_RE_PAGE_SHIFT_8K)
+#define BNXT_RE_PAGE_SIZE_64K BIT(BNXT_RE_PAGE_SHIFT_64K)
+#define BNXT_RE_PAGE_SIZE_2M BIT(BNXT_RE_PAGE_SHIFT_2M)
+#define BNXT_RE_PAGE_SIZE_8M BIT(BNXT_RE_PAGE_SHIFT_8M)
+#define BNXT_RE_PAGE_SIZE_1G BIT(BNXT_RE_PAGE_SHIFT_1G)
+
+#define BNXT_RE_MAX_MR_SIZE_LOW BIT(BNXT_RE_PAGE_SHIFT_1G)
+#define BNXT_RE_MAX_MR_SIZE_HIGH BIT(39)
+#define BNXT_RE_MAX_MR_SIZE BNXT_RE_MAX_MR_SIZE_HIGH
+
+/* Number of MRs to reserve for PF, leaving remainder for VFs */
+#define BNXT_RE_RESVD_MR_FOR_PF (32 * 1024)
+#define BNXT_RE_MAX_GID_PER_VF 128
+
+#define BNXT_RE_MAX_VF_QPS_PER_PF (6 * 1024)
+
+/**
+ * min_not_zero - return the minimum that is _not_ zero, unless both are zero
+ * @x: value1
+ * @y: value2
+ */
+#define min_not_zero(x, y) ({ \
+ typeof(x) __x = (x); \
+ typeof(y) __y = (y); \
+ __x == 0 ? __y : ((__y == 0) ? __x : min(__x, __y)); })
+
+struct ib_mr_init_attr {
+ int max_reg_descriptors;
+ u32 flags;
+};
+
+struct bnxt_re_dev;
+
+int bnxt_re_register_netdevice_notifier(struct notifier_block *nb);
+int bnxt_re_unregister_netdevice_notifier(struct notifier_block *nb);
+int ib_register_device_compat(struct bnxt_re_dev *rdev);
+
+#ifndef __struct_group
+#define __struct_group(TAG, NAME, ATTRS, MEMBERS...) \
+ union { \
+ struct { MEMBERS } ATTRS; \
+ struct TAG { MEMBERS } ATTRS NAME; \
+ }
+#endif /* __struct_group */
+#ifndef struct_group_attr
+#define struct_group_attr(NAME, ATTRS, MEMBERS...) \
+ __struct_group(/* no tag */, NAME, ATTRS, MEMBERS)
+#endif /* struct_group_attr */
+/*
+ * Percentage of resources of each type reserved for PF.
+ * Remaining resources are divided equally among VFs.
+ * [0, 100]
+ */
+
+#define BNXT_RE_RQ_WQE_THRESHOLD 32
+#define BNXT_RE_UD_QP_HW_STALL 0x400000
+
+/*
+ * Setting the default ack delay value to 16, which means
+ * the default timeout is approx. 260ms(4 usec * 2 ^(timeout))
+ */
+
+#define BNXT_RE_DEFAULT_ACK_DELAY 16
+#define BNXT_RE_BOND_PF_MAX 2
+
+#define BNXT_RE_STATS_CTX_UPDATE_TIMER 250
+#define BNXT_RE_30SEC_MSEC (30 * 1000)
+
+#define BNXT_RE_BOND_RESCHED_CNT 10
+
+#define BNXT_RE_CHIP_NUM_57454 0xC454
+#define BNXT_RE_CHIP_NUM_57452 0xC452
+
+#define BNXT_RE_CHIP_NUM_5745X(chip_num) \
+ ((chip_num) == BNXT_RE_CHIP_NUM_57454 || \
+ (chip_num) == BNXT_RE_CHIP_NUM_57452)
+
+#define BNXT_RE_MIN_KERNEL_QP_TX_DEPTH 4096
+#define BNXT_RE_STOP_QPS_BUDGET 200
+
+#define BNXT_RE_HWRM_CMD_TIMEOUT(rdev) \
+ ((rdev)->chip_ctx->hwrm_cmd_max_timeout * 1000)
+
+extern unsigned int min_tx_depth;
+extern struct mutex bnxt_re_dev_lock;
+extern struct mutex bnxt_re_mutex;
+extern struct list_head bnxt_re_dev_list;
+
+struct bnxt_re_ring_attr {
+ dma_addr_t *dma_arr;
+ int pages;
+ int type;
+ u32 depth;
+ u32 lrid; /* Logical ring id */
+ u16 flags;
+ u8 mode;
+ u8 rsvd;
+};
+
+#define BNXT_RE_MAX_DEVICES 256
+#define BNXT_RE_MSIX_FROM_MOD_PARAM -1
+#define BNXT_RE_MIN_MSIX 2
+#define BNXT_RE_MAX_MSIX_VF 2
+#define BNXT_RE_MAX_MSIX_PF 9
+#define BNXT_RE_MAX_MSIX_NPAR_PF 5
+#define BNXT_RE_MAX_MSIX 64
+#define BNXT_RE_MAX_MSIX_GEN_P5_PF BNXT_RE_MAX_MSIX
+#define BNXT_RE_GEN_P5_MAX_VF 64
+
+struct bnxt_re_nq_record {
+ struct bnxt_msix_entry msix_entries[BNXT_RE_MAX_MSIX];
+ /* FP Notification Queue (CQ & SRQ) */
+ struct bnxt_qplib_nq nq[BNXT_RE_MAX_MSIX];
+ int num_msix;
+ int max_init;
+ struct mutex load_lock;
+};
+
+struct bnxt_re_work {
+ struct work_struct work;
+ unsigned long event;
+ struct bnxt_re_dev *rdev;
+ struct ifnet *vlan_dev;
+ bool do_lag;
+
+ /* netdev where we received the event */
+ struct ifnet *netdev;
+ struct auxiliary_device *adev;
+};
+
+/*
+ * Data structure and defines to handle
+ * recovery
+ */
+#define BNXT_RE_RECOVERY_IB_UNINIT_WAIT_RETRY 20
+#define BNXT_RE_RECOVERY_IB_UNINIT_WAIT_TIME_MS 30000 /* 30sec timeout */
+#define BNXT_RE_PRE_RECOVERY_REMOVE 0x1
+#define BNXT_RE_COMPLETE_REMOVE 0x2
+#define BNXT_RE_POST_RECOVERY_INIT 0x4
+#define BNXT_RE_COMPLETE_INIT 0x8
+#define BNXT_RE_COMPLETE_SHUTDOWN 0x10
+
+/* QP1 SQ entry data strucutre */
+struct bnxt_re_sqp_entries {
+ u64 wrid;
+ struct bnxt_qplib_sge sge;
+ /* For storing the actual qp1 cqe */
+ struct bnxt_qplib_cqe cqe;
+ struct bnxt_re_qp *qp1_qp;
+};
+
+/* GSI QP mode enum */
+enum bnxt_re_gsi_mode {
+ BNXT_RE_GSI_MODE_INVALID = 0,
+ BNXT_RE_GSI_MODE_ALL = 1,
+ BNXT_RE_GSI_MODE_ROCE_V1,
+ BNXT_RE_GSI_MODE_ROCE_V2_IPV4,
+ BNXT_RE_GSI_MODE_ROCE_V2_IPV6,
+ BNXT_RE_GSI_MODE_UD
+};
+
+enum bnxt_re_roce_cap {
+ BNXT_RE_FLAG_ROCEV1_CAP = 1,
+ BNXT_RE_FLAG_ROCEV2_CAP,
+ BNXT_RE_FLAG_ROCEV1_V2_CAP,
+};
+
+#define BNXT_RE_MAX_GSI_SQP_ENTRIES 1024
+struct bnxt_re_gsi_context {
+ u8 gsi_qp_mode;
+ bool first_cq_created;
+ /* Start: used only in gsi_mode_all */
+ struct bnxt_re_qp *gsi_qp;
+ struct bnxt_re_qp *gsi_sqp;
+ struct bnxt_re_ah *gsi_sah;
+ struct bnxt_re_sqp_entries *sqp_tbl;
+ /* End: used only in gsi_mode_all */
+};
+
+struct bnxt_re_tc_rec {
+ u8 cos_id_roce;
+ u8 tc_roce;
+ u8 cos_id_cnp;
+ u8 tc_cnp;
+ u8 tc_def;
+ u8 cos_id_def;
+ u8 max_tc;
+ u8 roce_prio;
+ u8 cnp_prio;
+ u8 roce_dscp;
+ u8 cnp_dscp;
+ u8 prio_valid;
+ u8 dscp_valid;
+ bool ecn_enabled;
+ bool serv_type_enabled;
+ u64 cnp_dscp_bv;
+ u64 roce_dscp_bv;
+};
+
+struct bnxt_re_dscp2pri {
+ u8 dscp;
+ u8 mask;
+ u8 pri;
+};
+
+struct bnxt_re_cos2bw_cfg {
+ u8 pad[3];
+ struct_group_attr(cfg, __packed,
+ u8 queue_id;
+ __le32 min_bw;
+ __le32 max_bw;
+ u8 tsa;
+ u8 pri_lvl;
+ u8 bw_weight;
+ );
+ u8 unused;
+};
+
+#define BNXT_RE_AEQ_IDX 0
+#define BNXT_RE_MAX_SGID_ENTRIES 256
+
+#define BNXT_RE_DBGFS_FILE_MEM 65536
+enum {
+ BNXT_RE_STATS_QUERY = 1,
+ BNXT_RE_QP_QUERY = 2,
+ BNXT_RE_SERVICE_FN_QUERY = 3,
+};
+
+struct bnxt_re_dbg_file {
+ struct bnxt_re_dev *rdev;
+ u32 type;
+ union {
+ struct bnxt_qplib_query_stats_info sinfo;
+ struct bnxt_qplib_query_fn_info fninfo;
+ }params;
+ char dbg_buf[BNXT_RE_DBGFS_FILE_MEM];
+};
+
+struct bnxt_re_debug_entries {
+ /* Dir entries */
+ struct dentry *qpinfo_dir;
+ struct dentry *service_fn_dir;
+ /* file entries */
+ struct dentry *stat_query;
+ struct bnxt_re_dbg_file stat_file;
+ struct dentry *qplist_query;
+ struct bnxt_re_dbg_file qp_file;
+ struct dentry *service_fn_query;
+ struct bnxt_re_dbg_file service_fn_file;
+};
+
+struct bnxt_re_en_dev_info {
+ struct list_head en_list;
+ struct bnxt_en_dev *en_dev;
+ struct bnxt_re_dev *rdev;
+ unsigned long flags;
+#define BNXT_RE_FLAG_EN_DEV_NETDEV_REG 0
+#define BNXT_RE_FLAG_EN_DEV_PRIMARY_DEV 1
+#define BNXT_RE_FLAG_EN_DEV_SECONDARY_DEV 2
+ u8 wqe_mode;
+ u8 gsi_mode;
+ bool te_bypass;
+ bool ib_uninit_done;
+ u32 num_msix_requested;
+ wait_queue_head_t waitq;
+};
+
+#define BNXT_RE_DB_FIFO_ROOM_MASK_P5 0x1FFF8000
+#define BNXT_RE_MAX_FIFO_DEPTH_P5 0x2c00
+#define BNXT_RE_DB_FIFO_ROOM_SHIFT 15
+
+#define BNXT_RE_DB_FIFO_ROOM_MASK_P7 0x3FFF8000
+#define BNXT_RE_MAX_FIFO_DEPTH_P7 0x8000
+
+#define BNXT_RE_DB_FIFO_ROOM_MASK(ctx) \
+ (_is_chip_p7((ctx)) ? \
+ BNXT_RE_DB_FIFO_ROOM_MASK_P7 :\
+ BNXT_RE_DB_FIFO_ROOM_MASK_P5)
+#define BNXT_RE_MAX_FIFO_DEPTH(ctx) \
+ (_is_chip_p7((ctx)) ? \
+ BNXT_RE_MAX_FIFO_DEPTH_P7 :\
+ BNXT_RE_MAX_FIFO_DEPTH_P5)
+
+struct bnxt_dbq_nq_list {
+ int num_nql_entries;
+ u16 nq_id[16];
+};
+
+#define BNXT_RE_ASYNC_ERR_REP_BASE(_type) \
+ (ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_##_type)
+
+#define BNXT_RE_ASYNC_ERR_DBR_TRESH(_type) \
+ (ASYNC_EVENT_CMPL_ERROR_REPORT_DOORBELL_DROP_THRESHOLD_##_type)
+
+#define BNXT_RE_EVENT_DBR_EPOCH(data) \
+ (((data) & \
+ BNXT_RE_ASYNC_ERR_DBR_TRESH(EVENT_DATA1_EPOCH_MASK)) >> \
+ BNXT_RE_ASYNC_ERR_DBR_TRESH(EVENT_DATA1_EPOCH_SFT))
+
+#define BNXT_RE_EVENT_ERROR_REPORT_TYPE(data1) \
+ (((data1) & \
+ BNXT_RE_ASYNC_ERR_REP_BASE(TYPE_MASK)) >> \
+ BNXT_RE_ASYNC_ERR_REP_BASE(TYPE_SFT))
+
+#define BNXT_RE_DBR_LIST_ADD(_rdev, _res, _type) \
+{ \
+ spin_lock(&(_rdev)->res_list[_type].lock); \
+ list_add_tail(&(_res)->dbr_list, \
+ &(_rdev)->res_list[_type].head); \
+ spin_unlock(&(_rdev)->res_list[_type].lock); \
+}
+
+#define BNXT_RE_DBR_LIST_DEL(_rdev, _res, _type) \
+{ \
+ spin_lock(&(_rdev)->res_list[_type].lock); \
+ list_del(&(_res)->dbr_list); \
+ spin_unlock(&(_rdev)->res_list[_type].lock); \
+}
+
+#define BNXT_RE_CQ_PAGE_LIST_ADD(_uctx, _cq) \
+{ \
+ mutex_lock(&(_uctx)->cq_lock); \
+ list_add_tail(&(_cq)->cq_list, &(_uctx)->cq_list); \
+ mutex_unlock(&(_uctx)->cq_lock); \
+}
+
+#define BNXT_RE_CQ_PAGE_LIST_DEL(_uctx, _cq) \
+{ \
+ mutex_lock(&(_uctx)->cq_lock); \
+ list_del(&(_cq)->cq_list); \
+ mutex_unlock(&(_uctx)->cq_lock); \
+}
+
+#define BNXT_RE_NETDEV_EVENT(event, x) \
+ do { \
+ if ((event) == (x)) \
+ return #x; \
+ } while (0)
+
+/* Do not change the seq of this enum which is followed by dbr recov */
+enum {
+ BNXT_RE_RES_TYPE_CQ = 0,
+ BNXT_RE_RES_TYPE_UCTX,
+ BNXT_RE_RES_TYPE_QP,
+ BNXT_RE_RES_TYPE_SRQ,
+ BNXT_RE_RES_TYPE_MAX
+};
+
+struct bnxt_re_dbr_res_list {
+ struct list_head head;
+ spinlock_t lock;
+};
+
+struct bnxt_re_dbr_drop_recov_work {
+ struct work_struct work;
+ struct bnxt_re_dev *rdev;
+ u32 curr_epoch;
+};
+
+struct bnxt_re_aer_work {
+ struct work_struct work;
+ struct bnxt_re_dev *rdev;
+};
+
+struct bnxt_re_dbq_stats {
+ u64 fifo_occup_slab_1;
+ u64 fifo_occup_slab_2;
+ u64 fifo_occup_slab_3;
+ u64 fifo_occup_slab_4;
+ u64 fifo_occup_water_mark;
+ u64 do_pacing_slab_1;
+ u64 do_pacing_slab_2;
+ u64 do_pacing_slab_3;
+ u64 do_pacing_slab_4;
+ u64 do_pacing_slab_5;
+ u64 do_pacing_water_mark;
+};
+
+/* Device debug statistics */
+struct bnxt_re_drv_dbg_stats {
+ struct bnxt_re_dbq_stats dbq;
+};
+
+/* DB pacing counters */
+struct bnxt_re_dbr_sw_stats {
+ u64 dbq_int_recv;
+ u64 dbq_int_en;
+ u64 dbq_pacing_resched;
+ u64 dbq_pacing_complete;
+ u64 dbq_pacing_alerts;
+ u64 dbr_drop_recov_events;
+ u64 dbr_drop_recov_timeouts;
+ u64 dbr_drop_recov_timeout_users;
+ u64 dbr_drop_recov_event_skips;
+};
+
+struct bnxt_re_dev {
+ struct ib_device ibdev;
+ struct list_head list;
+ atomic_t ref_count;
+ atomic_t sched_count;
+ unsigned long flags;
+#define BNXT_RE_FLAG_NETDEV_REGISTERED 0
+#define BNXT_RE_FLAG_IBDEV_REGISTERED 1
+#define BNXT_RE_FLAG_GOT_MSIX 2
+#define BNXT_RE_FLAG_HAVE_L2_REF 3
+#define BNXT_RE_FLAG_ALLOC_RCFW 4
+#define BNXT_RE_FLAG_NET_RING_ALLOC 5
+#define BNXT_RE_FLAG_RCFW_CHANNEL_EN 6
+#define BNXT_RE_FLAG_ALLOC_CTX 7
+#define BNXT_RE_FLAG_STATS_CTX_ALLOC 8
+#define BNXT_RE_FLAG_STATS_CTX2_ALLOC 9
+#define BNXT_RE_FLAG_RCFW_CHANNEL_INIT 10
+#define BNXT_RE_FLAG_WORKER_REG 11
+#define BNXT_RE_FLAG_TBLS_ALLOCINIT 12
+#define BNXT_RE_FLAG_SETUP_NQ 13
+#define BNXT_RE_FLAG_BOND_DEV_REGISTERED 14
+#define BNXT_RE_FLAG_PER_PORT_DEBUG_INFO 15
+#define BNXT_RE_FLAG_DEV_LIST_INITIALIZED 16
+#define BNXT_RE_FLAG_ERR_DEVICE_DETACHED 17
+#define BNXT_RE_FLAG_INIT_DCBX_CC_PARAM 18
+#define BNXT_RE_FLAG_STOP_IN_PROGRESS 20
+#define BNXT_RE_FLAG_ISSUE_ROCE_STATS 29
+#define BNXT_RE_FLAG_ISSUE_CFA_FLOW_STATS 30
+ struct ifnet *netdev;
+ struct auxiliary_device *adev;
+ struct bnxt_qplib_chip_ctx *chip_ctx;
+ struct bnxt_en_dev *en_dev;
+ struct bnxt_re_nq_record nqr;
+ int id;
+ struct delayed_work worker;
+ u16 worker_30s;
+ struct bnxt_re_tc_rec tc_rec[2];
+ u8 cur_prio_map;
+ /* RCFW Channel */
+ struct bnxt_qplib_rcfw rcfw;
+ /* Device Resources */
+ struct bnxt_qplib_dev_attr *dev_attr;
+ struct bnxt_qplib_res qplib_res;
+ struct bnxt_qplib_dpi dpi_privileged;
+ struct bnxt_qplib_cc_param cc_param;
+ struct mutex cc_lock;
+ struct mutex qp_lock;
+ struct list_head qp_list;
+ u8 roce_mode;
+
+ /* Max of 2 lossless traffic class supported per port */
+ u16 cosq[2];
+ /* Start: QP for handling QP1 packets */
+ struct bnxt_re_gsi_context gsi_ctx;
+ /* End: QP for handling QP1 packets */
+ bool is_virtfn;
+ u32 num_vfs;
+ u32 espeed;
+ /*
+ * For storing the speed of slave interfaces.
+ * Same as espeed when bond is not configured
+ */
+ u32 sl_espeed;
+ /* To be used for a workaround for ISER stack */
+ u32 min_tx_depth;
+ /* To enable qp debug info. Disabled during driver load */
+ u32 en_qp_dbg;
+ /* Array to handle gid mapping */
+ char *gid_map;
+
+ struct bnxt_re_device_stats stats;
+ struct bnxt_re_drv_dbg_stats *dbg_stats;
+ /* debugfs to expose per port information*/
+ struct dentry *port_debug_dir;
+ struct dentry *info;
+ struct dentry *drv_dbg_stats;
+ struct dentry *sp_perf_stats;
+ struct dentry *pdev_debug_dir;
+ struct dentry *pdev_qpinfo_dir;
+ struct bnxt_re_debug_entries *dbg_ent;
+ struct workqueue_struct *resolve_wq;
+ struct list_head mac_wq_list;
+ struct workqueue_struct *dcb_wq;
+ struct workqueue_struct *aer_wq;
+ u32 event_bitmap[3];
+ bool unreg_sched;
+ u64 dbr_throttling_reg_off;
+ u64 dbr_aeq_arm_reg_off;
+ u64 dbr_db_fifo_reg_off;
+ void *dbr_page;
+ u64 dbr_bar_addr;
+ u32 pacing_algo_th;
+ u32 pacing_en_int_th;
+ u32 do_pacing_save;
+ struct workqueue_struct *dbq_wq;
+ struct workqueue_struct *dbr_drop_recov_wq;
+ struct work_struct dbq_fifo_check_work;
+ struct delayed_work dbq_pacing_work;
+ /* protect DB pacing */
+ struct mutex dbq_lock;
+ /* Control DBR pacing feature. Set if enabled */
+ bool dbr_pacing;
+ /* Control DBR recovery feature. Set if enabled */
+ bool dbr_drop_recov;
+ bool user_dbr_drop_recov;
+ /* DBR recovery feature. Set if running */
+ bool dbr_recovery_on;
+ u32 user_dbr_drop_recov_timeout;
+ /*
+ * Value used for pacing algo when pacing is active
+ */
+#define BNXT_RE_MAX_DBR_DO_PACING 0xFFFF
+ u32 dbr_do_pacing;
+ u32 dbq_watermark; /* Current watermark set in HW registers */
+ u32 dbq_nq_id; /* Current NQ ID for DBQ events */
+ u32 dbq_pacing_time; /* ms */
+ u32 dbr_def_do_pacing; /* do_pacing when no congestion */
+ u32 dbr_evt_curr_epoch;
+ bool dbq_int_disable;
+
+ bool mod_exit;
+ struct bnxt_re_dbr_sw_stats *dbr_sw_stats;
+ struct bnxt_re_dbr_res_list res_list[BNXT_RE_RES_TYPE_MAX];
+ struct bnxt_dbq_nq_list nq_list;
+ char dev_name[IB_DEVICE_NAME_MAX];
+ atomic_t dbq_intr_running;
+ u32 num_msix_requested;
+ unsigned char *dev_addr; /* For netdev->dev_addr */
+};
+
+#define BNXT_RE_RESOLVE_RETRY_COUNT_US 5000000 /* 5 sec */
+struct bnxt_re_resolve_dmac_work{
+ struct work_struct work;
+ struct list_head list;
+ struct bnxt_re_dev *rdev;
+ struct ib_ah_attr *ah_attr;
+ struct bnxt_re_ah_info *ah_info;
+ atomic_t status_wait;
+};
+
+static inline u8 bnxt_re_get_prio(u8 prio_map)
+{
+ u8 prio = 0xFF;
+
+ for (prio = 0; prio < 8; prio++)
+ if (prio_map & (1UL << prio))
+ break;
+ return prio;
+}
+
+/* This should be called with bnxt_re_dev_lock mutex held */
+static inline bool __bnxt_re_is_rdev_valid(struct bnxt_re_dev *rdev)
+{
+ struct bnxt_re_dev *tmp_rdev;
+
+ list_for_each_entry(tmp_rdev, &bnxt_re_dev_list, list) {
+ if (rdev == tmp_rdev)
+ return true;
+ }
+ return false;
+}
+
+static inline bool bnxt_re_is_rdev_valid(struct bnxt_re_dev *rdev)
+{
+ struct bnxt_re_dev *tmp_rdev;
+
+ mutex_lock(&bnxt_re_dev_lock);
+ list_for_each_entry(tmp_rdev, &bnxt_re_dev_list, list) {
+ if (rdev == tmp_rdev) {
+ mutex_unlock(&bnxt_re_dev_lock);
+ return true;
+ }
+ }
+ mutex_unlock(&bnxt_re_dev_lock);
+
+ pr_debug("bnxt_re: %s : Invalid rdev received rdev = %p\n",
+ __func__, rdev);
+ return false;
+}
+
+int bnxt_re_send_hwrm_cmd(struct bnxt_re_dev *rdev, void *cmd,
+ int cmdlen);
+void bnxt_re_stopqps_and_ib_uninit(struct bnxt_re_dev *rdev);
+int bnxt_re_set_hwrm_dscp2pri(struct bnxt_re_dev *rdev,
+ struct bnxt_re_dscp2pri *d2p, u16 count,
+ u16 target_id);
+int bnxt_re_query_hwrm_dscp2pri(struct bnxt_re_dev *rdev,
+ struct bnxt_re_dscp2pri *d2p, u16 *count,
+ u16 target_id);
+int bnxt_re_query_hwrm_qportcfg(struct bnxt_re_dev *rdev,
+ struct bnxt_re_tc_rec *cnprec, u16 tid);
+int bnxt_re_hwrm_cos2bw_qcfg(struct bnxt_re_dev *rdev, u16 target_id,
+ struct bnxt_re_cos2bw_cfg *cfg);
+int bnxt_re_hwrm_cos2bw_cfg(struct bnxt_re_dev *rdev, u16 target_id,
+ struct bnxt_re_cos2bw_cfg *cfg);
+int bnxt_re_hwrm_pri2cos_cfg(struct bnxt_re_dev *rdev,
+ u16 target_id, u16 port_id,
+ u8 *cos_id_map, u8 pri_map);
+int bnxt_re_prio_vlan_tx_update(struct bnxt_re_dev *rdev);
+int bnxt_re_get_slot_pf_count(struct bnxt_re_dev *rdev);
+struct bnxt_re_dev *bnxt_re_get_peer_pf(struct bnxt_re_dev *rdev);
+struct bnxt_re_dev *bnxt_re_from_netdev(struct ifnet *netdev);
+u8 bnxt_re_get_priority_mask(struct bnxt_re_dev *rdev, u8 selector);
+struct bnxt_qplib_nq * bnxt_re_get_nq(struct bnxt_re_dev *rdev);
+void bnxt_re_put_nq(struct bnxt_re_dev *rdev, struct bnxt_qplib_nq *nq);
+
+#define to_bnxt_re(ptr, type, member) \
+ container_of(ptr, type, member)
+
+#define to_bnxt_re_dev(ptr, member) \
+ container_of((ptr), struct bnxt_re_dev, member)
+
+/* Even number functions from port 0 and odd number from port 1 */
+#define BNXT_RE_IS_PORT0(rdev) (!(rdev->en_dev->pdev->devfn & 1))
+
+#define BNXT_RE_ROCE_V1_PACKET 0
+#define BNXT_RE_ROCEV2_IPV4_PACKET 2
+#define BNXT_RE_ROCEV2_IPV6_PACKET 3
+#define BNXT_RE_ACTIVE_MAP_PORT1 0x1 /*port-1 active */
+#define BNXT_RE_ACTIVE_MAP_PORT2 0x2 /*port-2 active */
+
+#define BNXT_RE_MEMBER_PORT_MAP (BNXT_RE_ACTIVE_MAP_PORT1 | \
+ BNXT_RE_ACTIVE_MAP_PORT2)
+
+#define rdev_to_dev(rdev) ((rdev) ? (&(rdev)->ibdev.dev) : NULL)
+
+void bnxt_re_set_dma_device(struct ib_device *ibdev, struct bnxt_re_dev *rdev);
+bool bnxt_re_is_rdev_valid(struct bnxt_re_dev *rdev);
+
+#define bnxt_re_rdev_ready(rdev) (bnxt_re_is_rdev_valid(rdev) && \
+ (test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)))
+#define BNXT_RE_SRIOV_CFG_TIMEOUT 6
+
+int bnxt_re_get_device_stats(struct bnxt_re_dev *rdev);
+void bnxt_re_remove_device(struct bnxt_re_dev *rdev, u8 removal_type,
+ struct auxiliary_device *aux_dev);
+void bnxt_re_destroy_lag(struct bnxt_re_dev **rdev);
+int bnxt_re_add_device(struct bnxt_re_dev **rdev,
+ struct ifnet *netdev,
+ u8 qp_mode, u8 op_type, u8 wqe_mode, u32 num_msix_requested,
+ struct auxiliary_device *aux_dev);
+void bnxt_re_create_base_interface(bool primary);
+int bnxt_re_schedule_work(struct bnxt_re_dev *rdev, unsigned long event,
+ struct ifnet *vlan_dev,
+ struct ifnet *netdev,
+ struct auxiliary_device *aux_dev);
+void bnxt_re_get_link_speed(struct bnxt_re_dev *rdev);
+int _bnxt_re_ib_init(struct bnxt_re_dev *rdev);
+int _bnxt_re_ib_init2(struct bnxt_re_dev *rdev);
+void bnxt_re_init_resolve_wq(struct bnxt_re_dev *rdev);
+void bnxt_re_uninit_resolve_wq(struct bnxt_re_dev *rdev);
+
+/* The rdev ref_count is to protect immature removal of the device */
+static inline void bnxt_re_hold(struct bnxt_re_dev *rdev)
+{
+ atomic_inc(&rdev->ref_count);
+ dev_dbg(rdev_to_dev(rdev),
+ "Hold ref_count = 0x%x", atomic_read(&rdev->ref_count));
+}
+
+static inline void bnxt_re_put(struct bnxt_re_dev *rdev)
+{
+ atomic_dec(&rdev->ref_count);
+ dev_dbg(rdev_to_dev(rdev),
+ "Put ref_count = 0x%x", atomic_read(&rdev->ref_count));
+}
+
+/*
+* Responder Error reason codes
+* FIXME: Remove these when the defs
+* are properly included in hsi header
+*/
+enum res_err_state_reason {
+ /* No error. */
+ CFCQ_RES_ERR_STATE_REASON_NO_ERROR = 0,
+ /*
+ * Incoming Send, RDMA write, or RDMA read exceeds the maximum
+ * transfer length. Detected on RX first and only packets for
+ * write. Detected on RX request for read. This is an RX
+ * Detected Error.
+ */
+ CFCQ_RES_ERR_STATE_REASON_RES_EXCEED_MAX,
+ /*
+ * RDMA write payload size does not match write length. Detected
+ * when total write payload is not equal to the RDMA write
+ * length that was given in the first or only packet of the
+ * request. This is an RX Detected Error.
+ */
+ CFCQ_RES_ERR_STATE_REASON_RES_PAYLOAD_LENGTH_MISMATCH,
+ /*
+ * Send payload exceeds RQ/SRQ WQE buffer capacity. The total
+ * send payload that arrived is more than the size of the WQE
+ * buffer that was fetched from the RQ/SRQ. This is an RX
+ * Detected Error.
+ */
+ CFCQ_RES_ERR_STATE_REASON_RES_EXCEEDS_WQE,
+ /*
+ * Responder detected opcode error. * First, only, middle, last
+ * for incoming requests are improperly ordered with respect to
+ * previous (PSN) packet. * First or middle packet is not full
+ * MTU size. This is an RX Detected Error.
+ */
+ CFCQ_RES_ERR_STATE_REASON_RES_OPCODE_ERROR,
+ /*
+ * PSN sequence error retry limit exceeded. The responder
+ * encountered a PSN sequence error for the same PSN too many
+ * times. This can occur via implicit or explicit NAK. This is
+ * an RX Detected Error.
+ */
+ CFCQ_RES_ERR_STATE_REASON_RES_PSN_SEQ_ERROR_RETRY_LIMIT,
+ /*
+ * Invalid R_Key. An incoming request contained an R_Key that
+ * did not reference a valid MR/MW. This error may be detected
+ * by the RX engine for RDMA write or by the TX engine for RDMA
+ * read (detected while servicing IRRQ). This is an RX Detected
+ * Error.
+ */
+ CFCQ_RES_ERR_STATE_REASON_RES_RX_INVALID_R_KEY,
+ /*
+ * Domain error. An incoming request specified an R_Key which
+ * referenced a MR/MW that was not in the same PD as the QP on
+ * which the request arrived. This is an RX Detected Error.
+ */
+ CFCQ_RES_ERR_STATE_REASON_RES_RX_DOMAIN_ERROR,
+ /*
+ * No permission. An incoming request contained an R_Key that
+ * referenced a MR/MW which did not have the access permission
+ * needed for the operation. This is an RX Detected Error.
+ */
+ CFCQ_RES_ERR_STATE_REASON_RES_RX_NO_PERMISSION,
+ /*
+ * Range error. An incoming request had a combination of R_Key,
+ * VA, and length that was out of bounds of the associated
+ * MR/MW. This is an RX Detected Error.
+ */
+ CFCQ_RES_ERR_STATE_REASON_RES_RX_RANGE_ERROR,
+ /*
+ * Invalid R_Key. An incoming request contained an R_Key that
+ * did not reference a valid MR/MW. This error may be detected
+ * by the RX engine for RDMA write or by the TX engine for RDMA
+ * read (detected while servicing IRRQ). This is a TX Detected
+ * Error.
+ */
+ CFCQ_RES_ERR_STATE_REASON_RES_TX_INVALID_R_KEY,
+ /*
+ * Domain error. An incoming request specified an R_Key which
+ * referenced a MR/MW that was not in the same PD as the QP on
+ * which the request arrived. This is a TX Detected Error.
+ */
+ CFCQ_RES_ERR_STATE_REASON_RES_TX_DOMAIN_ERROR,
+ /*
+ * No permission. An incoming request contained an R_Key that
+ * referenced a MR/MW which did not have the access permission
+ * needed for the operation. This is a TX Detected Error.
+ */
+ CFCQ_RES_ERR_STATE_REASON_RES_TX_NO_PERMISSION,
+ /*
+ * Range error. An incoming request had a combination of R_Key,
+ * VA, and length that was out of bounds of the associated
+ * MR/MW. This is a TX Detected Error.
+ */
+ CFCQ_RES_ERR_STATE_REASON_RES_TX_RANGE_ERROR,
+ /*
+ * IRRQ overflow. The peer sent us more RDMA read or atomic
+ * requests than the negotiated maximum. This is an RX Detected
+ * Error.
+ */
+ CFCQ_RES_ERR_STATE_REASON_RES_IRRQ_OFLOW,
+ /*
+ * Unsupported opcode. The peer sent us a request with an opcode
+ * for a request type that is not supported on this QP. This is
+ * an RX Detected Error.
+ */
+ CFCQ_RES_ERR_STATE_REASON_RES_UNSUPPORTED_OPCODE,
+ /*
+ * Unaligned atomic operation. The VA of an atomic request is on
+ * a memory boundary that prevents atomic execution. This is an
+ * RX Detected Error.
+ */
+ CFCQ_RES_ERR_STATE_REASON_RES_UNALIGN_ATOMIC,
+ /*
+ * Remote invalidate error. A send with invalidate request
+ * arrived in which the R_Key to invalidate did not describe a
+ * MR/MW which could be invalidated. RQ WQE completes with error
+ * status. This error is only reported if the send operation did
+ * not fail. If the send operation failed then the remote
+ * invalidate error is not reported. This is an RX Detected
+ * Error.
+ */
+ CFCQ_RES_ERR_STATE_REASON_RES_REM_INVALIDATE,
+ /*
+ * Local memory error. An RQ/SRQ SGE described an inaccessible
+ * memory. This is an RX Detected Error.
+ */
+ CFCQ_RES_ERR_STATE_REASON_RES_MEMORY_ERROR,
+ /*
+ * SRQ in error. The QP is moving to error state because it
+ * found SRQ it uses in error. This is an RX Detected Error.
+ */
+ CFCQ_RES_ERR_STATE_REASON_RES_SRQ_ERROR,
+ /*
+ * Completion error. No CQE space available on queue or CQ not
+ * in VALID state. This is a Completion Detected Error.
+ */
+ CFCQ_RES_ERR_STATE_REASON_RES_CMP_ERROR,
+ /*
+ * Invalid R_Key while resending responses to duplicate request.
+ * This is a TX Detected Error.
+ */
+ CFCQ_RES_ERR_STATE_REASON_RES_IVALID_DUP_RKEY,
+ /*
+ * Problem was found in the format of a WQE in the RQ/SRQ. This
+ * is an RX Detected Error.
+ */
+ CFCQ_RES_ERR_STATE_REASON_RES_WQE_FORMAT_ERROR,
+ /*
+ * A load error occurred on an attempt to load the CQ Context.
+ * This is a Completion Detected Error.
+ */
+ CFCQ_RES_ERR_STATE_REASON_RES_CQ_LOAD_ERROR = 0x18,
+ /*
+ * A load error occurred on an attempt to load the SRQ Context.
+ * This is an RX Detected Error.
+ */
+ CFCQ_RES_ERR_STATE_REASON_RES_SRQ_LOAD_ERROR,
+ /*
+ * A fatal error was detected on an attempt to read from or
+ * write to PCIe on the transmit side. This error is detected by
+ * the TX side, but has the priority of a Completion Detected
+ * Error.
+ */
+ CFCQ_RES_ERR_STATE_REASON_RES_TX_PCI_ERROR = 0x1b,
+ /*
+ * A fatal error was detected on an attempt to read from or
+ * write to PCIe on the receive side. This error is detected by
+ * the RX side (or CAGR), but has the priority of a Completion
+ * Detected Error.
+ */
+ CFCQ_RES_ERR_STATE_REASON_RES_RX_PCI_ERROR = 0x1c
+};
+
+int bnxt_re_host_pf_id_query(struct bnxt_re_dev *rdev,
+ struct bnxt_qplib_query_fn_info *fn_info,
+ u32 *pf_mask, u32 *first_pf);
+
+/* Default DCBx and CC values */
+#define BNXT_RE_DEFAULT_CNP_DSCP 48
+#define BNXT_RE_DEFAULT_CNP_PRI 7
+#define BNXT_RE_DEFAULT_ROCE_DSCP 26
+#define BNXT_RE_DEFAULT_ROCE_PRI 3
+
+#define BNXT_RE_DEFAULT_L2_BW 50
+#define BNXT_RE_DEFAULT_ROCE_BW 50
+
+#define ROCE_PRIO_VALID 0x0
+#define CNP_PRIO_VALID 0x1
+#define ROCE_DSCP_VALID 0x0
+#define CNP_DSCP_VALID 0x1
+
+int bnxt_re_get_pri_dscp_settings(struct bnxt_re_dev *rdev,
+ u16 target_id,
+ struct bnxt_re_tc_rec *tc_rec);
+
+int bnxt_re_setup_dscp(struct bnxt_re_dev *rdev);
+int bnxt_re_clear_dscp(struct bnxt_re_dev *rdev);
+int bnxt_re_setup_cnp_cos(struct bnxt_re_dev *rdev, bool reset);
+
+static inline enum ib_port_state bnxt_re_get_link_state(struct bnxt_re_dev *rdev)
+{
+ if (rdev->netdev->if_drv_flags & IFF_DRV_RUNNING &&
+ rdev->netdev->if_link_state == LINK_STATE_UP)
+ return IB_PORT_ACTIVE;
+ return IB_PORT_DOWN;
+}
+
+static inline int bnxt_re_link_state(struct bnxt_re_dev *rdev)
+{
+ return bnxt_re_get_link_state(rdev) == IB_PORT_ACTIVE ? 1:0;
+}
+
+static inline int is_cc_enabled(struct bnxt_re_dev *rdev)
+{
+ return rdev->cc_param.enable;
+}
+
+static inline void bnxt_re_init_hwrm_hdr(struct bnxt_re_dev *rdev,
+ struct input *hdr, u16 opcd,
+ u16 crid, u16 trid)
+{
+ hdr->req_type = cpu_to_le16(opcd);
+ hdr->cmpl_ring = cpu_to_le16(crid);
+ hdr->target_id = cpu_to_le16(trid);
+}
+
+static inline void bnxt_re_fill_fw_msg(struct bnxt_fw_msg *fw_msg,
+ void *msg, int msg_len, void *resp,
+ int resp_max_len, int timeout)
+{
+ fw_msg->msg = msg;
+ fw_msg->msg_len = msg_len;
+ fw_msg->resp = resp;
+ fw_msg->resp_max_len = resp_max_len;
+ fw_msg->timeout = timeout;
+}
+
+static inline bool is_qport_service_type_supported(struct bnxt_re_dev *rdev)
+{
+ return rdev->tc_rec[0].serv_type_enabled;
+}
+
+static inline bool is_bnxt_roce_queue(struct bnxt_re_dev *rdev, u8 ser_prof, u8 prof_type)
+{
+ if (is_qport_service_type_supported(rdev))
+ return (prof_type & HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID1_SERVICE_PROFILE_TYPE_ROCE);
+ else
+ return (ser_prof == HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID0_SERVICE_PROFILE_LOSSLESS_ROCE);
+}
+
+static inline bool is_bnxt_cnp_queue(struct bnxt_re_dev *rdev, u8 ser_prof, u8 prof_type)
+{
+ if (is_qport_service_type_supported(rdev))
+ return (prof_type & HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID1_SERVICE_PROFILE_TYPE_CNP);
+ else
+ return (ser_prof == HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID0_SERVICE_PROFILE_LOSSY_ROCE_CNP);
+}
+
+#define BNXT_RE_MAP_SH_PAGE 0x0
+#define BNXT_RE_MAP_WC 0x1
+#define BNXT_RE_DBR_PAGE 0x2
+#define BNXT_RE_MAP_DB_RECOVERY_PAGE 0x3
+
+#define BNXT_RE_DBR_RECOV_USERLAND_TIMEOUT (20) /* 20 ms */
+#define BNXT_RE_DBR_INT_TIME 5 /* ms */
+#define BNXT_RE_PACING_EN_INT_THRESHOLD 50 /* Entries in DB FIFO */
+#define BNXT_RE_PACING_ALGO_THRESHOLD 250 /* Entries in DB FIFO */
+/* Percentage of DB FIFO depth */
+#define BNXT_RE_PACING_DBQ_THRESHOLD BNXT_RE_PACING_DBQ_HIGH_WATERMARK
+
+#define BNXT_RE_PACING_ALARM_TH_MULTIPLE(ctx) (_is_chip_p7(ctx) ? 0 : 2)
+
+/*
+ * Maximum Percentage of configurable DB FIFO depth.
+ * The Doorbell FIFO depth is 0x2c00. But the DBR_REG_DB_THROTTLING register has only 12 bits
+ * to program the high watermark. This means user can configure maximum 36% only(4095/11264).
+ */
+#define BNXT_RE_PACING_DBQ_HIGH_WATERMARK 36
+
+/* Default do_pacing value when there is no congestion */
+#define BNXT_RE_DBR_DO_PACING_NO_CONGESTION 0x7F /* 1 in 512 probability */
+
+enum {
+ BNXT_RE_DBQ_EVENT_SCHED = 0,
+ BNXT_RE_DBR_PACING_EVENT = 1,
+ BNXT_RE_DBR_NQ_PACING_NOTIFICATION = 2,
+};
+
+struct bnxt_re_dbq_work {
+ struct work_struct work;
+ struct bnxt_re_dev *rdev;
+ struct hwrm_async_event_cmpl cmpl;
+ u32 event;
+};
+
+int bnxt_re_hwrm_qcaps(struct bnxt_re_dev *rdev);
+int bnxt_re_enable_dbr_pacing(struct bnxt_re_dev *rdev);
+int bnxt_re_disable_dbr_pacing(struct bnxt_re_dev *rdev);
+int bnxt_re_set_dbq_throttling_reg(struct bnxt_re_dev *rdev,
+ u16 nq_id, u32 throttle);
+void bnxt_re_pacing_alert(struct bnxt_re_dev *rdev);
+int bnxt_re_hwrm_pri2cos_qcfg(struct bnxt_re_dev *rdev, struct bnxt_re_tc_rec *tc_rec,
+ u16 target_id);
+void writel_fbsd(struct bnxt_softc *bp, u32, u8, u32);
+u32 readl_fbsd(struct bnxt_softc *bp, u32, u8);
+
+static inline unsigned int bnxt_re_get_total_mr_mw_count(struct bnxt_re_dev *rdev)
+{
+ return (atomic_read(&rdev->stats.rsors.mr_count) +
+ atomic_read(&rdev->stats.rsors.mw_count));
+}
+
+static inline void bnxt_re_set_def_pacing_threshold(struct bnxt_re_dev *rdev)
+{
+ rdev->qplib_res.pacing_data->pacing_th = rdev->pacing_algo_th;
+ rdev->qplib_res.pacing_data->alarm_th =
+ rdev->pacing_algo_th * BNXT_RE_PACING_ALARM_TH_MULTIPLE(rdev->chip_ctx);
+}
+
+static inline void bnxt_re_set_def_do_pacing(struct bnxt_re_dev *rdev)
+{
+ rdev->qplib_res.pacing_data->do_pacing = rdev->dbr_def_do_pacing;
+}
+
+static inline void bnxt_re_set_pacing_dev_state(struct bnxt_re_dev *rdev)
+{
+ rdev->qplib_res.pacing_data->dev_err_state =
+ test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags);
+}
+#endif
diff --git a/sys/dev/bnxt/bnxt_re/ib_verbs.c b/sys/dev/bnxt/bnxt_re/ib_verbs.c
new file mode 100644
index 000000000000..8d43fa96c048
--- /dev/null
+++ b/sys/dev/bnxt/bnxt_re/ib_verbs.c
@@ -0,0 +1,5498 @@
+/*
+ * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term
+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Description: IB Verbs interpreter
+ */
+
+#include <linux/if_ether.h>
+#include <linux/etherdevice.h>
+#include <rdma/uverbs_ioctl.h>
+
+#include "bnxt_re.h"
+#include "ib_verbs.h"
+
+static inline
+struct scatterlist *get_ib_umem_sgl(struct ib_umem *umem, u32 *nmap)
+{
+
+ *nmap = umem->nmap;
+ return umem->sg_head.sgl;
+}
+
+static inline void bnxt_re_peer_mem_release(struct ib_umem *umem)
+{
+ dev_dbg(NULL, "ib_umem_release getting invoked \n");
+ ib_umem_release(umem);
+}
+
+void bnxt_re_resolve_dmac_task(struct work_struct *work)
+{
+ int rc = -1;
+ struct bnxt_re_dev *rdev;
+ struct ib_ah_attr *ah_attr;
+ struct bnxt_re_resolve_dmac_work *dmac_work =
+ container_of(work, struct bnxt_re_resolve_dmac_work, work);
+
+ rdev = dmac_work->rdev;
+ ah_attr = dmac_work->ah_attr;
+ rc = ib_resolve_eth_dmac(&rdev->ibdev, ah_attr);
+ if (rc)
+ dev_err(rdev_to_dev(dmac_work->rdev),
+ "Failed to resolve dest mac rc = %d\n", rc);
+ atomic_set(&dmac_work->status_wait, rc << 8);
+}
+
+static int __from_ib_access_flags(int iflags)
+{
+ int qflags = 0;
+
+ if (iflags & IB_ACCESS_LOCAL_WRITE)
+ qflags |= BNXT_QPLIB_ACCESS_LOCAL_WRITE;
+ if (iflags & IB_ACCESS_REMOTE_READ)
+ qflags |= BNXT_QPLIB_ACCESS_REMOTE_READ;
+ if (iflags & IB_ACCESS_REMOTE_WRITE)
+ qflags |= BNXT_QPLIB_ACCESS_REMOTE_WRITE;
+ if (iflags & IB_ACCESS_REMOTE_ATOMIC)
+ qflags |= BNXT_QPLIB_ACCESS_REMOTE_ATOMIC;
+ if (iflags & IB_ACCESS_MW_BIND)
+ qflags |= BNXT_QPLIB_ACCESS_MW_BIND;
+ if (iflags & IB_ZERO_BASED)
+ qflags |= BNXT_QPLIB_ACCESS_ZERO_BASED;
+ if (iflags & IB_ACCESS_ON_DEMAND)
+ qflags |= BNXT_QPLIB_ACCESS_ON_DEMAND;
+ return qflags;
+};
+
+static enum ib_access_flags __to_ib_access_flags(int qflags)
+{
+ enum ib_access_flags iflags = 0;
+
+ if (qflags & BNXT_QPLIB_ACCESS_LOCAL_WRITE)
+ iflags |= IB_ACCESS_LOCAL_WRITE;
+ if (qflags & BNXT_QPLIB_ACCESS_REMOTE_WRITE)
+ iflags |= IB_ACCESS_REMOTE_WRITE;
+ if (qflags & BNXT_QPLIB_ACCESS_REMOTE_READ)
+ iflags |= IB_ACCESS_REMOTE_READ;
+ if (qflags & BNXT_QPLIB_ACCESS_REMOTE_ATOMIC)
+ iflags |= IB_ACCESS_REMOTE_ATOMIC;
+ if (qflags & BNXT_QPLIB_ACCESS_MW_BIND)
+ iflags |= IB_ACCESS_MW_BIND;
+ if (qflags & BNXT_QPLIB_ACCESS_ZERO_BASED)
+ iflags |= IB_ZERO_BASED;
+ if (qflags & BNXT_QPLIB_ACCESS_ON_DEMAND)
+ iflags |= IB_ACCESS_ON_DEMAND;
+ return iflags;
+};
+
+static int bnxt_re_copy_to_udata(struct bnxt_re_dev *rdev, void *data,
+ int len, struct ib_udata *udata)
+{
+ int rc;
+
+ rc = ib_copy_to_udata(udata, data, len);
+ if (rc)
+ dev_err(rdev_to_dev(rdev),
+ "ucontext copy failed from %ps rc %d\n",
+ __builtin_return_address(0), rc);
+
+ return rc;
+}
+
+struct ifnet *bnxt_re_get_netdev(struct ib_device *ibdev,
+ u8 port_num)
+{
+ struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
+ struct ifnet *netdev = NULL;
+
+ rcu_read_lock();
+
+ if (!rdev || !rdev->netdev)
+ goto end;
+
+ netdev = rdev->netdev;
+
+ /* In case of active-backup bond mode, return active slave */
+ if (netdev)
+ dev_hold(netdev);
+
+end:
+ rcu_read_unlock();
+ return netdev;
+}
+
+int bnxt_re_query_device(struct ib_device *ibdev,
+ struct ib_device_attr *ib_attr,
+ struct ib_udata *udata)
+{
+ struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
+ struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr;
+
+ memset(ib_attr, 0, sizeof(*ib_attr));
+
+ memcpy(&ib_attr->fw_ver, dev_attr->fw_ver, 4);
+ bnxt_qplib_get_guid(rdev->dev_addr, (u8 *)&ib_attr->sys_image_guid);
+ ib_attr->max_mr_size = BNXT_RE_MAX_MR_SIZE;
+ ib_attr->page_size_cap = dev_attr->page_size_cap;
+ ib_attr->vendor_id = rdev->en_dev->pdev->vendor;
+ ib_attr->vendor_part_id = rdev->en_dev->pdev->device;
+ ib_attr->hw_ver = rdev->en_dev->pdev->subsystem_device;
+ ib_attr->max_qp = dev_attr->max_qp;
+ ib_attr->max_qp_wr = dev_attr->max_qp_wqes;
+ /*
+ * Read and set from the module param 'min_tx_depth'
+ * only once after the driver load
+ */
+ if (rdev->min_tx_depth == 1 &&
+ min_tx_depth < dev_attr->max_qp_wqes)
+ rdev->min_tx_depth = min_tx_depth;
+ ib_attr->device_cap_flags =
+ IB_DEVICE_CURR_QP_STATE_MOD
+ | IB_DEVICE_RC_RNR_NAK_GEN
+ | IB_DEVICE_SHUTDOWN_PORT
+ | IB_DEVICE_SYS_IMAGE_GUID
+ | IB_DEVICE_LOCAL_DMA_LKEY
+ | IB_DEVICE_RESIZE_MAX_WR
+ | IB_DEVICE_PORT_ACTIVE_EVENT
+ | IB_DEVICE_N_NOTIFY_CQ
+ | IB_DEVICE_MEM_WINDOW
+ | IB_DEVICE_MEM_WINDOW_TYPE_2B
+ | IB_DEVICE_MEM_MGT_EXTENSIONS;
+ ib_attr->max_send_sge = dev_attr->max_qp_sges;
+ ib_attr->max_recv_sge = dev_attr->max_qp_sges;
+ ib_attr->max_sge_rd = dev_attr->max_qp_sges;
+ ib_attr->max_cq = dev_attr->max_cq;
+ ib_attr->max_cqe = dev_attr->max_cq_wqes;
+ ib_attr->max_mr = dev_attr->max_mr;
+ ib_attr->max_pd = dev_attr->max_pd;
+ ib_attr->max_qp_rd_atom = dev_attr->max_qp_rd_atom;
+ ib_attr->max_qp_init_rd_atom = dev_attr->max_qp_init_rd_atom;
+ if (dev_attr->is_atomic) {
+ ib_attr->atomic_cap = IB_ATOMIC_GLOB;
+ ib_attr->masked_atomic_cap = IB_ATOMIC_GLOB;
+ }
+ ib_attr->max_ee_rd_atom = 0;
+ ib_attr->max_res_rd_atom = 0;
+ ib_attr->max_ee_init_rd_atom = 0;
+ ib_attr->max_ee = 0;
+ ib_attr->max_rdd = 0;
+ ib_attr->max_mw = dev_attr->max_mw;
+ ib_attr->max_raw_ipv6_qp = 0;
+ ib_attr->max_raw_ethy_qp = dev_attr->max_raw_ethy_qp;
+ ib_attr->max_mcast_grp = 0;
+ ib_attr->max_mcast_qp_attach = 0;
+ ib_attr->max_total_mcast_qp_attach = 0;
+ ib_attr->max_ah = dev_attr->max_ah;
+ ib_attr->max_srq = dev_attr->max_srq;
+ ib_attr->max_srq_wr = dev_attr->max_srq_wqes;
+ ib_attr->max_srq_sge = dev_attr->max_srq_sges;
+
+ ib_attr->max_fast_reg_page_list_len = MAX_PBL_LVL_1_PGS;
+ ib_attr->max_pkeys = 1;
+ ib_attr->local_ca_ack_delay = BNXT_RE_DEFAULT_ACK_DELAY;
+ ib_attr->sig_prot_cap = 0;
+ ib_attr->sig_guard_cap = 0;
+ ib_attr->odp_caps.general_caps = 0;
+
+ return 0;
+}
+
+int bnxt_re_modify_device(struct ib_device *ibdev,
+ int device_modify_mask,
+ struct ib_device_modify *device_modify)
+{
+ dev_dbg(rdev_to_dev(rdev), "Modify device with mask 0x%x\n",
+ device_modify_mask);
+
+ switch (device_modify_mask) {
+ case IB_DEVICE_MODIFY_SYS_IMAGE_GUID:
+ /* Modify the GUID requires the modification of the GID table */
+ /* GUID should be made as READ-ONLY */
+ break;
+ case IB_DEVICE_MODIFY_NODE_DESC:
+ /* Node Desc should be made as READ-ONLY */
+ break;
+ default:
+ break;
+ }
+ return 0;
+}
+
+static void __to_ib_speed_width(u32 espeed, u8 *speed, u8 *width)
+{
+ switch (espeed) {
+ case SPEED_1000:
+ *speed = IB_SPEED_SDR;
+ *width = IB_WIDTH_1X;
+ break;
+ case SPEED_10000:
+ *speed = IB_SPEED_QDR;
+ *width = IB_WIDTH_1X;
+ break;
+ case SPEED_20000:
+ *speed = IB_SPEED_DDR;
+ *width = IB_WIDTH_4X;
+ break;
+ case SPEED_25000:
+ *speed = IB_SPEED_EDR;
+ *width = IB_WIDTH_1X;
+ break;
+ case SPEED_40000:
+ *speed = IB_SPEED_QDR;
+ *width = IB_WIDTH_4X;
+ break;
+ case SPEED_50000:
+ *speed = IB_SPEED_EDR;
+ *width = IB_WIDTH_2X;
+ break;
+ case SPEED_100000:
+ *speed = IB_SPEED_EDR;
+ *width = IB_WIDTH_4X;
+ break;
+ case SPEED_200000:
+ *speed = IB_SPEED_HDR;
+ *width = IB_WIDTH_4X;
+ break;
+ default:
+ *speed = IB_SPEED_SDR;
+ *width = IB_WIDTH_1X;
+ break;
+ }
+}
+
+/* Port */
+int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num,
+ struct ib_port_attr *port_attr)
+{
+ struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
+ struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr;
+ u8 active_speed = 0, active_width = 0;
+
+ dev_dbg(rdev_to_dev(rdev), "QUERY PORT with port_num 0x%x\n", port_num);
+ memset(port_attr, 0, sizeof(*port_attr));
+
+ port_attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
+ port_attr->state = bnxt_re_get_link_state(rdev);
+ if (port_attr->state == IB_PORT_ACTIVE)
+ port_attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
+ port_attr->max_mtu = IB_MTU_4096;
+ port_attr->active_mtu = iboe_get_mtu(rdev->netdev->if_mtu);
+ port_attr->gid_tbl_len = dev_attr->max_sgid;
+ port_attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP |
+ IB_PORT_DEVICE_MGMT_SUP |
+ IB_PORT_VENDOR_CLASS_SUP |
+ IB_PORT_IP_BASED_GIDS;
+
+ port_attr->max_msg_sz = (u32)BNXT_RE_MAX_MR_SIZE_LOW;
+ port_attr->bad_pkey_cntr = 0;
+ port_attr->qkey_viol_cntr = 0;
+ port_attr->pkey_tbl_len = dev_attr->max_pkey;
+ port_attr->lid = 0;
+ port_attr->sm_lid = 0;
+ port_attr->lmc = 0;
+ port_attr->max_vl_num = 4;
+ port_attr->sm_sl = 0;
+ port_attr->subnet_timeout = 0;
+ port_attr->init_type_reply = 0;
+ rdev->espeed = rdev->en_dev->espeed;
+
+ if (test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags))
+ __to_ib_speed_width(rdev->espeed, &active_speed,
+ &active_width);
+
+ port_attr->active_speed = active_speed;
+ port_attr->active_width = active_width;
+
+ return 0;
+}
+
+int bnxt_re_modify_port(struct ib_device *ibdev, u8 port_num,
+ int port_modify_mask,
+ struct ib_port_modify *port_modify)
+{
+ dev_dbg(rdev_to_dev(rdev), "Modify port with mask 0x%x\n",
+ port_modify_mask);
+
+ switch (port_modify_mask) {
+ case IB_PORT_SHUTDOWN:
+ break;
+ case IB_PORT_INIT_TYPE:
+ break;
+ case IB_PORT_RESET_QKEY_CNTR:
+ break;
+ default:
+ break;
+ }
+ return 0;
+}
+
+int bnxt_re_get_port_immutable(struct ib_device *ibdev, u8 port_num,
+ struct ib_port_immutable *immutable)
+{
+ struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
+ struct ib_port_attr port_attr;
+
+ if (bnxt_re_query_port(ibdev, port_num, &port_attr))
+ return -EINVAL;
+
+ immutable->pkey_tbl_len = port_attr.pkey_tbl_len;
+ immutable->gid_tbl_len = port_attr.gid_tbl_len;
+ if (rdev->roce_mode == BNXT_RE_FLAG_ROCEV1_CAP)
+ immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE;
+ else if (rdev->roce_mode == BNXT_RE_FLAG_ROCEV2_CAP)
+ immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
+ else
+ immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE |
+ RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
+ immutable->max_mad_size = IB_MGMT_MAD_SIZE;
+ return 0;
+}
+
+void bnxt_re_compat_qfwstr(void)
+{
+ struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
+
+ sprintf(str, "%d.%d.%d.%d", rdev->dev_attr->fw_ver[0],
+ rdev->dev_attr->fw_ver[1], rdev->dev_attr->fw_ver[2],
+ rdev->dev_attr->fw_ver[3]);
+}
+
+int bnxt_re_query_pkey(struct ib_device *ibdev, u8 port_num,
+ u16 index, u16 *pkey)
+{
+ if (index > 0)
+ return -EINVAL;
+
+ *pkey = IB_DEFAULT_PKEY_FULL;
+
+ return 0;
+}
+
+int bnxt_re_query_gid(struct ib_device *ibdev, u8 port_num,
+ int index, union ib_gid *gid)
+{
+ struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
+ int rc = 0;
+
+ /* Ignore port_num */
+ memset(gid, 0, sizeof(*gid));
+ rc = bnxt_qplib_get_sgid(&rdev->qplib_res,
+ &rdev->qplib_res.sgid_tbl, index,
+ (struct bnxt_qplib_gid *)gid);
+ return rc;
+}
+
+int bnxt_re_del_gid(struct ib_device *ibdev, u8 port_num,
+ unsigned int index, void **context)
+{
+ int rc = 0;
+ struct bnxt_re_gid_ctx *ctx, **ctx_tbl;
+ struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
+ struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl;
+ struct bnxt_qplib_gid *gid_to_del;
+ u16 vlan_id = 0xFFFF;
+
+ /* Delete the entry from the hardware */
+ ctx = *context;
+ if (!ctx) {
+ dev_err(rdev_to_dev(rdev), "GID entry has no ctx?!\n");
+ return -EINVAL;
+ }
+ if (sgid_tbl && sgid_tbl->active) {
+ if (ctx->idx >= sgid_tbl->max) {
+ dev_dbg(rdev_to_dev(rdev), "GID index out of range?!\n");
+ return -EINVAL;
+ }
+ gid_to_del = &sgid_tbl->tbl[ctx->idx].gid;
+ vlan_id = sgid_tbl->tbl[ctx->idx].vlan_id;
+ ctx->refcnt--;
+ /* DEL_GID is called via WQ context(netdevice_event_work_handler)
+ * or via the ib_unregister_device path. In the former case QP1
+ * may not be destroyed yet, in which case just return as FW
+ * needs that entry to be present and will fail it's deletion.
+ * We could get invoked again after QP1 is destroyed OR get an
+ * ADD_GID call with a different GID value for the same index
+ * where we issue MODIFY_GID cmd to update the GID entry -- TBD
+ */
+ if (ctx->idx == 0 &&
+ rdma_link_local_addr((struct in6_addr *)gid_to_del) &&
+ (rdev->gsi_ctx.gsi_sqp ||
+ rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_UD)) {
+ dev_dbg(rdev_to_dev(rdev),
+ "Trying to delete GID0 while QP1 is alive\n");
+ if (!ctx->refcnt) {
+ rdev->gid_map[index] = -1;
+ ctx_tbl = sgid_tbl->ctx;
+ ctx_tbl[ctx->idx] = NULL;
+ kfree(ctx);
+ }
+ return 0;
+ }
+ rdev->gid_map[index] = -1;
+ if (!ctx->refcnt) {
+ rc = bnxt_qplib_del_sgid(sgid_tbl, gid_to_del,
+ vlan_id, true);
+ if (!rc) {
+ dev_dbg(rdev_to_dev(rdev), "GID remove success\n");
+ ctx_tbl = sgid_tbl->ctx;
+ ctx_tbl[ctx->idx] = NULL;
+ kfree(ctx);
+ } else {
+ dev_err(rdev_to_dev(rdev),
+ "Remove GID failed rc = 0x%x\n", rc);
+ }
+ }
+ } else {
+ dev_dbg(rdev_to_dev(rdev), "GID sgid_tbl does not exist!\n");
+ return -EINVAL;
+ }
+ return rc;
+}
+
+int bnxt_re_add_gid(struct ib_device *ibdev, u8 port_num,
+ unsigned int index, const union ib_gid *gid,
+ const struct ib_gid_attr *attr, void **context)
+{
+ int rc;
+ u32 tbl_idx = 0;
+ u16 vlan_id = 0xFFFF;
+ struct bnxt_re_gid_ctx *ctx, **ctx_tbl;
+ struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
+ struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl;
+ if ((attr->ndev) && is_vlan_dev(attr->ndev))
+ vlan_id = vlan_dev_vlan_id(attr->ndev);
+
+ rc = bnxt_qplib_add_sgid(sgid_tbl, gid,
+ rdev->dev_addr,
+ vlan_id, true, &tbl_idx);
+ if (rc == -EALREADY) {
+ dev_dbg(rdev_to_dev(rdev), "GID %pI6 is already present\n", gid);
+ ctx_tbl = sgid_tbl->ctx;
+ if (!ctx_tbl[tbl_idx]) {
+ ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
+ if (!ctx)
+ return -ENOMEM;
+ ctx->idx = tbl_idx;
+ ctx->refcnt = 1;
+ ctx_tbl[tbl_idx] = ctx;
+ } else {
+ ctx_tbl[tbl_idx]->refcnt++;
+ }
+ *context = ctx_tbl[tbl_idx];
+ /* tbl_idx is the HW table index and index is the stack index */
+ rdev->gid_map[index] = tbl_idx;
+ return 0;
+ } else if (rc < 0) {
+ dev_err(rdev_to_dev(rdev), "Add GID failed rc = 0x%x\n", rc);
+ return rc;
+ } else {
+ ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
+ if (!ctx) {
+ dev_err(rdev_to_dev(rdev), "Add GID ctx failed\n");
+ return -ENOMEM;
+ }
+ ctx_tbl = sgid_tbl->ctx;
+ ctx->idx = tbl_idx;
+ ctx->refcnt = 1;
+ ctx_tbl[tbl_idx] = ctx;
+ /* tbl_idx is the HW table index and index is the stack index */
+ rdev->gid_map[index] = tbl_idx;
+ *context = ctx;
+ }
+ return rc;
+}
+
+enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev,
+ u8 port_num)
+{
+ return IB_LINK_LAYER_ETHERNET;
+}
+
+static void bnxt_re_legacy_create_fence_wqe(struct bnxt_re_pd *pd)
+{
+ struct bnxt_re_legacy_fence_data *fence = &pd->fence;
+ struct ib_mr *ib_mr = &fence->mr->ib_mr;
+ struct bnxt_qplib_swqe *wqe = &fence->bind_wqe;
+ struct bnxt_re_dev *rdev = pd->rdev;
+
+ if (!_is_chip_gen_p5_p7(rdev->chip_ctx))
+ return;
+
+ memset(wqe, 0, sizeof(*wqe));
+ wqe->type = BNXT_QPLIB_SWQE_TYPE_BIND_MW;
+ wqe->wr_id = BNXT_QPLIB_FENCE_WRID;
+ wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
+ wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
+ wqe->bind.zero_based = false;
+ wqe->bind.parent_l_key = ib_mr->lkey;
+ wqe->bind.va = (u64)fence->va;
+ wqe->bind.length = fence->size;
+ wqe->bind.access_cntl = __from_ib_access_flags(IB_ACCESS_REMOTE_READ);
+ wqe->bind.mw_type = SQ_BIND_MW_TYPE_TYPE1;
+
+ /* Save the initial rkey in fence structure for now;
+ * wqe->bind.r_key will be set at (re)bind time.
+ */
+ fence->bind_rkey = ib_inc_rkey(fence->mw->rkey);
+}
+
+static int bnxt_re_legacy_bind_fence_mw(struct bnxt_qplib_qp *qplib_qp)
+{
+ struct bnxt_re_qp *qp = container_of(qplib_qp, struct bnxt_re_qp,
+ qplib_qp);
+ struct ib_pd *ib_pd = qp->ib_qp.pd;
+ struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd);
+ struct bnxt_re_legacy_fence_data *fence = &pd->fence;
+ struct bnxt_qplib_swqe *fence_wqe = &fence->bind_wqe;
+ struct bnxt_qplib_swqe wqe;
+ int rc;
+
+ /* TODO: Need SQ locking here when Fence WQE
+ * posting moves up into bnxt_re from bnxt_qplib.
+ */
+ memcpy(&wqe, fence_wqe, sizeof(wqe));
+ wqe.bind.r_key = fence->bind_rkey;
+ fence->bind_rkey = ib_inc_rkey(fence->bind_rkey);
+
+ dev_dbg(rdev_to_dev(qp->rdev),
+ "Posting bind fence-WQE: rkey: %#x QP: %d PD: %p\n",
+ wqe.bind.r_key, qp->qplib_qp.id, pd);
+ rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
+ if (rc) {
+ dev_err(rdev_to_dev(qp->rdev), "Failed to bind fence-WQE\n");
+ return rc;
+ }
+ bnxt_qplib_post_send_db(&qp->qplib_qp);
+
+ return rc;
+}
+
+static int bnxt_re_legacy_create_fence_mr(struct bnxt_re_pd *pd)
+{
+ int mr_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_MW_BIND;
+ struct bnxt_re_legacy_fence_data *fence = &pd->fence;
+ struct bnxt_re_dev *rdev = pd->rdev;
+ struct bnxt_qplib_mrinfo mrinfo;
+ struct bnxt_re_mr *mr = NULL;
+ struct ib_mw *ib_mw = NULL;
+ dma_addr_t dma_addr = 0;
+ u32 max_mr_count;
+ u64 pbl_tbl;
+ int rc;
+
+ if (!_is_chip_gen_p5_p7(rdev->chip_ctx))
+ return 0;
+
+ memset(&mrinfo, 0, sizeof(mrinfo));
+ /* Allocate a small chunk of memory and dma-map it */
+ fence->va = kzalloc(BNXT_RE_LEGACY_FENCE_BYTES, GFP_KERNEL);
+ if (!fence->va)
+ return -ENOMEM;
+ dma_addr = ib_dma_map_single(&rdev->ibdev, fence->va,
+ BNXT_RE_LEGACY_FENCE_BYTES,
+ DMA_BIDIRECTIONAL);
+ rc = ib_dma_mapping_error(&rdev->ibdev, dma_addr);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "Failed to dma-map fence-MR-mem\n");
+ rc = -EIO;
+ fence->dma_addr = 0;
+ goto free_va;
+ }
+ fence->dma_addr = dma_addr;
+
+ /* Allocate a MR */
+ mr = kzalloc(sizeof(*mr), GFP_KERNEL);
+ if (!mr)
+ goto free_dma_addr;
+ fence->mr = mr;
+ mr->rdev = rdev;
+ mr->qplib_mr.pd = &pd->qplib_pd;
+ mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
+ mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags);
+ if (!_is_alloc_mr_unified(rdev->qplib_res.dattr)) {
+ rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "Failed to alloc fence-HW-MR\n");
+ goto free_mr;
+ }
+ /* Register MR */
+ mr->ib_mr.lkey = mr->qplib_mr.lkey;
+ }
+ mr->qplib_mr.va = (u64)fence->va;
+ mr->qplib_mr.total_size = BNXT_RE_LEGACY_FENCE_BYTES;
+ pbl_tbl = dma_addr;
+
+ mrinfo.mrw = &mr->qplib_mr;
+ mrinfo.ptes = &pbl_tbl;
+ mrinfo.sg.npages = BNXT_RE_LEGACY_FENCE_PBL_SIZE;
+
+ mrinfo.sg.nmap = 0;
+ mrinfo.sg.sghead = 0;
+ mrinfo.sg.pgshft = PAGE_SHIFT;
+ mrinfo.sg.pgsize = PAGE_SIZE;
+ rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mrinfo, false);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "Failed to register fence-MR\n");
+ goto free_mr;
+ }
+ mr->ib_mr.lkey = mr->qplib_mr.lkey;
+ mr->ib_mr.rkey = mr->qplib_mr.rkey;
+ atomic_inc(&rdev->stats.rsors.mr_count);
+ max_mr_count = atomic_read(&rdev->stats.rsors.mr_count);
+ if (max_mr_count > (atomic_read(&rdev->stats.rsors.max_mr_count)))
+ atomic_set(&rdev->stats.rsors.max_mr_count, max_mr_count);
+
+ ib_mw = bnxt_re_alloc_mw(&pd->ibpd, IB_MW_TYPE_1, NULL);
+ /* Create a fence MW only for kernel consumers */
+ if (!ib_mw) {
+ dev_err(rdev_to_dev(rdev),
+ "Failed to create fence-MW for PD: %p\n", pd);
+ rc = -EINVAL;
+ goto free_mr;
+ }
+ fence->mw = ib_mw;
+
+ bnxt_re_legacy_create_fence_wqe(pd);
+ return 0;
+
+free_mr:
+ if (mr->ib_mr.lkey) {
+ bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
+ atomic_dec(&rdev->stats.rsors.mr_count);
+ }
+ kfree(mr);
+ fence->mr = NULL;
+
+free_dma_addr:
+ ib_dma_unmap_single(&rdev->ibdev, fence->dma_addr,
+ BNXT_RE_LEGACY_FENCE_BYTES, DMA_BIDIRECTIONAL);
+ fence->dma_addr = 0;
+
+free_va:
+ kfree(fence->va);
+ fence->va = NULL;
+ return rc;
+}
+
+static void bnxt_re_legacy_destroy_fence_mr(struct bnxt_re_pd *pd)
+{
+ struct bnxt_re_legacy_fence_data *fence = &pd->fence;
+ struct bnxt_re_dev *rdev = pd->rdev;
+ struct bnxt_re_mr *mr = fence->mr;
+
+ if (!_is_chip_gen_p5_p7(rdev->chip_ctx))
+ return;
+
+ if (fence->mw) {
+ bnxt_re_dealloc_mw(fence->mw);
+ fence->mw = NULL;
+ }
+ if (mr) {
+ if (mr->ib_mr.rkey)
+ bnxt_qplib_dereg_mrw(&rdev->qplib_res, &mr->qplib_mr,
+ false);
+ if (mr->ib_mr.lkey)
+ bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
+ kfree(mr);
+ fence->mr = NULL;
+ atomic_dec(&rdev->stats.rsors.mr_count);
+ }
+ if (fence->dma_addr) {
+ ib_dma_unmap_single(&rdev->ibdev, fence->dma_addr,
+ BNXT_RE_LEGACY_FENCE_BYTES,
+ DMA_BIDIRECTIONAL);
+ fence->dma_addr = 0;
+ }
+ kfree(fence->va);
+ fence->va = NULL;
+}
+
+
+static int bnxt_re_get_user_dpi(struct bnxt_re_dev *rdev,
+ struct bnxt_re_ucontext *cntx)
+{
+ struct bnxt_qplib_chip_ctx *cctx = rdev->chip_ctx;
+ int ret = 0;
+ u8 type;
+ /* Allocate DPI in alloc_pd or in create_cq to avoid failing of
+ * ibv_devinfo and family of application when DPIs are depleted.
+ */
+ type = BNXT_QPLIB_DPI_TYPE_UC;
+ ret = bnxt_qplib_alloc_dpi(&rdev->qplib_res, &cntx->dpi, cntx, type);
+ if (ret) {
+ dev_err(rdev_to_dev(rdev), "Alloc doorbell page failed!\n");
+ goto out;
+ }
+
+ if (cctx->modes.db_push) {
+ type = BNXT_QPLIB_DPI_TYPE_WC;
+ ret = bnxt_qplib_alloc_dpi(&rdev->qplib_res, &cntx->wcdpi,
+ cntx, type);
+ if (ret)
+ dev_err(rdev_to_dev(rdev), "push dp alloc failed\n");
+ }
+out:
+ return ret;
+}
+
+/* Protection Domains */
+void bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata)
+{
+ struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd);
+ struct bnxt_re_dev *rdev = pd->rdev;
+ int rc;
+
+ bnxt_re_legacy_destroy_fence_mr(pd);
+
+ rc = bnxt_qplib_dealloc_pd(&rdev->qplib_res,
+ &rdev->qplib_res.pd_tbl,
+ &pd->qplib_pd);
+ if (rc)
+ dev_err_ratelimited(rdev_to_dev(rdev),
+ "%s failed rc = %d\n", __func__, rc);
+ atomic_dec(&rdev->stats.rsors.pd_count);
+
+ return;
+}
+
+int bnxt_re_alloc_pd(struct ib_pd *pd_in,
+ struct ib_udata *udata)
+{
+ struct ib_pd *ibpd = pd_in;
+ struct ib_device *ibdev = ibpd->device;
+ struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
+ struct bnxt_re_ucontext *ucntx =
+ rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext,
+ ibucontext);
+ u32 max_pd_count;
+ int rc;
+ struct bnxt_re_pd *pd = container_of(ibpd, struct bnxt_re_pd, ibpd);
+
+ pd->rdev = rdev;
+ if (bnxt_qplib_alloc_pd(&rdev->qplib_res, &pd->qplib_pd)) {
+ dev_err(rdev_to_dev(rdev),
+ "Allocate HW Protection Domain failed!\n");
+ rc = -ENOMEM;
+ goto fail;
+ }
+
+ if (udata) {
+ struct bnxt_re_pd_resp resp = {};
+
+ if (!ucntx->dpi.dbr) {
+ rc = bnxt_re_get_user_dpi(rdev, ucntx);
+ if (rc)
+ goto dbfail;
+ }
+
+ resp.pdid = pd->qplib_pd.id;
+ /* Still allow mapping this DBR to the new user PD. */
+ resp.dpi = ucntx->dpi.dpi;
+ resp.dbr = (u64)ucntx->dpi.umdbr;
+ /* Copy only on a valid wcpdi */
+ if (ucntx->wcdpi.dpi) {
+ resp.wcdpi = ucntx->wcdpi.dpi;
+ resp.comp_mask = BNXT_RE_COMP_MASK_PD_HAS_WC_DPI;
+ }
+ if (rdev->dbr_pacing) {
+ WARN_ON(!rdev->dbr_bar_addr);
+ resp.dbr_bar_addr = (u64)rdev->dbr_bar_addr;
+ resp.comp_mask |= BNXT_RE_COMP_MASK_PD_HAS_DBR_BAR_ADDR;
+ }
+
+ rc = bnxt_re_copy_to_udata(rdev, &resp,
+ min(udata->outlen, sizeof(resp)),
+ udata);
+ if (rc)
+ goto dbfail;
+ }
+
+ if (!udata)
+ if (bnxt_re_legacy_create_fence_mr(pd))
+ dev_warn(rdev_to_dev(rdev),
+ "Failed to create Fence-MR\n");
+
+ atomic_inc(&rdev->stats.rsors.pd_count);
+ max_pd_count = atomic_read(&rdev->stats.rsors.pd_count);
+ if (max_pd_count > atomic_read(&rdev->stats.rsors.max_pd_count))
+ atomic_set(&rdev->stats.rsors.max_pd_count, max_pd_count);
+
+ return 0;
+dbfail:
+ (void)bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl,
+ &pd->qplib_pd);
+fail:
+ return rc;
+}
+
+/* Address Handles */
+void bnxt_re_destroy_ah(struct ib_ah *ib_ah, u32 flags)
+{
+ struct bnxt_re_ah *ah = to_bnxt_re(ib_ah, struct bnxt_re_ah, ibah);
+ struct bnxt_re_dev *rdev = ah->rdev;
+ int rc = 0;
+ bool block = true;
+
+ block = !(flags & RDMA_DESTROY_AH_SLEEPABLE);
+
+ rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, &ah->qplib_ah, block);
+ if (rc)
+ dev_err_ratelimited(rdev_to_dev(rdev),
+ "%s id = %d blocking %d failed rc = %d\n",
+ __func__, ah->qplib_ah.id, block, rc);
+ atomic_dec(&rdev->stats.rsors.ah_count);
+
+ return;
+}
+
+static u8 _to_bnxt_re_nw_type(enum rdma_network_type ntype)
+{
+ u8 nw_type;
+ switch (ntype) {
+ case RDMA_NETWORK_IPV4:
+ nw_type = CMDQ_CREATE_AH_TYPE_V2IPV4;
+ break;
+ case RDMA_NETWORK_IPV6:
+ nw_type = CMDQ_CREATE_AH_TYPE_V2IPV6;
+ break;
+ default:
+ nw_type = CMDQ_CREATE_AH_TYPE_V1;
+ break;
+ }
+ return nw_type;
+}
+
+static inline int
+bnxt_re_get_cached_gid(struct ib_device *dev, u8 port_num, int index,
+ union ib_gid *sgid, struct ib_gid_attr **sgid_attr,
+ struct ib_global_route *grh, struct ib_ah *ah)
+{
+ int ret = 0;
+
+ ret = ib_get_cached_gid(dev, port_num, index, sgid, *sgid_attr);
+ return ret;
+}
+
+static inline enum rdma_network_type
+bnxt_re_gid_to_network_type(struct ib_gid_attr *sgid_attr,
+ union ib_gid *sgid)
+{
+ return ib_gid_to_network_type(sgid_attr->gid_type, sgid);
+}
+
+static int bnxt_re_get_ah_info(struct bnxt_re_dev *rdev,
+ struct ib_ah_attr *ah_attr,
+ struct bnxt_re_ah_info *ah_info)
+{
+ struct ib_gid_attr *gattr;
+ enum rdma_network_type ib_ntype;
+ u8 ntype;
+ union ib_gid *gid;
+ int rc = 0;
+
+ gid = &ah_info->sgid;
+ gattr = &ah_info->sgid_attr;
+
+ rc = bnxt_re_get_cached_gid(&rdev->ibdev, 1, ah_attr->grh.sgid_index,
+ gid, &gattr, &ah_attr->grh, NULL);
+ if (rc)
+ return rc;
+
+ /* Get vlan tag */
+ if (gattr->ndev) {
+ if (is_vlan_dev(gattr->ndev))
+ ah_info->vlan_tag = vlan_dev_vlan_id(gattr->ndev);
+ if_rele(gattr->ndev);
+ }
+
+ /* Get network header type for this GID */
+
+ ib_ntype = bnxt_re_gid_to_network_type(gattr, gid);
+ ntype = _to_bnxt_re_nw_type(ib_ntype);
+ ah_info->nw_type = ntype;
+
+ return rc;
+}
+
+static u8 _get_sgid_index(struct bnxt_re_dev *rdev, u8 gindx)
+{
+ gindx = rdev->gid_map[gindx];
+ return gindx;
+}
+
+static int bnxt_re_init_dmac(struct bnxt_re_dev *rdev, struct ib_ah_attr *ah_attr,
+ struct bnxt_re_ah_info *ah_info, bool is_user,
+ struct bnxt_re_ah *ah)
+{
+ int rc = 0;
+ u8 *dmac;
+
+ if (is_user && !rdma_is_multicast_addr((struct in6_addr *)
+ ah_attr->grh.dgid.raw) &&
+ !rdma_link_local_addr((struct in6_addr *)ah_attr->grh.dgid.raw)) {
+
+ u32 retry_count = BNXT_RE_RESOLVE_RETRY_COUNT_US;
+ struct bnxt_re_resolve_dmac_work *resolve_dmac_work;
+
+
+ resolve_dmac_work = kzalloc(sizeof(*resolve_dmac_work), GFP_ATOMIC);
+
+ resolve_dmac_work->rdev = rdev;
+ resolve_dmac_work->ah_attr = ah_attr;
+ resolve_dmac_work->ah_info = ah_info;
+
+ atomic_set(&resolve_dmac_work->status_wait, 1);
+ INIT_WORK(&resolve_dmac_work->work, bnxt_re_resolve_dmac_task);
+ queue_work(rdev->resolve_wq, &resolve_dmac_work->work);
+
+ do {
+ rc = atomic_read(&resolve_dmac_work->status_wait) & 0xFF;
+ if (!rc)
+ break;
+ udelay(1);
+ } while (--retry_count);
+ if (atomic_read(&resolve_dmac_work->status_wait)) {
+ INIT_LIST_HEAD(&resolve_dmac_work->list);
+ list_add_tail(&resolve_dmac_work->list,
+ &rdev->mac_wq_list);
+ return -EFAULT;
+ }
+ kfree(resolve_dmac_work);
+ }
+ dmac = ROCE_DMAC(ah_attr);
+ if (dmac)
+ memcpy(ah->qplib_ah.dmac, dmac, ETH_ALEN);
+ return rc;
+}
+
+int bnxt_re_create_ah(struct ib_ah *ah_in, struct ib_ah_attr *attr,
+ u32 flags, struct ib_udata *udata)
+{
+
+ struct ib_ah *ib_ah = ah_in;
+ struct ib_pd *ib_pd = ib_ah->pd;
+ struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ibah);
+ struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ibpd);
+ struct bnxt_re_dev *rdev = pd->rdev;
+ struct bnxt_re_ah_info ah_info;
+ u32 max_ah_count;
+ bool is_user;
+ int rc;
+ bool block = true;
+ struct ib_ah_attr *ah_attr = attr;
+ block = !(flags & RDMA_CREATE_AH_SLEEPABLE);
+
+ if (!(ah_attr->ah_flags & IB_AH_GRH))
+ dev_err(rdev_to_dev(rdev), "ah_attr->ah_flags GRH is not set\n");
+
+ ah->rdev = rdev;
+ ah->qplib_ah.pd = &pd->qplib_pd;
+ is_user = ib_pd->uobject ? true : false;
+
+ /* Supply the configuration for the HW */
+ memcpy(ah->qplib_ah.dgid.data, ah_attr->grh.dgid.raw,
+ sizeof(union ib_gid));
+ ah->qplib_ah.sgid_index = _get_sgid_index(rdev, ah_attr->grh.sgid_index);
+ if (ah->qplib_ah.sgid_index == 0xFF) {
+ dev_err(rdev_to_dev(rdev), "invalid sgid_index!\n");
+ rc = -EINVAL;
+ goto fail;
+ }
+ ah->qplib_ah.host_sgid_index = ah_attr->grh.sgid_index;
+ ah->qplib_ah.traffic_class = ah_attr->grh.traffic_class;
+ ah->qplib_ah.flow_label = ah_attr->grh.flow_label;
+ ah->qplib_ah.hop_limit = ah_attr->grh.hop_limit;
+ ah->qplib_ah.sl = ah_attr->sl;
+ rc = bnxt_re_get_ah_info(rdev, ah_attr, &ah_info);
+ if (rc)
+ goto fail;
+ ah->qplib_ah.nw_type = ah_info.nw_type;
+
+ rc = bnxt_re_init_dmac(rdev, ah_attr, &ah_info, is_user, ah);
+ if (rc)
+ goto fail;
+
+ rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah, block);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev),
+ "Allocate HW Address Handle failed!\n");
+ goto fail;
+ }
+
+ /* Write AVID to shared page. */
+ if (ib_pd->uobject) {
+ struct ib_ucontext *ib_uctx = ib_pd->uobject->context;
+ struct bnxt_re_ucontext *uctx;
+ unsigned long flag;
+ u32 *wrptr;
+
+ uctx = to_bnxt_re(ib_uctx, struct bnxt_re_ucontext, ibucontext);
+ spin_lock_irqsave(&uctx->sh_lock, flag);
+ wrptr = (u32 *)((u8 *)uctx->shpg + BNXT_RE_AVID_OFFT);
+ *wrptr = ah->qplib_ah.id;
+ wmb(); /* make sure cache is updated. */
+ spin_unlock_irqrestore(&uctx->sh_lock, flag);
+ }
+ atomic_inc(&rdev->stats.rsors.ah_count);
+ max_ah_count = atomic_read(&rdev->stats.rsors.ah_count);
+ if (max_ah_count > atomic_read(&rdev->stats.rsors.max_ah_count))
+ atomic_set(&rdev->stats.rsors.max_ah_count, max_ah_count);
+
+ return 0;
+fail:
+ return rc;
+}
+
+int bnxt_re_modify_ah(struct ib_ah *ib_ah, struct ib_ah_attr *ah_attr)
+{
+ return 0;
+}
+
+int bnxt_re_query_ah(struct ib_ah *ib_ah, struct ib_ah_attr *ah_attr)
+{
+ struct bnxt_re_ah *ah = to_bnxt_re(ib_ah, struct bnxt_re_ah, ibah);
+
+ memcpy(ah_attr->grh.dgid.raw, ah->qplib_ah.dgid.data,
+ sizeof(union ib_gid));
+ ah_attr->grh.sgid_index = ah->qplib_ah.host_sgid_index;
+ ah_attr->grh.traffic_class = ah->qplib_ah.traffic_class;
+ ah_attr->sl = ah->qplib_ah.sl;
+ memcpy(ROCE_DMAC(ah_attr), ah->qplib_ah.dmac, ETH_ALEN);
+ ah_attr->ah_flags = IB_AH_GRH;
+ ah_attr->port_num = 1;
+ ah_attr->static_rate = 0;
+
+ return 0;
+}
+
+/* Shared Receive Queues */
+void bnxt_re_destroy_srq(struct ib_srq *ib_srq,
+ struct ib_udata *udata)
+{
+ struct bnxt_re_srq *srq = to_bnxt_re(ib_srq, struct bnxt_re_srq, ibsrq);
+ struct bnxt_re_dev *rdev = srq->rdev;
+ struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq;
+ int rc = 0;
+
+
+ rc = bnxt_qplib_destroy_srq(&rdev->qplib_res, qplib_srq);
+ if (rc)
+ dev_err_ratelimited(rdev_to_dev(rdev),
+ "%s id = %d failed rc = %d\n",
+ __func__, qplib_srq->id, rc);
+
+ if (srq->umem && !IS_ERR(srq->umem))
+ ib_umem_release(srq->umem);
+
+ atomic_dec(&rdev->stats.rsors.srq_count);
+
+ return;
+}
+
+static u16 _max_rwqe_sz(int nsge)
+{
+ return sizeof(struct rq_wqe_hdr) + (nsge * sizeof(struct sq_sge));
+}
+
+static u16 bnxt_re_get_rwqe_size(struct bnxt_qplib_qp *qplqp,
+ int rsge, int max)
+{
+ if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
+ rsge = max;
+
+ return _max_rwqe_sz(rsge);
+}
+
+static inline
+struct ib_umem *ib_umem_get_compat(struct bnxt_re_dev *rdev,
+ struct ib_ucontext *ucontext,
+ struct ib_udata *udata,
+ unsigned long addr,
+ size_t size, int access, int dmasync)
+{
+ return ib_umem_get(ucontext, addr, size, access, dmasync);
+}
+
+static inline
+struct ib_umem *ib_umem_get_flags_compat(struct bnxt_re_dev *rdev,
+ struct ib_ucontext *ucontext,
+ struct ib_udata *udata,
+ unsigned long addr,
+ size_t size, int access, int dmasync)
+{
+ return ib_umem_get_compat(rdev, ucontext, udata, addr, size,
+ access, 0);
+}
+
+static inline size_t ib_umem_num_pages_compat(struct ib_umem *umem)
+{
+ return ib_umem_num_pages(umem);
+}
+
+static int bnxt_re_init_user_srq(struct bnxt_re_dev *rdev,
+ struct bnxt_re_pd *pd,
+ struct bnxt_re_srq *srq,
+ struct ib_udata *udata)
+{
+ struct bnxt_qplib_sg_info *sginfo;
+ struct bnxt_qplib_srq *qplib_srq;
+ struct bnxt_re_ucontext *cntx;
+ struct ib_ucontext *context;
+ struct bnxt_re_srq_req ureq;
+ struct ib_umem *umem;
+ int rc, bytes = 0;
+
+ context = pd->ibpd.uobject->context;
+ cntx = to_bnxt_re(context, struct bnxt_re_ucontext, ibucontext);
+ qplib_srq = &srq->qplib_srq;
+ sginfo = &qplib_srq->sginfo;
+
+ if (udata->inlen < sizeof(ureq))
+ dev_warn(rdev_to_dev(rdev),
+ "Update the library ulen %d klen %d\n",
+ (unsigned int)udata->inlen,
+ (unsigned int)sizeof(ureq));
+
+ rc = ib_copy_from_udata(&ureq, udata,
+ min(udata->inlen, sizeof(ureq)));
+ if (rc)
+ return rc;
+
+ bytes = (qplib_srq->max_wqe * qplib_srq->wqe_size);
+ bytes = PAGE_ALIGN(bytes);
+ umem = ib_umem_get_compat(rdev, context, udata, ureq.srqva, bytes,
+ IB_ACCESS_LOCAL_WRITE, 1);
+ if (IS_ERR(umem)) {
+ dev_err(rdev_to_dev(rdev), "%s: ib_umem_get failed with %ld\n",
+ __func__, PTR_ERR(umem));
+ return PTR_ERR(umem);
+ }
+
+ srq->umem = umem;
+ sginfo->sghead = get_ib_umem_sgl(umem, &sginfo->nmap);
+ sginfo->npages = ib_umem_num_pages_compat(umem);
+ qplib_srq->srq_handle = ureq.srq_handle;
+ qplib_srq->dpi = &cntx->dpi;
+ qplib_srq->is_user = true;
+
+ return 0;
+}
+
+int bnxt_re_create_srq(struct ib_srq *srq_in, struct ib_srq_init_attr *srq_init_attr,
+ struct ib_udata *udata)
+{
+ struct bnxt_qplib_dev_attr *dev_attr;
+ struct bnxt_re_ucontext *cntx = NULL;
+ struct ib_ucontext *context;
+ struct bnxt_re_dev *rdev;
+ struct bnxt_re_pd *pd;
+ int rc, entries;
+ struct ib_srq *ib_srq = srq_in;
+ struct ib_pd *ib_pd = ib_srq->pd;
+ struct bnxt_re_srq *srq =
+ container_of(ib_srq, struct bnxt_re_srq, ibsrq);
+ u32 max_srq_count;
+
+ pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd);
+ rdev = pd->rdev;
+ dev_attr = rdev->dev_attr;
+
+ if (rdev->mod_exit) {
+ dev_dbg(rdev_to_dev(rdev), "%s(): in mod_exit, just return!\n", __func__);
+ rc = -EIO;
+ goto exit;
+ }
+
+ if (srq_init_attr->srq_type != IB_SRQT_BASIC) {
+ dev_err(rdev_to_dev(rdev), "SRQ type not supported\n");
+ rc = -ENOTSUPP;
+ goto exit;
+ }
+
+ if (udata) {
+ context = pd->ibpd.uobject->context;
+ cntx = to_bnxt_re(context, struct bnxt_re_ucontext, ibucontext);
+ }
+
+ if (atomic_read(&rdev->stats.rsors.srq_count) >= dev_attr->max_srq) {
+ dev_err(rdev_to_dev(rdev), "Create SRQ failed - max exceeded(SRQs)\n");
+ rc = -EINVAL;
+ goto exit;
+ }
+
+ if (srq_init_attr->attr.max_wr >= dev_attr->max_srq_wqes) {
+ dev_err(rdev_to_dev(rdev), "Create SRQ failed - max exceeded(SRQ_WQs)\n");
+ rc = -EINVAL;
+ goto exit;
+ }
+
+ srq->rdev = rdev;
+ srq->qplib_srq.pd = &pd->qplib_pd;
+ srq->qplib_srq.dpi = &rdev->dpi_privileged;
+
+ /* Allocate 1 more than what's provided so posting max doesn't
+ mean empty */
+ entries = srq_init_attr->attr.max_wr + 1;
+ entries = bnxt_re_init_depth(entries, cntx);
+ if (entries > dev_attr->max_srq_wqes + 1)
+ entries = dev_attr->max_srq_wqes + 1;
+
+ srq->qplib_srq.wqe_size = _max_rwqe_sz(6); /* 128 byte wqe size */
+ srq->qplib_srq.max_wqe = entries;
+ srq->qplib_srq.max_sge = srq_init_attr->attr.max_sge;
+ srq->qplib_srq.threshold = srq_init_attr->attr.srq_limit;
+ srq->srq_limit = srq_init_attr->attr.srq_limit;
+ srq->qplib_srq.eventq_hw_ring_id = rdev->nqr.nq[0].ring_id;
+ srq->qplib_srq.sginfo.pgsize = PAGE_SIZE;
+ srq->qplib_srq.sginfo.pgshft = PAGE_SHIFT;
+
+ if (udata) {
+ rc = bnxt_re_init_user_srq(rdev, pd, srq, udata);
+ if (rc)
+ goto fail;
+ }
+
+ rc = bnxt_qplib_create_srq(&rdev->qplib_res, &srq->qplib_srq);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "Create HW SRQ failed!\n");
+ goto fail;
+ }
+
+ if (udata) {
+ struct bnxt_re_srq_resp resp;
+
+ resp.srqid = srq->qplib_srq.id;
+ rc = bnxt_re_copy_to_udata(rdev, &resp,
+ min(udata->outlen, sizeof(resp)),
+ udata);
+ if (rc) {
+ bnxt_qplib_destroy_srq(&rdev->qplib_res, &srq->qplib_srq);
+ goto fail;
+ }
+ }
+ atomic_inc(&rdev->stats.rsors.srq_count);
+ max_srq_count = atomic_read(&rdev->stats.rsors.srq_count);
+ if (max_srq_count > atomic_read(&rdev->stats.rsors.max_srq_count))
+ atomic_set(&rdev->stats.rsors.max_srq_count, max_srq_count);
+ spin_lock_init(&srq->lock);
+
+ return 0;
+fail:
+ if (udata && srq->umem && !IS_ERR(srq->umem)) {
+ ib_umem_release(srq->umem);
+ srq->umem = NULL;
+ }
+exit:
+ return rc;
+}
+
+int bnxt_re_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr,
+ enum ib_srq_attr_mask srq_attr_mask,
+ struct ib_udata *udata)
+{
+ struct bnxt_re_srq *srq = to_bnxt_re(ib_srq, struct bnxt_re_srq,
+ ibsrq);
+ struct bnxt_re_dev *rdev = srq->rdev;
+ int rc;
+
+ switch (srq_attr_mask) {
+ case IB_SRQ_MAX_WR:
+ /* SRQ resize is not supported */
+ break;
+ case IB_SRQ_LIMIT:
+ /* Change the SRQ threshold */
+ if (srq_attr->srq_limit > srq->qplib_srq.max_wqe)
+ return -EINVAL;
+
+ srq->qplib_srq.threshold = srq_attr->srq_limit;
+ rc = bnxt_qplib_modify_srq(&rdev->qplib_res, &srq->qplib_srq);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "Modify HW SRQ failed!\n");
+ return rc;
+ }
+ /* On success, update the shadow */
+ srq->srq_limit = srq_attr->srq_limit;
+
+ if (udata) {
+ /* Build and send response back to udata */
+ rc = bnxt_re_copy_to_udata(rdev, srq, 0, udata);
+ if (rc)
+ return rc;
+ }
+ break;
+ default:
+ dev_err(rdev_to_dev(rdev),
+ "Unsupported srq_attr_mask 0x%x\n", srq_attr_mask);
+ return -EINVAL;
+ }
+ return 0;
+}
+
+int bnxt_re_query_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr)
+{
+ struct bnxt_re_srq *srq = to_bnxt_re(ib_srq, struct bnxt_re_srq,
+ ibsrq);
+ struct bnxt_re_dev *rdev = srq->rdev;
+ int rc;
+
+ rc = bnxt_qplib_query_srq(&rdev->qplib_res, &srq->qplib_srq);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "Query HW SRQ (0x%x) failed! rc = %d\n",
+ srq->qplib_srq.id, rc);
+ return rc;
+ }
+ srq_attr->max_wr = srq->qplib_srq.max_wqe;
+ srq_attr->max_sge = srq->qplib_srq.max_sge;
+ srq_attr->srq_limit = srq->qplib_srq.threshold;
+
+ return 0;
+}
+
+int bnxt_re_post_srq_recv(struct ib_srq *ib_srq, const struct ib_recv_wr *wr,
+ const struct ib_recv_wr **bad_wr)
+{
+ struct bnxt_re_srq *srq = to_bnxt_re(ib_srq, struct bnxt_re_srq,
+ ibsrq);
+ struct bnxt_qplib_swqe wqe = {};
+ unsigned long flags;
+ int rc = 0;
+
+ spin_lock_irqsave(&srq->lock, flags);
+ while (wr) {
+ /* Transcribe each ib_recv_wr to qplib_swqe */
+ wqe.num_sge = wr->num_sge;
+ wqe.sg_list = (struct bnxt_qplib_sge *)wr->sg_list;
+ wqe.wr_id = wr->wr_id;
+ wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
+ rc = bnxt_qplib_post_srq_recv(&srq->qplib_srq, &wqe);
+ if (rc) {
+ *bad_wr = wr;
+ break;
+ }
+ wr = wr->next;
+ }
+ spin_unlock_irqrestore(&srq->lock, flags);
+
+ return rc;
+}
+
+unsigned long bnxt_re_lock_cqs(struct bnxt_re_qp *qp)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&qp->scq->cq_lock, flags);
+ if (qp->rcq && qp->rcq != qp->scq)
+ spin_lock(&qp->rcq->cq_lock);
+
+ return flags;
+}
+
+void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp,
+ unsigned long flags)
+{
+ if (qp->rcq && qp->rcq != qp->scq)
+ spin_unlock(&qp->rcq->cq_lock);
+ spin_unlock_irqrestore(&qp->scq->cq_lock, flags);
+}
+
+/* Queue Pairs */
+static int bnxt_re_destroy_gsi_sqp(struct bnxt_re_qp *qp)
+{
+ struct bnxt_re_qp *gsi_sqp;
+ struct bnxt_re_ah *gsi_sah;
+ struct bnxt_re_dev *rdev;
+ unsigned long flags;
+ int rc = 0;
+
+ rdev = qp->rdev;
+ gsi_sqp = rdev->gsi_ctx.gsi_sqp;
+ gsi_sah = rdev->gsi_ctx.gsi_sah;
+
+ /* remove from active qp list */
+ mutex_lock(&rdev->qp_lock);
+ list_del(&gsi_sqp->list);
+ mutex_unlock(&rdev->qp_lock);
+
+ if (gsi_sah) {
+ dev_dbg(rdev_to_dev(rdev), "Destroy the shadow AH\n");
+ rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, &gsi_sah->qplib_ah,
+ true);
+ if (rc)
+ dev_err(rdev_to_dev(rdev),
+ "Destroy HW AH for shadow QP failed!\n");
+ atomic_dec(&rdev->stats.rsors.ah_count);
+ }
+
+ dev_dbg(rdev_to_dev(rdev), "Destroy the shadow QP\n");
+ rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &gsi_sqp->qplib_qp);
+ if (rc)
+ dev_err(rdev_to_dev(rdev), "Destroy Shadow QP failed\n");
+
+ /* Clean the CQ for shadow QP completions */
+ flags = bnxt_re_lock_cqs(gsi_sqp);
+ bnxt_qplib_clean_qp(&gsi_sqp->qplib_qp);
+ bnxt_re_unlock_cqs(gsi_sqp, flags);
+
+ bnxt_qplib_free_qp_res(&rdev->qplib_res, &gsi_sqp->qplib_qp);
+ bnxt_qplib_free_hdr_buf(&rdev->qplib_res, &gsi_sqp->qplib_qp);
+ kfree(rdev->gsi_ctx.sqp_tbl);
+ kfree(gsi_sah);
+ kfree(gsi_sqp);
+ rdev->gsi_ctx.gsi_sqp = NULL;
+ rdev->gsi_ctx.gsi_sah = NULL;
+ rdev->gsi_ctx.sqp_tbl = NULL;
+ atomic_dec(&rdev->stats.rsors.qp_count);
+
+ return 0;
+}
+
+static void bnxt_re_dump_debug_stats(struct bnxt_re_dev *rdev, u32 active_qps)
+{
+ u32 total_qp = 0;
+ u64 avg_time = 0;
+ int i;
+
+ if (!rdev->rcfw.sp_perf_stats_enabled)
+ return;
+
+ switch (active_qps) {
+ case 1:
+ /* Potential hint for Test Stop */
+ for (i = 0; i < RCFW_MAX_STAT_INDEX; i++) {
+ if (rdev->rcfw.qp_destroy_stats[i]) {
+ total_qp++;
+ avg_time += rdev->rcfw.qp_destroy_stats[i];
+ }
+ }
+ if (total_qp >= 0 || avg_time >= 0)
+ dev_dbg(rdev_to_dev(rdev),
+ "Perf Debug: %ps Total (%d) QP destroyed in (%ld) msec\n",
+ __builtin_return_address(0), total_qp,
+ (long)jiffies_to_msecs(avg_time));
+ break;
+ case 2:
+ /* Potential hint for Test Start */
+ dev_dbg(rdev_to_dev(rdev),
+ "Perf Debug: %ps active_qps = %d\n",
+ __builtin_return_address(0), active_qps);
+ break;
+ default:
+ /* Potential hint to know latency of QP destroy.
+ * Average time taken for 1K QP Destroy.
+ */
+ if (active_qps > 1024 && !(active_qps % 1024))
+ dev_dbg(rdev_to_dev(rdev),
+ "Perf Debug: %ps Active QP (%d) Watermark (%d)\n",
+ __builtin_return_address(0), active_qps,
+ atomic_read(&rdev->stats.rsors.max_qp_count));
+ break;
+ }
+}
+
+int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata)
+{
+ struct bnxt_re_qp *qp = to_bnxt_re(ib_qp, struct bnxt_re_qp, ib_qp);
+ struct bnxt_re_dev *rdev = qp->rdev;
+ unsigned long flags;
+ u32 active_qps;
+ int rc;
+
+ mutex_lock(&rdev->qp_lock);
+ list_del(&qp->list);
+ active_qps = atomic_dec_return(&rdev->stats.rsors.qp_count);
+ if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_RC)
+ atomic_dec(&rdev->stats.rsors.rc_qp_count);
+ else if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD)
+ atomic_dec(&rdev->stats.rsors.ud_qp_count);
+ mutex_unlock(&rdev->qp_lock);
+
+ rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp);
+ if (rc)
+ dev_err_ratelimited(rdev_to_dev(rdev),
+ "%s id = %d failed rc = %d\n",
+ __func__, qp->qplib_qp.id, rc);
+
+ if (!ib_qp->uobject) {
+ flags = bnxt_re_lock_cqs(qp);
+ bnxt_qplib_clean_qp(&qp->qplib_qp);
+ bnxt_re_unlock_cqs(qp, flags);
+ }
+
+ bnxt_qplib_free_qp_res(&rdev->qplib_res, &qp->qplib_qp);
+ if (ib_qp->qp_type == IB_QPT_GSI &&
+ rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) {
+ if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL &&
+ rdev->gsi_ctx.gsi_sqp) {
+ bnxt_re_destroy_gsi_sqp(qp);
+ }
+ bnxt_qplib_free_hdr_buf(&rdev->qplib_res, &qp->qplib_qp);
+ }
+
+ if (qp->rumem && !IS_ERR(qp->rumem))
+ ib_umem_release(qp->rumem);
+ if (qp->sumem && !IS_ERR(qp->sumem))
+ ib_umem_release(qp->sumem);
+ kfree(qp);
+
+ bnxt_re_dump_debug_stats(rdev, active_qps);
+
+ return 0;
+}
+
+static u8 __from_ib_qp_type(enum ib_qp_type type)
+{
+ switch (type) {
+ case IB_QPT_GSI:
+ return CMDQ_CREATE_QP1_TYPE_GSI;
+ case IB_QPT_RC:
+ return CMDQ_CREATE_QP_TYPE_RC;
+ case IB_QPT_UD:
+ return CMDQ_CREATE_QP_TYPE_UD;
+ case IB_QPT_RAW_ETHERTYPE:
+ return CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE;
+ default:
+ return IB_QPT_MAX;
+ }
+}
+
+static u16 _get_swqe_sz(int nsge)
+{
+ return sizeof(struct sq_send_hdr) + nsge * sizeof(struct sq_sge);
+}
+
+static int bnxt_re_get_swqe_size(int ilsize, int nsge)
+{
+ u16 wqe_size, calc_ils;
+
+ wqe_size = _get_swqe_sz(nsge);
+ if (ilsize) {
+ calc_ils = (sizeof(struct sq_send_hdr) + ilsize);
+ wqe_size = max_t(int, calc_ils, wqe_size);
+ wqe_size = ALIGN(wqe_size, 32);
+ }
+ return wqe_size;
+}
+
+static int bnxt_re_setup_swqe_size(struct bnxt_re_qp *qp,
+ struct ib_qp_init_attr *init_attr)
+{
+ struct bnxt_qplib_dev_attr *dev_attr;
+ struct bnxt_qplib_qp *qplqp;
+ struct bnxt_re_dev *rdev;
+ struct bnxt_qplib_q *sq;
+ int align, ilsize;
+
+ rdev = qp->rdev;
+ qplqp = &qp->qplib_qp;
+ sq = &qplqp->sq;
+ dev_attr = rdev->dev_attr;
+
+ align = sizeof(struct sq_send_hdr);
+ ilsize = ALIGN(init_attr->cap.max_inline_data, align);
+
+ sq->wqe_size = bnxt_re_get_swqe_size(ilsize, sq->max_sge);
+ if (sq->wqe_size > _get_swqe_sz(dev_attr->max_qp_sges))
+ return -EINVAL;
+ /* For Cu/Wh and gen p5 backward compatibility mode
+ * wqe size is fixed to 128 bytes
+ */
+ if (sq->wqe_size < _get_swqe_sz(dev_attr->max_qp_sges) &&
+ qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
+ sq->wqe_size = _get_swqe_sz(dev_attr->max_qp_sges);
+
+ if (init_attr->cap.max_inline_data) {
+ qplqp->max_inline_data = sq->wqe_size -
+ sizeof(struct sq_send_hdr);
+ init_attr->cap.max_inline_data = qplqp->max_inline_data;
+ if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
+ sq->max_sge = qplqp->max_inline_data /
+ sizeof(struct sq_sge);
+ }
+
+ return 0;
+}
+
+static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev,
+ struct bnxt_re_pd *pd, struct bnxt_re_qp *qp,
+ struct ib_udata *udata)
+{
+ struct bnxt_qplib_sg_info *sginfo;
+ struct bnxt_qplib_qp *qplib_qp;
+ struct bnxt_re_ucontext *cntx;
+ struct ib_ucontext *context;
+ struct bnxt_re_qp_req ureq;
+ struct ib_umem *umem;
+ int rc, bytes = 0;
+ int psn_nume;
+ int psn_sz;
+
+ qplib_qp = &qp->qplib_qp;
+ context = pd->ibpd.uobject->context;
+ cntx = to_bnxt_re(context, struct bnxt_re_ucontext, ibucontext);
+ sginfo = &qplib_qp->sq.sginfo;
+
+ if (udata->inlen < sizeof(ureq))
+ dev_warn(rdev_to_dev(rdev),
+ "Update the library ulen %d klen %d\n",
+ (unsigned int)udata->inlen,
+ (unsigned int)sizeof(ureq));
+
+ rc = ib_copy_from_udata(&ureq, udata,
+ min(udata->inlen, sizeof(ureq)));
+ if (rc)
+ return rc;
+
+ bytes = (qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size);
+ /* Consider mapping PSN search memory only for RC QPs. */
+ if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) {
+ psn_sz = _is_chip_gen_p5_p7(rdev->chip_ctx) ?
+ sizeof(struct sq_psn_search_ext) :
+ sizeof(struct sq_psn_search);
+ if (rdev->dev_attr && BNXT_RE_HW_RETX(rdev->dev_attr->dev_cap_flags))
+ psn_sz = sizeof(struct sq_msn_search);
+ psn_nume = (qplib_qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?
+ qplib_qp->sq.max_wqe :
+ ((qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size) /
+ sizeof(struct bnxt_qplib_sge));
+ if (BNXT_RE_HW_RETX(rdev->dev_attr->dev_cap_flags))
+ psn_nume = roundup_pow_of_two(psn_nume);
+
+ bytes += (psn_nume * psn_sz);
+ }
+ bytes = PAGE_ALIGN(bytes);
+ umem = ib_umem_get_compat(rdev, context, udata, ureq.qpsva, bytes,
+ IB_ACCESS_LOCAL_WRITE, 1);
+ if (IS_ERR(umem)) {
+ dev_err(rdev_to_dev(rdev), "%s: ib_umem_get failed with %ld\n",
+ __func__, PTR_ERR(umem));
+ return PTR_ERR(umem);
+ }
+
+ qp->sumem = umem;
+ /* pgsize and pgshft were initialize already. */
+ sginfo->sghead = get_ib_umem_sgl(umem, &sginfo->nmap);
+ sginfo->npages = ib_umem_num_pages_compat(umem);
+ qplib_qp->qp_handle = ureq.qp_handle;
+
+ if (!qp->qplib_qp.srq) {
+ sginfo = &qplib_qp->rq.sginfo;
+ bytes = (qplib_qp->rq.max_wqe * qplib_qp->rq.wqe_size);
+ bytes = PAGE_ALIGN(bytes);
+ umem = ib_umem_get_compat(rdev,
+ context, udata, ureq.qprva, bytes,
+ IB_ACCESS_LOCAL_WRITE, 1);
+ if (IS_ERR(umem)) {
+ dev_err(rdev_to_dev(rdev),
+ "%s: ib_umem_get failed ret =%ld\n",
+ __func__, PTR_ERR(umem));
+ goto rqfail;
+ }
+ qp->rumem = umem;
+ /* pgsize and pgshft were initialize already. */
+ sginfo->sghead = get_ib_umem_sgl(umem, &sginfo->nmap);
+ sginfo->npages = ib_umem_num_pages_compat(umem);
+ }
+
+ qplib_qp->dpi = &cntx->dpi;
+ qplib_qp->is_user = true;
+
+ return 0;
+rqfail:
+ ib_umem_release(qp->sumem);
+ qp->sumem = NULL;
+ qplib_qp->sq.sginfo.sghead = NULL;
+ qplib_qp->sq.sginfo.nmap = 0;
+
+ return PTR_ERR(umem);
+}
+
+static struct bnxt_re_ah *bnxt_re_create_shadow_qp_ah(struct bnxt_re_pd *pd,
+ struct bnxt_qplib_res *qp1_res,
+ struct bnxt_qplib_qp *qp1_qp)
+{
+ struct bnxt_re_dev *rdev = pd->rdev;
+ struct bnxt_re_ah *ah;
+ union ib_gid sgid;
+ int rc;
+
+ ah = kzalloc(sizeof(*ah), GFP_KERNEL);
+ if (!ah) {
+ dev_err(rdev_to_dev(rdev), "Allocate Address Handle failed!\n");
+ return NULL;
+ }
+ memset(ah, 0, sizeof(*ah));
+ ah->rdev = rdev;
+ ah->qplib_ah.pd = &pd->qplib_pd;
+
+ rc = bnxt_re_query_gid(&rdev->ibdev, 1, 0, &sgid);
+ if (rc)
+ goto fail;
+
+ /* supply the dgid data same as sgid */
+ memcpy(ah->qplib_ah.dgid.data, &sgid.raw,
+ sizeof(union ib_gid));
+ ah->qplib_ah.sgid_index = 0;
+
+ ah->qplib_ah.traffic_class = 0;
+ ah->qplib_ah.flow_label = 0;
+ ah->qplib_ah.hop_limit = 1;
+ ah->qplib_ah.sl = 0;
+ /* Have DMAC same as SMAC */
+ ether_addr_copy(ah->qplib_ah.dmac, rdev->dev_addr);
+ dev_dbg(rdev_to_dev(rdev), "ah->qplib_ah.dmac = %x:%x:%x:%x:%x:%x\n",
+ ah->qplib_ah.dmac[0], ah->qplib_ah.dmac[1], ah->qplib_ah.dmac[2],
+ ah->qplib_ah.dmac[3], ah->qplib_ah.dmac[4], ah->qplib_ah.dmac[5]);
+
+ rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah, true);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev),
+ "Allocate HW AH for Shadow QP failed!\n");
+ goto fail;
+ }
+ dev_dbg(rdev_to_dev(rdev), "AH ID = %d\n", ah->qplib_ah.id);
+ atomic_inc(&rdev->stats.rsors.ah_count);
+
+ return ah;
+fail:
+ kfree(ah);
+ return NULL;
+}
+
+void bnxt_re_update_shadow_ah(struct bnxt_re_dev *rdev)
+{
+ struct bnxt_re_qp *gsi_qp;
+ struct bnxt_re_ah *sah;
+ struct bnxt_re_pd *pd;
+ struct ib_pd *ib_pd;
+ int rc;
+
+ if (!rdev)
+ return;
+
+ sah = rdev->gsi_ctx.gsi_sah;
+
+ dev_dbg(rdev_to_dev(rdev), "Updating the AH\n");
+ if (sah) {
+ /* Check if the AH created with current mac address */
+ if (!compare_ether_header(sah->qplib_ah.dmac, rdev->dev_addr)) {
+ dev_dbg(rdev_to_dev(rdev),
+ "Not modifying shadow AH during AH update\n");
+ return;
+ }
+
+ gsi_qp = rdev->gsi_ctx.gsi_qp;
+ ib_pd = gsi_qp->ib_qp.pd;
+ pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd);
+ rc = bnxt_qplib_destroy_ah(&rdev->qplib_res,
+ &sah->qplib_ah, false);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev),
+ "Failed to destroy shadow AH during AH update\n");
+ return;
+ }
+ atomic_dec(&rdev->stats.rsors.ah_count);
+ kfree(sah);
+ rdev->gsi_ctx.gsi_sah = NULL;
+
+ sah = bnxt_re_create_shadow_qp_ah(pd, &rdev->qplib_res,
+ &gsi_qp->qplib_qp);
+ if (!sah) {
+ dev_err(rdev_to_dev(rdev),
+ "Failed to update AH for ShadowQP\n");
+ return;
+ }
+ rdev->gsi_ctx.gsi_sah = sah;
+ atomic_inc(&rdev->stats.rsors.ah_count);
+ }
+}
+
+static struct bnxt_re_qp *bnxt_re_create_shadow_qp(struct bnxt_re_pd *pd,
+ struct bnxt_qplib_res *qp1_res,
+ struct bnxt_qplib_qp *qp1_qp)
+{
+ struct bnxt_re_dev *rdev = pd->rdev;
+ struct bnxt_re_qp *qp;
+ int rc;
+
+ qp = kzalloc(sizeof(*qp), GFP_KERNEL);
+ if (!qp) {
+ dev_err(rdev_to_dev(rdev), "Allocate internal UD QP failed!\n");
+ return NULL;
+ }
+ memset(qp, 0, sizeof(*qp));
+ qp->rdev = rdev;
+
+ /* Initialize the shadow QP structure from the QP1 values */
+ ether_addr_copy(qp->qplib_qp.smac, rdev->dev_addr);
+ qp->qplib_qp.pd = &pd->qplib_pd;
+ qp->qplib_qp.qp_handle = (u64)&qp->qplib_qp;
+ qp->qplib_qp.type = IB_QPT_UD;
+
+ qp->qplib_qp.max_inline_data = 0;
+ qp->qplib_qp.sig_type = true;
+
+ /* Shadow QP SQ depth should be same as QP1 RQ depth */
+ qp->qplib_qp.sq.wqe_size = bnxt_re_get_swqe_size(0, 6);
+ qp->qplib_qp.sq.max_wqe = qp1_qp->rq.max_wqe;
+ qp->qplib_qp.sq.max_sge = 2;
+ /* Q full delta can be 1 since it is internal QP */
+ qp->qplib_qp.sq.q_full_delta = 1;
+ qp->qplib_qp.sq.sginfo.pgsize = PAGE_SIZE;
+ qp->qplib_qp.sq.sginfo.pgshft = PAGE_SHIFT;
+
+ qp->qplib_qp.scq = qp1_qp->scq;
+ qp->qplib_qp.rcq = qp1_qp->rcq;
+
+ qp->qplib_qp.rq.wqe_size = _max_rwqe_sz(6); /* 128 Byte wqe size */
+ qp->qplib_qp.rq.max_wqe = qp1_qp->rq.max_wqe;
+ qp->qplib_qp.rq.max_sge = qp1_qp->rq.max_sge;
+ qp->qplib_qp.rq.sginfo.pgsize = PAGE_SIZE;
+ qp->qplib_qp.rq.sginfo.pgshft = PAGE_SHIFT;
+ /* Q full delta can be 1 since it is internal QP */
+ qp->qplib_qp.rq.q_full_delta = 1;
+ qp->qplib_qp.mtu = qp1_qp->mtu;
+ qp->qplib_qp.dpi = &rdev->dpi_privileged;
+
+ rc = bnxt_qplib_alloc_hdr_buf(qp1_res, &qp->qplib_qp, 0,
+ BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6);
+ if (rc)
+ goto fail;
+
+ rc = bnxt_qplib_create_qp(qp1_res, &qp->qplib_qp);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "create HW QP failed!\n");
+ goto qp_fail;
+ }
+
+ dev_dbg(rdev_to_dev(rdev), "Created shadow QP with ID = %d\n",
+ qp->qplib_qp.id);
+ spin_lock_init(&qp->sq_lock);
+ INIT_LIST_HEAD(&qp->list);
+ mutex_lock(&rdev->qp_lock);
+ list_add_tail(&qp->list, &rdev->qp_list);
+ atomic_inc(&rdev->stats.rsors.qp_count);
+ mutex_unlock(&rdev->qp_lock);
+ return qp;
+qp_fail:
+ bnxt_qplib_free_hdr_buf(qp1_res, &qp->qplib_qp);
+fail:
+ kfree(qp);
+ return NULL;
+}
+
+static int bnxt_re_init_rq_attr(struct bnxt_re_qp *qp,
+ struct ib_qp_init_attr *init_attr, void *cntx)
+{
+ struct bnxt_qplib_dev_attr *dev_attr;
+ struct bnxt_qplib_qp *qplqp;
+ struct bnxt_re_dev *rdev;
+ struct bnxt_qplib_q *rq;
+ int entries;
+
+ rdev = qp->rdev;
+ qplqp = &qp->qplib_qp;
+ rq = &qplqp->rq;
+ dev_attr = rdev->dev_attr;
+
+ if (init_attr->srq) {
+ struct bnxt_re_srq *srq;
+
+ srq = to_bnxt_re(init_attr->srq, struct bnxt_re_srq, ibsrq);
+ if (!srq) {
+ dev_err(rdev_to_dev(rdev), "SRQ not found\n");
+ return -EINVAL;
+ }
+ qplqp->srq = &srq->qplib_srq;
+ rq->max_wqe = 0;
+ } else {
+ rq->max_sge = init_attr->cap.max_recv_sge;
+ if (rq->max_sge > dev_attr->max_qp_sges)
+ rq->max_sge = dev_attr->max_qp_sges;
+ init_attr->cap.max_recv_sge = rq->max_sge;
+ rq->wqe_size = bnxt_re_get_rwqe_size(qplqp, rq->max_sge,
+ dev_attr->max_qp_sges);
+
+ /* Allocate 1 more than what's provided so posting max doesn't
+ mean empty */
+ entries = init_attr->cap.max_recv_wr + 1;
+ entries = bnxt_re_init_depth(entries, cntx);
+ rq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + 1);
+ rq->q_full_delta = 0;
+ rq->sginfo.pgsize = PAGE_SIZE;
+ rq->sginfo.pgshft = PAGE_SHIFT;
+ }
+
+ return 0;
+}
+
+static void bnxt_re_adjust_gsi_rq_attr(struct bnxt_re_qp *qp)
+{
+ struct bnxt_qplib_dev_attr *dev_attr;
+ struct bnxt_qplib_qp *qplqp;
+ struct bnxt_re_dev *rdev;
+
+ rdev = qp->rdev;
+ qplqp = &qp->qplib_qp;
+ dev_attr = rdev->dev_attr;
+
+ if (rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD)
+ qplqp->rq.max_sge = dev_attr->max_qp_sges;
+}
+
+static int bnxt_re_init_sq_attr(struct bnxt_re_qp *qp,
+ struct ib_qp_init_attr *init_attr,
+ void *cntx)
+{
+ struct bnxt_qplib_dev_attr *dev_attr;
+ struct bnxt_qplib_qp *qplqp;
+ struct bnxt_re_dev *rdev;
+ struct bnxt_qplib_q *sq;
+ int diff = 0;
+ int entries;
+ int rc;
+
+ rdev = qp->rdev;
+ qplqp = &qp->qplib_qp;
+ sq = &qplqp->sq;
+ dev_attr = rdev->dev_attr;
+
+ sq->max_sge = init_attr->cap.max_send_sge;
+ if (sq->max_sge > dev_attr->max_qp_sges) {
+ sq->max_sge = dev_attr->max_qp_sges;
+ init_attr->cap.max_send_sge = sq->max_sge;
+ }
+ rc = bnxt_re_setup_swqe_size(qp, init_attr);
+ if (rc)
+ return rc;
+ /*
+ * Change the SQ depth if user has requested minimum using
+ * configfs. Only supported for kernel consumers. Setting
+ * min_tx_depth to 4096 to handle iser SQ full condition
+ * in most of the newer OS distros
+ */
+ entries = init_attr->cap.max_send_wr;
+ if (!cntx && rdev->min_tx_depth && init_attr->qp_type != IB_QPT_GSI) {
+ /*
+ * If users specify any value greater than 1 use min_tx_depth
+ * provided by user for comparison. Else, compare it with the
+ * BNXT_RE_MIN_KERNEL_QP_TX_DEPTH and adjust it accordingly.
+ */
+ if (rdev->min_tx_depth > 1 && entries < rdev->min_tx_depth)
+ entries = rdev->min_tx_depth;
+ else if (entries < BNXT_RE_MIN_KERNEL_QP_TX_DEPTH)
+ entries = BNXT_RE_MIN_KERNEL_QP_TX_DEPTH;
+ }
+ diff = bnxt_re_get_diff(cntx, rdev->chip_ctx);
+ entries = bnxt_re_init_depth(entries + diff + 1, cntx);
+ sq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + diff + 1);
+ sq->q_full_delta = diff + 1;
+ /*
+ * Reserving one slot for Phantom WQE. Application can
+ * post one extra entry in this case. But allowing this to avoid
+ * unexpected Queue full condition
+ */
+ sq->q_full_delta -= 1; /* becomes 0 for gen-p5 */
+ sq->sginfo.pgsize = PAGE_SIZE;
+ sq->sginfo.pgshft = PAGE_SHIFT;
+ return 0;
+}
+
+static void bnxt_re_adjust_gsi_sq_attr(struct bnxt_re_qp *qp,
+ struct ib_qp_init_attr *init_attr,
+ void *cntx)
+{
+ struct bnxt_qplib_dev_attr *dev_attr;
+ struct bnxt_qplib_qp *qplqp;
+ struct bnxt_re_dev *rdev;
+ int entries;
+
+ rdev = qp->rdev;
+ qplqp = &qp->qplib_qp;
+ dev_attr = rdev->dev_attr;
+
+ if (rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) {
+ entries = init_attr->cap.max_send_wr + 1;
+ entries = bnxt_re_init_depth(entries, cntx);
+ qplqp->sq.max_wqe = min_t(u32, entries,
+ dev_attr->max_qp_wqes + 1);
+ qplqp->sq.q_full_delta = qplqp->sq.max_wqe -
+ init_attr->cap.max_send_wr;
+ qplqp->sq.max_sge++; /* Need one extra sge to put UD header */
+ if (qplqp->sq.max_sge > dev_attr->max_qp_sges)
+ qplqp->sq.max_sge = dev_attr->max_qp_sges;
+ }
+}
+
+static int bnxt_re_init_qp_type(struct bnxt_re_dev *rdev,
+ struct ib_qp_init_attr *init_attr)
+{
+ struct bnxt_qplib_chip_ctx *chip_ctx;
+ struct bnxt_re_gsi_context *gsi_ctx;
+ int qptype;
+
+ chip_ctx = rdev->chip_ctx;
+ gsi_ctx = &rdev->gsi_ctx;
+
+ qptype = __from_ib_qp_type(init_attr->qp_type);
+ if (qptype == IB_QPT_MAX) {
+ dev_err(rdev_to_dev(rdev), "QP type 0x%x not supported\n",
+ qptype);
+ qptype = -EINVAL;
+ goto out;
+ }
+
+ if (_is_chip_gen_p5_p7(chip_ctx) && init_attr->qp_type == IB_QPT_GSI) {
+ /* For Thor always force UD mode. */
+ qptype = CMDQ_CREATE_QP_TYPE_GSI;
+ gsi_ctx->gsi_qp_mode = BNXT_RE_GSI_MODE_UD;
+ }
+out:
+ return qptype;
+}
+
+static int bnxt_re_init_qp_wqe_mode(struct bnxt_re_dev *rdev)
+{
+ return rdev->chip_ctx->modes.wqe_mode;
+}
+
+static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd,
+ struct ib_qp_init_attr *init_attr,
+ struct ib_udata *udata)
+{
+ struct bnxt_qplib_dev_attr *dev_attr;
+ struct bnxt_re_ucontext *cntx = NULL;
+ struct ib_ucontext *context;
+ struct bnxt_qplib_qp *qplqp;
+ struct bnxt_re_dev *rdev;
+ struct bnxt_re_cq *cq;
+ int rc = 0, qptype;
+
+ rdev = qp->rdev;
+ qplqp = &qp->qplib_qp;
+ dev_attr = rdev->dev_attr;
+
+ if (udata) {
+ context = pd->ibpd.uobject->context;
+ cntx = to_bnxt_re(context, struct bnxt_re_ucontext, ibucontext);
+ }
+
+ /* Setup misc params */
+ qplqp->is_user = false;
+ qplqp->pd = &pd->qplib_pd;
+ qplqp->qp_handle = (u64)qplqp;
+ qplqp->sig_type = ((init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ?
+ true : false);
+ qptype = bnxt_re_init_qp_type(rdev, init_attr);
+ if (qptype < 0) {
+ rc = qptype;
+ goto out;
+ }
+ qplqp->type = (u8)qptype;
+ qplqp->wqe_mode = bnxt_re_init_qp_wqe_mode(rdev);
+ ether_addr_copy(qplqp->smac, rdev->dev_addr);
+
+ if (init_attr->qp_type == IB_QPT_RC) {
+ qplqp->max_rd_atomic = dev_attr->max_qp_rd_atom;
+ qplqp->max_dest_rd_atomic = dev_attr->max_qp_init_rd_atom;
+ }
+ qplqp->mtu = ib_mtu_enum_to_int(iboe_get_mtu(rdev->netdev->if_mtu));
+ qplqp->dpi = &rdev->dpi_privileged; /* Doorbell page */
+ if (init_attr->create_flags) {
+ dev_dbg(rdev_to_dev(rdev),
+ "QP create flags 0x%x not supported\n",
+ init_attr->create_flags);
+ return -EOPNOTSUPP;
+ }
+
+ /* Setup CQs */
+ if (init_attr->send_cq) {
+ cq = to_bnxt_re(init_attr->send_cq, struct bnxt_re_cq, ibcq);
+ if (!cq) {
+ dev_err(rdev_to_dev(rdev), "Send CQ not found\n");
+ rc = -EINVAL;
+ goto out;
+ }
+ qplqp->scq = &cq->qplib_cq;
+ qp->scq = cq;
+ }
+
+ if (init_attr->recv_cq) {
+ cq = to_bnxt_re(init_attr->recv_cq, struct bnxt_re_cq, ibcq);
+ if (!cq) {
+ dev_err(rdev_to_dev(rdev), "Receive CQ not found\n");
+ rc = -EINVAL;
+ goto out;
+ }
+ qplqp->rcq = &cq->qplib_cq;
+ qp->rcq = cq;
+ }
+
+ /* Setup RQ/SRQ */
+ rc = bnxt_re_init_rq_attr(qp, init_attr, cntx);
+ if (rc)
+ goto out;
+ if (init_attr->qp_type == IB_QPT_GSI)
+ bnxt_re_adjust_gsi_rq_attr(qp);
+
+ /* Setup SQ */
+ rc = bnxt_re_init_sq_attr(qp, init_attr, cntx);
+ if (rc)
+ goto out;
+ if (init_attr->qp_type == IB_QPT_GSI)
+ bnxt_re_adjust_gsi_sq_attr(qp, init_attr, cntx);
+
+ if (udata) /* This will update DPI and qp_handle */
+ rc = bnxt_re_init_user_qp(rdev, pd, qp, udata);
+out:
+ return rc;
+}
+
+static int bnxt_re_create_shadow_gsi(struct bnxt_re_qp *qp,
+ struct bnxt_re_pd *pd)
+{
+ struct bnxt_re_sqp_entries *sqp_tbl = NULL;
+ struct bnxt_re_dev *rdev;
+ struct bnxt_re_qp *sqp;
+ struct bnxt_re_ah *sah;
+ int rc = 0;
+
+ rdev = qp->rdev;
+ /* Create a shadow QP to handle the QP1 traffic */
+ sqp_tbl = kzalloc(sizeof(*sqp_tbl) * BNXT_RE_MAX_GSI_SQP_ENTRIES,
+ GFP_KERNEL);
+ if (!sqp_tbl)
+ return -ENOMEM;
+ rdev->gsi_ctx.sqp_tbl = sqp_tbl;
+
+ sqp = bnxt_re_create_shadow_qp(pd, &rdev->qplib_res, &qp->qplib_qp);
+ if (!sqp) {
+ rc = -ENODEV;
+ dev_err(rdev_to_dev(rdev),
+ "Failed to create Shadow QP for QP1\n");
+ goto out;
+ }
+ rdev->gsi_ctx.gsi_sqp = sqp;
+
+ sqp->rcq = qp->rcq;
+ sqp->scq = qp->scq;
+ sah = bnxt_re_create_shadow_qp_ah(pd, &rdev->qplib_res,
+ &qp->qplib_qp);
+ if (!sah) {
+ bnxt_qplib_destroy_qp(&rdev->qplib_res,
+ &sqp->qplib_qp);
+ rc = -ENODEV;
+ dev_err(rdev_to_dev(rdev),
+ "Failed to create AH entry for ShadowQP\n");
+ goto out;
+ }
+ rdev->gsi_ctx.gsi_sah = sah;
+
+ return 0;
+out:
+ kfree(sqp_tbl);
+ return rc;
+}
+
+static int __get_rq_hdr_buf_size(u8 gsi_mode)
+{
+ return (gsi_mode == BNXT_RE_GSI_MODE_ALL) ?
+ BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2 :
+ BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE;
+}
+
+static int __get_sq_hdr_buf_size(u8 gsi_mode)
+{
+ return (gsi_mode != BNXT_RE_GSI_MODE_ROCE_V1) ?
+ BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2 :
+ BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE;
+}
+
+static int bnxt_re_create_gsi_qp(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd)
+{
+ struct bnxt_qplib_qp *qplqp;
+ struct bnxt_qplib_res *res;
+ struct bnxt_re_dev *rdev;
+ u32 sstep, rstep;
+ u8 gsi_mode;
+ int rc = 0;
+
+ rdev = qp->rdev;
+ qplqp = &qp->qplib_qp;
+ res = &rdev->qplib_res;
+ gsi_mode = rdev->gsi_ctx.gsi_qp_mode;
+
+ rstep = __get_rq_hdr_buf_size(gsi_mode);
+ sstep = __get_sq_hdr_buf_size(gsi_mode);
+ rc = bnxt_qplib_alloc_hdr_buf(res, qplqp, sstep, rstep);
+ if (rc)
+ goto out;
+
+ rc = bnxt_qplib_create_qp1(res, qplqp);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "create HW QP1 failed!\n");
+ goto out;
+ }
+
+ if (gsi_mode == BNXT_RE_GSI_MODE_ALL)
+ rc = bnxt_re_create_shadow_gsi(qp, pd);
+out:
+ return rc;
+}
+
+static bool bnxt_re_test_qp_limits(struct bnxt_re_dev *rdev,
+ struct ib_qp_init_attr *init_attr,
+ struct bnxt_qplib_dev_attr *dev_attr)
+{
+ bool rc = true;
+ int ilsize;
+
+ ilsize = ALIGN(init_attr->cap.max_inline_data, sizeof(struct sq_sge));
+ if ((init_attr->cap.max_send_wr > dev_attr->max_qp_wqes) ||
+ (init_attr->cap.max_recv_wr > dev_attr->max_qp_wqes) ||
+ (init_attr->cap.max_send_sge > dev_attr->max_qp_sges) ||
+ (init_attr->cap.max_recv_sge > dev_attr->max_qp_sges) ||
+ (ilsize > dev_attr->max_inline_data)) {
+ dev_err(rdev_to_dev(rdev), "Create QP failed - max exceeded! "
+ "0x%x/0x%x 0x%x/0x%x 0x%x/0x%x "
+ "0x%x/0x%x 0x%x/0x%x\n",
+ init_attr->cap.max_send_wr, dev_attr->max_qp_wqes,
+ init_attr->cap.max_recv_wr, dev_attr->max_qp_wqes,
+ init_attr->cap.max_send_sge, dev_attr->max_qp_sges,
+ init_attr->cap.max_recv_sge, dev_attr->max_qp_sges,
+ init_attr->cap.max_inline_data,
+ dev_attr->max_inline_data);
+ rc = false;
+ }
+ return rc;
+}
+
+static inline struct
+bnxt_re_qp *__get_qp_from_qp_in(struct ib_pd *qp_in,
+ struct bnxt_re_dev *rdev)
+{
+ struct bnxt_re_qp *qp;
+
+ qp = kzalloc(sizeof(*qp), GFP_KERNEL);
+ if (!qp)
+ dev_err(rdev_to_dev(rdev), "Allocate QP failed!\n");
+ return qp;
+}
+
+struct ib_qp *bnxt_re_create_qp(struct ib_pd *qp_in,
+ struct ib_qp_init_attr *qp_init_attr,
+ struct ib_udata *udata)
+{
+ struct bnxt_re_pd *pd;
+ struct ib_pd *ib_pd = qp_in;
+ struct bnxt_qplib_dev_attr *dev_attr;
+ struct bnxt_re_dev *rdev;
+ u32 active_qps, tmp_qps;
+ struct bnxt_re_qp *qp;
+ int rc;
+
+ pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd);
+ rdev = pd->rdev;
+ dev_attr = rdev->dev_attr;
+ if (rdev->mod_exit) {
+ rc = -EIO;
+ dev_dbg(rdev_to_dev(rdev), "%s(): in mod_exit, just return!\n", __func__);
+ goto exit;
+ }
+
+ if (atomic_read(&rdev->stats.rsors.qp_count) >= dev_attr->max_qp) {
+ dev_err(rdev_to_dev(rdev), "Create QP failed - max exceeded(QPs Alloc'd %u of max %u)\n",
+ atomic_read(&rdev->stats.rsors.qp_count), dev_attr->max_qp);
+ rc = -EINVAL;
+ goto exit;
+ }
+
+ rc = bnxt_re_test_qp_limits(rdev, qp_init_attr, dev_attr);
+ if (!rc) {
+ rc = -EINVAL;
+ goto exit;
+ }
+ qp = __get_qp_from_qp_in(qp_in, rdev);
+ if (!qp) {
+ rc = -ENOMEM;
+ goto exit;
+ }
+ qp->rdev = rdev;
+
+ rc = bnxt_re_init_qp_attr(qp, pd, qp_init_attr, udata);
+ if (rc)
+ goto fail;
+
+ if (qp_init_attr->qp_type == IB_QPT_GSI &&
+ !_is_chip_gen_p5_p7(rdev->chip_ctx)) {
+ rc = bnxt_re_create_gsi_qp(qp, pd);
+ if (rc == -ENODEV)
+ goto qp_destroy;
+ if (rc)
+ goto fail;
+ } else {
+ rc = bnxt_qplib_create_qp(&rdev->qplib_res, &qp->qplib_qp);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "create HW QP failed!\n");
+ goto free_umem;
+ }
+
+ if (udata) {
+ struct bnxt_re_qp_resp resp;
+
+ resp.qpid = qp->qplib_qp.id;
+ rc = bnxt_re_copy_to_udata(rdev, &resp,
+ min(udata->outlen, sizeof(resp)),
+ udata);
+ if (rc)
+ goto qp_destroy;
+ }
+ }
+
+ qp->ib_qp.qp_num = qp->qplib_qp.id;
+ if (qp_init_attr->qp_type == IB_QPT_GSI)
+ rdev->gsi_ctx.gsi_qp = qp;
+ spin_lock_init(&qp->sq_lock);
+ spin_lock_init(&qp->rq_lock);
+ INIT_LIST_HEAD(&qp->list);
+ mutex_lock(&rdev->qp_lock);
+ list_add_tail(&qp->list, &rdev->qp_list);
+ mutex_unlock(&rdev->qp_lock);
+ atomic_inc(&rdev->stats.rsors.qp_count);
+ active_qps = atomic_read(&rdev->stats.rsors.qp_count);
+ if (active_qps > atomic_read(&rdev->stats.rsors.max_qp_count))
+ atomic_set(&rdev->stats.rsors.max_qp_count, active_qps);
+
+ bnxt_re_dump_debug_stats(rdev, active_qps);
+
+ /* Get the counters for RC QPs and UD QPs */
+ if (qp_init_attr->qp_type == IB_QPT_RC) {
+ tmp_qps = atomic_inc_return(&rdev->stats.rsors.rc_qp_count);
+ if (tmp_qps > atomic_read(&rdev->stats.rsors.max_rc_qp_count))
+ atomic_set(&rdev->stats.rsors.max_rc_qp_count, tmp_qps);
+ } else if (qp_init_attr->qp_type == IB_QPT_UD) {
+ tmp_qps = atomic_inc_return(&rdev->stats.rsors.ud_qp_count);
+ if (tmp_qps > atomic_read(&rdev->stats.rsors.max_ud_qp_count))
+ atomic_set(&rdev->stats.rsors.max_ud_qp_count, tmp_qps);
+ }
+
+ return &qp->ib_qp;
+
+qp_destroy:
+ bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp);
+free_umem:
+ if (udata) {
+ if (qp->rumem && !IS_ERR(qp->rumem))
+ ib_umem_release(qp->rumem);
+ if (qp->sumem && !IS_ERR(qp->sumem))
+ ib_umem_release(qp->sumem);
+ }
+fail:
+ kfree(qp);
+exit:
+ return ERR_PTR(rc);
+}
+
+static int bnxt_re_modify_shadow_qp(struct bnxt_re_dev *rdev,
+ struct bnxt_re_qp *qp1_qp,
+ int qp_attr_mask)
+{
+ struct bnxt_re_qp *qp = rdev->gsi_ctx.gsi_sqp;
+ int rc = 0;
+
+ if (qp_attr_mask & IB_QP_STATE) {
+ qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE;
+ qp->qplib_qp.state = qp1_qp->qplib_qp.state;
+ }
+ if (qp_attr_mask & IB_QP_PKEY_INDEX) {
+ qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY;
+ qp->qplib_qp.pkey_index = qp1_qp->qplib_qp.pkey_index;
+ }
+
+ if (qp_attr_mask & IB_QP_QKEY) {
+ qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY;
+ /* Using a Random QKEY */
+ qp->qplib_qp.qkey = BNXT_RE_QP_RANDOM_QKEY;
+ }
+ if (qp_attr_mask & IB_QP_SQ_PSN) {
+ qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN;
+ qp->qplib_qp.sq.psn = qp1_qp->qplib_qp.sq.psn;
+ }
+
+ rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp);
+ if (rc)
+ dev_err(rdev_to_dev(rdev), "Modify Shadow QP for QP1 failed\n");
+ return rc;
+}
+
+static u32 ipv4_from_gid(u8 *gid)
+{
+ return (gid[15] << 24 | gid[14] << 16 | gid[13] << 8 | gid[12]);
+}
+
+static u16 get_source_port(struct bnxt_re_dev *rdev,
+ struct bnxt_re_qp *qp)
+{
+ u8 ip_off, data[48], smac[ETH_ALEN];
+ u16 crc = 0, buf_len = 0, i;
+ u8 addr_len;
+ u32 qpn;
+
+ if (qp->qplib_qp.nw_type == CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6) {
+ addr_len = 6;
+ ip_off = 10;
+ } else {
+ addr_len = 4;
+ ip_off = 12;
+ }
+
+ memcpy(smac, qp->qplib_qp.smac, ETH_ALEN);
+
+ memset(data, 0, 48);
+ memcpy(data, qp->qplib_qp.ah.dmac, ETH_ALEN);
+ buf_len += ETH_ALEN;
+
+ memcpy(data + buf_len, smac, ETH_ALEN);
+ buf_len += ETH_ALEN;
+
+ memcpy(data + buf_len, qp->qplib_qp.ah.dgid.data + ip_off, addr_len);
+ buf_len += addr_len;
+
+ memcpy(data + buf_len, qp->qp_info_entry.sgid.raw + ip_off, addr_len);
+ buf_len += addr_len;
+
+ qpn = htonl(qp->qplib_qp.dest_qpn);
+ memcpy(data + buf_len, (u8 *)&qpn + 1, 3);
+ buf_len += 3;
+
+ for (i = 0; i < buf_len; i++)
+ crc = crc16(crc, (data + i), 1);
+
+ return crc;
+}
+
+static void bnxt_re_update_qp_info(struct bnxt_re_dev *rdev, struct bnxt_re_qp *qp)
+{
+ u16 type;
+
+ type = __from_hw_to_ib_qp_type(qp->qplib_qp.type);
+
+ /* User-space can extract ip address with sgid_index. */
+ if (ipv6_addr_v4mapped((struct in6_addr *)&qp->qplib_qp.ah.dgid)) {
+ qp->qp_info_entry.s_ip.ipv4_addr = ipv4_from_gid(qp->qp_info_entry.sgid.raw);
+ qp->qp_info_entry.d_ip.ipv4_addr = ipv4_from_gid(qp->qplib_qp.ah.dgid.data);
+ } else {
+ memcpy(&qp->qp_info_entry.s_ip.ipv6_addr, qp->qp_info_entry.sgid.raw,
+ sizeof(qp->qp_info_entry.s_ip.ipv6_addr));
+ memcpy(&qp->qp_info_entry.d_ip.ipv6_addr, qp->qplib_qp.ah.dgid.data,
+ sizeof(qp->qp_info_entry.d_ip.ipv6_addr));
+ }
+
+ if (type == IB_QPT_RC &&
+ (qp->qplib_qp.nw_type == CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4 ||
+ qp->qplib_qp.nw_type == CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6)) {
+ qp->qp_info_entry.s_port = get_source_port(rdev, qp);
+ }
+ qp->qp_info_entry.d_port = BNXT_RE_QP_DEST_PORT;
+}
+
+static void bnxt_qplib_manage_flush_qp(struct bnxt_re_qp *qp)
+{
+ struct bnxt_qplib_q *rq, *sq;
+ unsigned long flags;
+
+ if (qp->sumem)
+ return;
+
+ if (qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
+ rq = &qp->qplib_qp.rq;
+ sq = &qp->qplib_qp.sq;
+
+ dev_dbg(rdev_to_dev(qp->rdev),
+ "Move QP = %p to flush list\n", qp);
+ flags = bnxt_re_lock_cqs(qp);
+ bnxt_qplib_add_flush_qp(&qp->qplib_qp);
+ bnxt_re_unlock_cqs(qp, flags);
+
+ if (sq->hwq.prod != sq->hwq.cons)
+ bnxt_re_handle_cqn(&qp->scq->qplib_cq);
+
+ if (qp->rcq && (qp->rcq != qp->scq) &&
+ (rq->hwq.prod != rq->hwq.cons))
+ bnxt_re_handle_cqn(&qp->rcq->qplib_cq);
+ }
+
+ if (qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_RESET) {
+ dev_dbg(rdev_to_dev(qp->rdev),
+ "Move QP = %p out of flush list\n", qp);
+ flags = bnxt_re_lock_cqs(qp);
+ bnxt_qplib_clean_qp(&qp->qplib_qp);
+ bnxt_re_unlock_cqs(qp, flags);
+ }
+}
+
+bool ib_modify_qp_is_ok_compat(enum ib_qp_state cur_state,
+ enum ib_qp_state next_state,
+ enum ib_qp_type type,
+ enum ib_qp_attr_mask mask)
+{
+ return (ib_modify_qp_is_ok(cur_state, next_state,
+ type, mask));
+}
+
+int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
+ int qp_attr_mask, struct ib_udata *udata)
+{
+ enum ib_qp_state curr_qp_state, new_qp_state;
+ struct bnxt_re_modify_qp_ex_resp resp = {};
+ struct bnxt_re_modify_qp_ex_req ureq = {};
+ struct bnxt_qplib_dev_attr *dev_attr;
+ struct bnxt_qplib_ppp *ppp = NULL;
+ struct bnxt_re_dev *rdev;
+ struct bnxt_re_qp *qp;
+ struct ib_gid_attr *sgid_attr;
+ struct ib_gid_attr gid_attr;
+ union ib_gid sgid, *gid_ptr = NULL;
+ u8 nw_type;
+ int rc, entries, status;
+ bool is_copy_to_udata = false;
+ bool is_qpmtu_high = false;
+
+ qp = to_bnxt_re(ib_qp, struct bnxt_re_qp, ib_qp);
+ rdev = qp->rdev;
+ dev_attr = rdev->dev_attr;
+
+ qp->qplib_qp.modify_flags = 0;
+ ppp = &qp->qplib_qp.ppp;
+ if (qp_attr_mask & IB_QP_STATE) {
+ curr_qp_state = __to_ib_qp_state(qp->qplib_qp.cur_qp_state);
+ new_qp_state = qp_attr->qp_state;
+ if (!ib_modify_qp_is_ok_compat(curr_qp_state, new_qp_state,
+ ib_qp->qp_type, qp_attr_mask)) {
+ dev_err(rdev_to_dev(rdev),"invalid attribute mask=0x%x"
+ " specified for qpn=0x%x of type=0x%x"
+ " current_qp_state=0x%x, new_qp_state=0x%x\n",
+ qp_attr_mask, ib_qp->qp_num, ib_qp->qp_type,
+ curr_qp_state, new_qp_state);
+ return -EINVAL;
+ }
+ dev_dbg(rdev_to_dev(rdev), "%s:%d INFO attribute mask=0x%x qpn=0x%x "
+ "of type=0x%x current_qp_state=0x%x, new_qp_state=0x%x\n",
+ __func__, __LINE__, qp_attr_mask, ib_qp->qp_num,
+ ib_qp->qp_type, curr_qp_state, new_qp_state);
+ qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE;
+ qp->qplib_qp.state = __from_ib_qp_state(qp_attr->qp_state);
+
+ if (udata && curr_qp_state == IB_QPS_RESET &&
+ new_qp_state == IB_QPS_INIT) {
+ if (!ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
+ if (ureq.comp_mask &
+ BNXT_RE_COMP_MASK_MQP_EX_PPP_REQ_EN_MASK) {
+ ppp->req = BNXT_QPLIB_PPP_REQ;
+ ppp->dpi = ureq.dpi;
+ }
+ }
+ }
+ }
+ if (qp_attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) {
+ qp->qplib_qp.modify_flags |=
+ CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY;
+ qp->qplib_qp.en_sqd_async_notify = true;
+ }
+ if (qp_attr_mask & IB_QP_ACCESS_FLAGS) {
+ qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS;
+ qp->qplib_qp.access =
+ __from_ib_access_flags(qp_attr->qp_access_flags);
+ /* LOCAL_WRITE access must be set to allow RC receive */
+ qp->qplib_qp.access |= BNXT_QPLIB_ACCESS_LOCAL_WRITE;
+ qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE;
+ qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_READ;
+ }
+ if (qp_attr_mask & IB_QP_PKEY_INDEX) {
+ qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY;
+ qp->qplib_qp.pkey_index = qp_attr->pkey_index;
+ }
+ if (qp_attr_mask & IB_QP_QKEY) {
+ qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY;
+ qp->qplib_qp.qkey = qp_attr->qkey;
+ }
+ if (qp_attr_mask & IB_QP_AV) {
+ qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_DGID |
+ CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL |
+ CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX |
+ CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT |
+ CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS |
+ CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC |
+ CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID;
+ memcpy(qp->qplib_qp.ah.dgid.data, qp_attr->ah_attr.grh.dgid.raw,
+ sizeof(qp->qplib_qp.ah.dgid.data));
+ qp->qplib_qp.ah.flow_label = qp_attr->ah_attr.grh.flow_label;
+ qp->qplib_qp.ah.sgid_index = _get_sgid_index(rdev,
+ qp_attr->ah_attr.grh.sgid_index);
+ qp->qplib_qp.ah.host_sgid_index = qp_attr->ah_attr.grh.sgid_index;
+ qp->qplib_qp.ah.hop_limit = qp_attr->ah_attr.grh.hop_limit;
+ qp->qplib_qp.ah.traffic_class =
+ qp_attr->ah_attr.grh.traffic_class;
+ qp->qplib_qp.ah.sl = qp_attr->ah_attr.sl;
+ ether_addr_copy(qp->qplib_qp.ah.dmac, ROCE_DMAC(&qp_attr->ah_attr));
+ sgid_attr = &gid_attr;
+ status = bnxt_re_get_cached_gid(&rdev->ibdev, 1,
+ qp_attr->ah_attr.grh.sgid_index,
+ &sgid, &sgid_attr,
+ &qp_attr->ah_attr.grh, NULL);
+ if (!status)
+ if_rele(sgid_attr->ndev);
+ gid_ptr = &sgid;
+ if (sgid_attr->ndev) {
+ memcpy(qp->qplib_qp.smac, rdev->dev_addr,
+ ETH_ALEN);
+ nw_type = bnxt_re_gid_to_network_type(sgid_attr, &sgid);
+ dev_dbg(rdev_to_dev(rdev),
+ "Connection using the nw_type %d\n", nw_type);
+ switch (nw_type) {
+ case RDMA_NETWORK_IPV4:
+ qp->qplib_qp.nw_type =
+ CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4;
+ break;
+ case RDMA_NETWORK_IPV6:
+ qp->qplib_qp.nw_type =
+ CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6;
+ break;
+ default:
+ qp->qplib_qp.nw_type =
+ CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1;
+ break;
+ }
+ }
+ memcpy(&qp->qp_info_entry.sgid, gid_ptr, sizeof(qp->qp_info_entry.sgid));
+ }
+
+ /* MTU settings allowed only during INIT -> RTR */
+ if (qp_attr->qp_state == IB_QPS_RTR) {
+ bnxt_re_init_qpmtu(qp, rdev->netdev->if_mtu, qp_attr_mask, qp_attr,
+ &is_qpmtu_high);
+ if (udata && !ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
+ if (ureq.comp_mask & BNXT_RE_COMP_MASK_MQP_EX_PATH_MTU_MASK) {
+ resp.comp_mask |= BNXT_RE_COMP_MASK_MQP_EX_PATH_MTU_MASK;
+ resp.path_mtu = qp->qplib_qp.mtu;
+ is_copy_to_udata = true;
+ } else if (is_qpmtu_high) {
+ dev_err(rdev_to_dev(rdev), "qp %#x invalid mtu\n",
+ qp->qplib_qp.id);
+ return -EINVAL;
+ }
+ }
+ }
+
+ if (qp_attr_mask & IB_QP_TIMEOUT) {
+ qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT;
+ qp->qplib_qp.timeout = qp_attr->timeout;
+ }
+ if (qp_attr_mask & IB_QP_RETRY_CNT) {
+ qp->qplib_qp.modify_flags |=
+ CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT;
+ qp->qplib_qp.retry_cnt = qp_attr->retry_cnt;
+ }
+ if (qp_attr_mask & IB_QP_RNR_RETRY) {
+ qp->qplib_qp.modify_flags |=
+ CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY;
+ qp->qplib_qp.rnr_retry = qp_attr->rnr_retry;
+ }
+ if (qp_attr_mask & IB_QP_MIN_RNR_TIMER) {
+ qp->qplib_qp.modify_flags |=
+ CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER;
+ qp->qplib_qp.min_rnr_timer = qp_attr->min_rnr_timer;
+ }
+ if (qp_attr_mask & IB_QP_RQ_PSN) {
+ qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN;
+ qp->qplib_qp.rq.psn = qp_attr->rq_psn;
+ }
+ if (qp_attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
+ qp->qplib_qp.modify_flags |=
+ CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC;
+ /* Cap the max_rd_atomic to device max */
+ if (qp_attr->max_rd_atomic > dev_attr->max_qp_rd_atom)
+ dev_dbg(rdev_to_dev(rdev),
+ "max_rd_atomic requested %d is > device max %d\n",
+ qp_attr->max_rd_atomic,
+ dev_attr->max_qp_rd_atom);
+ qp->qplib_qp.max_rd_atomic = min_t(u32, qp_attr->max_rd_atomic,
+ dev_attr->max_qp_rd_atom);
+ }
+ if (qp_attr_mask & IB_QP_SQ_PSN) {
+ qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN;
+ qp->qplib_qp.sq.psn = qp_attr->sq_psn;
+ }
+ if (qp_attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
+ if (qp_attr->max_dest_rd_atomic >
+ dev_attr->max_qp_init_rd_atom) {
+ dev_err(rdev_to_dev(rdev),
+ "max_dest_rd_atomic requested %d is > device max %d\n",
+ qp_attr->max_dest_rd_atomic,
+ dev_attr->max_qp_init_rd_atom);
+ return -EINVAL;
+ }
+ qp->qplib_qp.modify_flags |=
+ CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC;
+ qp->qplib_qp.max_dest_rd_atomic = qp_attr->max_dest_rd_atomic;
+ }
+ if (qp_attr_mask & IB_QP_CAP) {
+ qp->qplib_qp.modify_flags |=
+ CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE |
+ CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE |
+ CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE |
+ CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE |
+ CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA;
+ if ((qp_attr->cap.max_send_wr >= dev_attr->max_qp_wqes) ||
+ (qp_attr->cap.max_recv_wr >= dev_attr->max_qp_wqes) ||
+ (qp_attr->cap.max_send_sge >= dev_attr->max_qp_sges) ||
+ (qp_attr->cap.max_recv_sge >= dev_attr->max_qp_sges) ||
+ (qp_attr->cap.max_inline_data >=
+ dev_attr->max_inline_data)) {
+ dev_err(rdev_to_dev(rdev),
+ "Create QP failed - max exceeded\n");
+ return -EINVAL;
+ }
+ entries = roundup_pow_of_two(qp_attr->cap.max_send_wr);
+ if (entries > dev_attr->max_qp_wqes)
+ entries = dev_attr->max_qp_wqes;
+ entries = min_t(u32, entries, dev_attr->max_qp_wqes);
+ qp->qplib_qp.sq.max_wqe = entries;
+ qp->qplib_qp.sq.q_full_delta = qp->qplib_qp.sq.max_wqe -
+ qp_attr->cap.max_send_wr;
+ /*
+ * Reserving one slot for Phantom WQE. Some application can
+ * post one extra entry in this case. Allowing this to avoid
+ * unexpected Queue full condition
+ */
+ qp->qplib_qp.sq.q_full_delta -= 1;
+ qp->qplib_qp.sq.max_sge = qp_attr->cap.max_send_sge;
+ if (qp->qplib_qp.rq.max_wqe) {
+ entries = roundup_pow_of_two(qp_attr->cap.max_recv_wr);
+ if (entries > dev_attr->max_qp_wqes)
+ entries = dev_attr->max_qp_wqes;
+ qp->qplib_qp.rq.max_wqe = entries;
+ qp->qplib_qp.rq.q_full_delta = qp->qplib_qp.rq.max_wqe -
+ qp_attr->cap.max_recv_wr;
+ qp->qplib_qp.rq.max_sge = qp_attr->cap.max_recv_sge;
+ } else {
+ /* SRQ was used prior, just ignore the RQ caps */
+ }
+ }
+ if (qp_attr_mask & IB_QP_DEST_QPN) {
+ qp->qplib_qp.modify_flags |=
+ CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID;
+ qp->qplib_qp.dest_qpn = qp_attr->dest_qp_num;
+ }
+
+ rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "Modify HW QP failed!\n");
+ return rc;
+ }
+ if (qp_attr_mask & IB_QP_STATE)
+ bnxt_qplib_manage_flush_qp(qp);
+ if (ureq.comp_mask & BNXT_RE_COMP_MASK_MQP_EX_PPP_REQ_EN_MASK &&
+ ppp->st_idx_en & CREQ_MODIFY_QP_RESP_PINGPONG_PUSH_ENABLED) {
+ resp.comp_mask |= BNXT_RE_COMP_MASK_MQP_EX_PPP_REQ_EN;
+ resp.ppp_st_idx = ppp->st_idx_en >>
+ BNXT_QPLIB_PPP_ST_IDX_SHIFT;
+ is_copy_to_udata = true;
+ }
+
+ if (is_copy_to_udata) {
+ rc = bnxt_re_copy_to_udata(rdev, &resp,
+ min(udata->outlen, sizeof(resp)),
+ udata);
+ if (rc)
+ return rc;
+ }
+
+ if (ib_qp->qp_type == IB_QPT_GSI &&
+ rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL &&
+ rdev->gsi_ctx.gsi_sqp)
+ rc = bnxt_re_modify_shadow_qp(rdev, qp, qp_attr_mask);
+ /*
+ * Update info when qp_info_info
+ */
+ bnxt_re_update_qp_info(rdev, qp);
+ return rc;
+}
+
+int bnxt_re_query_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
+ int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
+{
+ struct bnxt_re_qp *qp = to_bnxt_re(ib_qp, struct bnxt_re_qp, ib_qp);
+ struct bnxt_re_dev *rdev = qp->rdev;
+ struct bnxt_qplib_qp *qplib_qp;
+ int rc;
+
+ qplib_qp = kcalloc(1, sizeof(*qplib_qp), GFP_KERNEL);
+ if (!qplib_qp)
+ return -ENOMEM;
+
+ qplib_qp->id = qp->qplib_qp.id;
+ qplib_qp->ah.host_sgid_index = qp->qplib_qp.ah.host_sgid_index;
+
+ rc = bnxt_qplib_query_qp(&rdev->qplib_res, qplib_qp);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "Query HW QP (0x%x) failed! rc = %d\n",
+ qplib_qp->id, rc);
+ goto free_mem;
+ }
+ qp_attr->qp_state = __to_ib_qp_state(qplib_qp->state);
+ qp_attr->cur_qp_state = __to_ib_qp_state(qplib_qp->cur_qp_state);
+ qp_attr->en_sqd_async_notify = qplib_qp->en_sqd_async_notify ? 1 : 0;
+ qp_attr->qp_access_flags = __to_ib_access_flags(qplib_qp->access);
+ qp_attr->pkey_index = qplib_qp->pkey_index;
+ qp_attr->qkey = qplib_qp->qkey;
+ memcpy(qp_attr->ah_attr.grh.dgid.raw, qplib_qp->ah.dgid.data,
+ sizeof(qplib_qp->ah.dgid.data));
+ qp_attr->ah_attr.grh.flow_label = qplib_qp->ah.flow_label;
+ qp_attr->ah_attr.grh.sgid_index = qplib_qp->ah.host_sgid_index;
+ qp_attr->ah_attr.grh.hop_limit = qplib_qp->ah.hop_limit;
+ qp_attr->ah_attr.grh.traffic_class = qplib_qp->ah.traffic_class;
+ qp_attr->ah_attr.sl = qplib_qp->ah.sl;
+ ether_addr_copy(ROCE_DMAC(&qp_attr->ah_attr), qplib_qp->ah.dmac);
+ qp_attr->path_mtu = __to_ib_mtu(qplib_qp->path_mtu);
+ qp_attr->timeout = qplib_qp->timeout;
+ qp_attr->retry_cnt = qplib_qp->retry_cnt;
+ qp_attr->rnr_retry = qplib_qp->rnr_retry;
+ qp_attr->min_rnr_timer = qplib_qp->min_rnr_timer;
+ qp_attr->rq_psn = qplib_qp->rq.psn;
+ qp_attr->max_rd_atomic = qplib_qp->max_rd_atomic;
+ qp_attr->sq_psn = qplib_qp->sq.psn;
+ qp_attr->max_dest_rd_atomic = qplib_qp->max_dest_rd_atomic;
+ qp_init_attr->sq_sig_type = qplib_qp->sig_type ? IB_SIGNAL_ALL_WR :
+ IB_SIGNAL_REQ_WR;
+ qp_attr->dest_qp_num = qplib_qp->dest_qpn;
+
+ qp_attr->cap.max_send_wr = qp->qplib_qp.sq.max_wqe;
+ qp_attr->cap.max_send_sge = qp->qplib_qp.sq.max_sge;
+ qp_attr->cap.max_recv_wr = qp->qplib_qp.rq.max_wqe;
+ qp_attr->cap.max_recv_sge = qp->qplib_qp.rq.max_sge;
+ qp_attr->cap.max_inline_data = qp->qplib_qp.max_inline_data;
+ qp_init_attr->cap = qp_attr->cap;
+
+free_mem:
+ kfree(qplib_qp);
+ return rc;
+}
+
+/* Builders */
+
+/* For Raw, the application is responsible to build the entire packet */
+static void bnxt_re_build_raw_send(const struct ib_send_wr *wr,
+ struct bnxt_qplib_swqe *wqe)
+{
+ switch (wr->send_flags) {
+ case IB_SEND_IP_CSUM:
+ wqe->rawqp1.lflags |= SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM;
+ break;
+ default:
+ /* Pad HW RoCE iCRC */
+ wqe->rawqp1.lflags |= SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC;
+ break;
+ }
+}
+
+/* For QP1, the driver must build the entire RoCE (v1/v2) packet hdr
+ * as according to the sgid and AV
+ */
+static int bnxt_re_build_qp1_send(struct bnxt_re_qp *qp, const struct ib_send_wr *wr,
+ struct bnxt_qplib_swqe *wqe, int payload_size)
+{
+ struct bnxt_re_ah *ah = to_bnxt_re(ud_wr(wr)->ah, struct bnxt_re_ah,
+ ibah);
+ struct bnxt_qplib_ah *qplib_ah = &ah->qplib_ah;
+ struct bnxt_qplib_sge sge;
+ int i, rc = 0;
+ union ib_gid sgid;
+ u16 vlan_id;
+ u8 *ptmac;
+ void *buf;
+
+ memset(&qp->qp1_hdr, 0, sizeof(qp->qp1_hdr));
+
+ /* Get sgid */
+ rc = bnxt_re_query_gid(&qp->rdev->ibdev, 1, qplib_ah->sgid_index, &sgid);
+ if (rc)
+ return rc;
+
+ /* ETH */
+ qp->qp1_hdr.eth_present = 1;
+ ptmac = ah->qplib_ah.dmac;
+ memcpy(qp->qp1_hdr.eth.dmac_h, ptmac, 4);
+ ptmac += 4;
+ memcpy(qp->qp1_hdr.eth.dmac_l, ptmac, 2);
+
+ ptmac = qp->qplib_qp.smac;
+ memcpy(qp->qp1_hdr.eth.smac_h, ptmac, 2);
+ ptmac += 2;
+ memcpy(qp->qp1_hdr.eth.smac_l, ptmac, 4);
+
+ qp->qp1_hdr.eth.type = cpu_to_be16(BNXT_QPLIB_ETHTYPE_ROCEV1);
+
+ /* For vlan, check the sgid for vlan existence */
+ vlan_id = rdma_get_vlan_id(&sgid);
+ if (vlan_id && vlan_id < 0x1000) {
+ qp->qp1_hdr.vlan_present = 1;
+ qp->qp1_hdr.eth.type = cpu_to_be16(ETH_P_8021Q);
+ }
+ /* GRH */
+ qp->qp1_hdr.grh_present = 1;
+ qp->qp1_hdr.grh.ip_version = 6;
+ qp->qp1_hdr.grh.payload_length =
+ cpu_to_be16((IB_BTH_BYTES + IB_DETH_BYTES + payload_size + 7)
+ & ~3);
+ qp->qp1_hdr.grh.next_header = 0x1b;
+ memcpy(qp->qp1_hdr.grh.source_gid.raw, sgid.raw, sizeof(sgid));
+ memcpy(qp->qp1_hdr.grh.destination_gid.raw, qplib_ah->dgid.data,
+ sizeof(sgid));
+
+ /* BTH */
+ if (wr->opcode == IB_WR_SEND_WITH_IMM) {
+ qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE;
+ qp->qp1_hdr.immediate_present = 1;
+ } else {
+ qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY;
+ }
+ if (wr->send_flags & IB_SEND_SOLICITED)
+ qp->qp1_hdr.bth.solicited_event = 1;
+ qp->qp1_hdr.bth.pad_count = (4 - payload_size) & 3;
+ /* P_key for QP1 is for all members */
+ qp->qp1_hdr.bth.pkey = cpu_to_be16(0xFFFF);
+ qp->qp1_hdr.bth.destination_qpn = IB_QP1;
+ qp->qp1_hdr.bth.ack_req = 0;
+ qp->send_psn++;
+ qp->send_psn &= BTH_PSN_MASK;
+ qp->qp1_hdr.bth.psn = cpu_to_be32(qp->send_psn);
+ /* DETH */
+ /* Use the priviledged Q_Key for QP1 */
+ qp->qp1_hdr.deth.qkey = cpu_to_be32(IB_QP1_QKEY);
+ qp->qp1_hdr.deth.source_qpn = IB_QP1;
+
+ /* Pack the QP1 to the transmit buffer */
+ buf = bnxt_qplib_get_qp1_sq_buf(&qp->qplib_qp, &sge);
+ if (!buf) {
+ dev_err(rdev_to_dev(qp->rdev), "QP1 buffer is empty!\n");
+ rc = -ENOMEM;
+ }
+ for (i = wqe->num_sge; i; i--) {
+ wqe->sg_list[i].addr = wqe->sg_list[i - 1].addr;
+ wqe->sg_list[i].lkey = wqe->sg_list[i - 1].lkey;
+ wqe->sg_list[i].size = wqe->sg_list[i - 1].size;
+ }
+ wqe->sg_list[0].addr = sge.addr;
+ wqe->sg_list[0].lkey = sge.lkey;
+ wqe->sg_list[0].size = sge.size;
+ wqe->num_sge++;
+
+ return rc;
+}
+
+static int bnxt_re_build_gsi_send(struct bnxt_re_qp *qp,
+ const struct ib_send_wr *wr,
+ struct bnxt_qplib_swqe *wqe)
+{
+ struct bnxt_re_dev *rdev;
+ int rc, indx, len = 0;
+
+ rdev = qp->rdev;
+
+ /* Mode UD is applicable to Gen P5 only */
+ if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_UD)
+ return 0;
+
+ for (indx = 0; indx < wr->num_sge; indx++) {
+ wqe->sg_list[indx].addr = wr->sg_list[indx].addr;
+ wqe->sg_list[indx].lkey = wr->sg_list[indx].lkey;
+ wqe->sg_list[indx].size = wr->sg_list[indx].length;
+ len += wr->sg_list[indx].length;
+ }
+ rc = bnxt_re_build_qp1_send(qp, wr, wqe, len);
+ wqe->rawqp1.lflags |= SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC;
+
+ return rc;
+}
+
+/* For the MAD layer, it only provides the recv SGE the size of
+ ib_grh + MAD datagram. No Ethernet headers, Ethertype, BTH, DETH,
+ nor RoCE iCRC. The Cu+ solution must provide buffer for the entire
+ receive packet (334 bytes) with no VLAN and then copy the GRH
+ and the MAD datagram out to the provided SGE.
+*/
+
+static int bnxt_re_build_qp1_recv(struct bnxt_re_qp *qp,
+ const struct ib_recv_wr *wr,
+ struct bnxt_qplib_swqe *wqe)
+{
+ struct bnxt_re_dev *rdev = qp->rdev;
+ struct bnxt_qplib_sge ref, sge;
+ u8 udp_hdr_size = 0;
+ u8 ip_hdr_size = 0;
+ int rc = 0;
+ int size;
+
+ if (bnxt_qplib_get_qp1_rq_buf(&qp->qplib_qp, &sge)) {
+ /* Create 5 SGEs as according to the following:
+ * Ethernet header (14)
+ * ib_grh (40) - as provided from the wr
+ * ib_bth + ib_deth + UDP(RoCE v2 only) (28)
+ * MAD (256) - as provided from the wr
+ * iCRC (4)
+ */
+
+ /* Set RoCE v2 header size and offsets */
+ if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ROCE_V2_IPV4)
+ ip_hdr_size = 20;
+ if (rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_ROCE_V1)
+ udp_hdr_size = 8;
+
+ /* Save the reference from ULP */
+ ref.addr = wr->sg_list[0].addr;
+ ref.lkey = wr->sg_list[0].lkey;
+ ref.size = wr->sg_list[0].length;
+
+ /* SGE 1 */
+ size = sge.size;
+ wqe->sg_list[0].addr = sge.addr;
+ wqe->sg_list[0].lkey = sge.lkey;
+ wqe->sg_list[0].size = BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE;
+ size -= wqe->sg_list[0].size;
+ if (size <= 0) {
+ dev_err(rdev_to_dev(qp->rdev),"QP1 rq buffer is empty!\n");
+ rc = -ENOMEM;
+ goto done;
+ }
+ sge.size = (u32)size;
+ sge.addr += wqe->sg_list[0].size;
+
+ /* SGE 2 */
+ /* In case of RoCE v2 ipv4 lower 20 bytes should have IP hdr */
+ wqe->sg_list[1].addr = ref.addr + ip_hdr_size;
+ wqe->sg_list[1].lkey = ref.lkey;
+ wqe->sg_list[1].size = sizeof(struct ib_grh) - ip_hdr_size;
+ ref.size -= wqe->sg_list[1].size;
+ if (ref.size <= 0) {
+ dev_err(rdev_to_dev(qp->rdev),
+ "QP1 ref buffer is empty!\n");
+ rc = -ENOMEM;
+ goto done;
+ }
+ ref.addr += wqe->sg_list[1].size + ip_hdr_size;
+
+ /* SGE 3 */
+ wqe->sg_list[2].addr = sge.addr;
+ wqe->sg_list[2].lkey = sge.lkey;
+ wqe->sg_list[2].size = BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE +
+ udp_hdr_size;
+ size -= wqe->sg_list[2].size;
+ if (size <= 0) {
+ dev_err(rdev_to_dev(qp->rdev),
+ "QP1 rq buffer is empty!\n");
+ rc = -ENOMEM;
+ goto done;
+ }
+ sge.size = (u32)size;
+ sge.addr += wqe->sg_list[2].size;
+
+ /* SGE 4 */
+ wqe->sg_list[3].addr = ref.addr;
+ wqe->sg_list[3].lkey = ref.lkey;
+ wqe->sg_list[3].size = ref.size;
+ ref.size -= wqe->sg_list[3].size;
+ if (ref.size) {
+ dev_err(rdev_to_dev(qp->rdev),
+ "QP1 ref buffer is incorrect!\n");
+ rc = -ENOMEM;
+ goto done;
+ }
+ /* SGE 5 */
+ wqe->sg_list[4].addr = sge.addr;
+ wqe->sg_list[4].lkey = sge.lkey;
+ wqe->sg_list[4].size = sge.size;
+ size -= wqe->sg_list[4].size;
+ if (size) {
+ dev_err(rdev_to_dev(qp->rdev),
+ "QP1 rq buffer is incorrect!\n");
+ rc = -ENOMEM;
+ goto done;
+ }
+ sge.size = (u32)size;
+ wqe->num_sge = 5;
+ } else {
+ dev_err(rdev_to_dev(qp->rdev), "QP1 buffer is empty!\n");
+ rc = -ENOMEM;
+ }
+done:
+ return rc;
+}
+
+static int bnxt_re_build_qp1_shadow_qp_recv(struct bnxt_re_qp *qp,
+ const struct ib_recv_wr *wr,
+ struct bnxt_qplib_swqe *wqe)
+{
+ struct bnxt_re_sqp_entries *sqp_entry;
+ struct bnxt_qplib_sge sge;
+ struct bnxt_re_dev *rdev;
+ u32 rq_prod_index;
+ int rc = 0;
+
+ rdev = qp->rdev;
+
+ rq_prod_index = bnxt_qplib_get_rq_prod_index(&qp->qplib_qp);
+
+ if (bnxt_qplib_get_qp1_rq_buf(&qp->qplib_qp, &sge)) {
+ /* Create 1 SGE to receive the entire
+ * ethernet packet
+ */
+ /* SGE 1 */
+ wqe->sg_list[0].addr = sge.addr;
+ /* TODO check the lkey to be used */
+ wqe->sg_list[0].lkey = sge.lkey;
+ wqe->sg_list[0].size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2;
+ if (sge.size < wqe->sg_list[0].size) {
+ dev_err(rdev_to_dev(qp->rdev),
+ "QP1 rq buffer is empty!\n");
+ rc = -ENOMEM;
+ goto done;
+ }
+
+ sqp_entry = &rdev->gsi_ctx.sqp_tbl[rq_prod_index];
+ sqp_entry->sge.addr = wr->sg_list[0].addr;
+ sqp_entry->sge.lkey = wr->sg_list[0].lkey;
+ sqp_entry->sge.size = wr->sg_list[0].length;
+ /* Store the wrid for reporting completion */
+ sqp_entry->wrid = wqe->wr_id;
+ /* change the wqe->wrid to table index */
+ wqe->wr_id = rq_prod_index;
+ }
+done:
+ return rc;
+}
+
+static bool is_ud_qp(struct bnxt_re_qp *qp)
+{
+ return (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD ||
+ qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_GSI);
+}
+
+static int bnxt_re_build_send_wqe(struct bnxt_re_qp *qp,
+ const struct ib_send_wr *wr,
+ struct bnxt_qplib_swqe *wqe)
+{
+ struct bnxt_re_ah *ah = NULL;
+
+ if(is_ud_qp(qp)) {
+ ah = to_bnxt_re(ud_wr(wr)->ah, struct bnxt_re_ah, ibah);
+ wqe->send.q_key = ud_wr(wr)->remote_qkey;
+ wqe->send.dst_qp = ud_wr(wr)->remote_qpn;
+ wqe->send.avid = ah->qplib_ah.id;
+ }
+ switch (wr->opcode) {
+ case IB_WR_SEND:
+ wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND;
+ break;
+ case IB_WR_SEND_WITH_IMM:
+ wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM;
+ wqe->send.imm_data = wr->ex.imm_data;
+ break;
+ case IB_WR_SEND_WITH_INV:
+ wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV;
+ wqe->send.inv_key = wr->ex.invalidate_rkey;
+ break;
+ default:
+ dev_err(rdev_to_dev(qp->rdev), "%s Invalid opcode %d!\n",
+ __func__, wr->opcode);
+ return -EINVAL;
+ }
+ if (wr->send_flags & IB_SEND_SIGNALED)
+ wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
+ if (wr->send_flags & IB_SEND_FENCE)
+ wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
+ if (wr->send_flags & IB_SEND_SOLICITED)
+ wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
+ if (wr->send_flags & IB_SEND_INLINE)
+ wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE;
+
+ return 0;
+}
+
+static int bnxt_re_build_rdma_wqe(const struct ib_send_wr *wr,
+ struct bnxt_qplib_swqe *wqe)
+{
+ switch (wr->opcode) {
+ case IB_WR_RDMA_WRITE:
+ wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE;
+ break;
+ case IB_WR_RDMA_WRITE_WITH_IMM:
+ wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM;
+ wqe->rdma.imm_data = wr->ex.imm_data;
+ break;
+ case IB_WR_RDMA_READ:
+ wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_READ;
+ wqe->rdma.inv_key = wr->ex.invalidate_rkey;
+ break;
+ default:
+ return -EINVAL;
+ }
+ wqe->rdma.remote_va = rdma_wr(wr)->remote_addr;
+ wqe->rdma.r_key = rdma_wr(wr)->rkey;
+ if (wr->send_flags & IB_SEND_SIGNALED)
+ wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
+ if (wr->send_flags & IB_SEND_FENCE)
+ wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
+ if (wr->send_flags & IB_SEND_SOLICITED)
+ wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
+ if (wr->send_flags & IB_SEND_INLINE)
+ wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE;
+
+ return 0;
+}
+
+static int bnxt_re_build_atomic_wqe(const struct ib_send_wr *wr,
+ struct bnxt_qplib_swqe *wqe)
+{
+ switch (wr->opcode) {
+ case IB_WR_ATOMIC_CMP_AND_SWP:
+ wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP;
+ wqe->atomic.cmp_data = atomic_wr(wr)->compare_add;
+ wqe->atomic.swap_data = atomic_wr(wr)->swap;
+ break;
+ case IB_WR_ATOMIC_FETCH_AND_ADD:
+ wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD;
+ wqe->atomic.cmp_data = atomic_wr(wr)->compare_add;
+ break;
+ default:
+ return -EINVAL;
+ }
+ wqe->atomic.remote_va = atomic_wr(wr)->remote_addr;
+ wqe->atomic.r_key = atomic_wr(wr)->rkey;
+ if (wr->send_flags & IB_SEND_SIGNALED)
+ wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
+ if (wr->send_flags & IB_SEND_FENCE)
+ wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
+ if (wr->send_flags & IB_SEND_SOLICITED)
+ wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
+ return 0;
+}
+
+static int bnxt_re_build_inv_wqe(const struct ib_send_wr *wr,
+ struct bnxt_qplib_swqe *wqe)
+{
+ wqe->type = BNXT_QPLIB_SWQE_TYPE_LOCAL_INV;
+ wqe->local_inv.inv_l_key = wr->ex.invalidate_rkey;
+ if (wr->send_flags & IB_SEND_SIGNALED)
+ wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
+ if (wr->send_flags & IB_SEND_FENCE)
+ wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
+ if (wr->send_flags & IB_SEND_SOLICITED)
+ wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
+
+ return 0;
+}
+
+static int bnxt_re_build_reg_wqe(const struct ib_reg_wr *wr,
+ struct bnxt_qplib_swqe *wqe)
+{
+ struct bnxt_re_mr *mr = to_bnxt_re(wr->mr, struct bnxt_re_mr, ib_mr);
+ struct bnxt_qplib_frpl *qplib_frpl = &mr->qplib_frpl;
+ int reg_len, i, access = wr->access;
+
+ if (mr->npages > qplib_frpl->max_pg_ptrs) {
+ dev_err_ratelimited(rdev_to_dev(mr->rdev),
+ " %s: failed npages %d > %d\n", __func__,
+ mr->npages, qplib_frpl->max_pg_ptrs);
+ return -EINVAL;
+ }
+
+ wqe->frmr.pbl_ptr = (__le64 *)qplib_frpl->hwq.pbl_ptr[0];
+ wqe->frmr.pbl_dma_ptr = qplib_frpl->hwq.pbl_dma_ptr[0];
+ wqe->frmr.levels = qplib_frpl->hwq.level;
+ wqe->frmr.page_list = mr->pages;
+ wqe->frmr.page_list_len = mr->npages;
+ wqe->type = BNXT_QPLIB_SWQE_TYPE_REG_MR;
+
+ if (wr->wr.send_flags & IB_SEND_SIGNALED)
+ wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
+ if (access & IB_ACCESS_LOCAL_WRITE)
+ wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE;
+ if (access & IB_ACCESS_REMOTE_READ)
+ wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ;
+ if (access & IB_ACCESS_REMOTE_WRITE)
+ wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE;
+ if (access & IB_ACCESS_REMOTE_ATOMIC)
+ wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC;
+ if (access & IB_ACCESS_MW_BIND)
+ wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND;
+
+ /* TODO: OFED provides the rkey of the MR instead of the lkey */
+ wqe->frmr.l_key = wr->key;
+ wqe->frmr.length = wr->mr->length;
+ wqe->frmr.pbl_pg_sz_log = ilog2(PAGE_SIZE >> PAGE_SHIFT_4K);
+ wqe->frmr.pg_sz_log = ilog2(wr->mr->page_size >> PAGE_SHIFT_4K);
+ wqe->frmr.va = wr->mr->iova;
+ reg_len = wqe->frmr.page_list_len * wr->mr->page_size;
+
+ if (wqe->frmr.length > reg_len) {
+ dev_err_ratelimited(rdev_to_dev(mr->rdev),
+ "%s: bnxt_re_mr 0x%px len (%d > %d)\n",
+ __func__, (void *)mr, wqe->frmr.length,
+ reg_len);
+
+ for (i = 0; i < mr->npages; i++)
+ dev_dbg(rdev_to_dev(mr->rdev),
+ "%s: build_reg_wqe page[%d] = 0x%llx\n",
+ __func__, i, mr->pages[i]);
+
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static void bnxt_re_set_sg_list(const struct ib_send_wr *wr,
+ struct bnxt_qplib_swqe *wqe)
+{
+ wqe->sg_list = (struct bnxt_qplib_sge *)wr->sg_list;
+ wqe->num_sge = wr->num_sge;
+}
+
+static void bnxt_ud_qp_hw_stall_workaround(struct bnxt_re_qp *qp)
+{
+ if ((qp->ib_qp.qp_type == IB_QPT_UD || qp->ib_qp.qp_type == IB_QPT_GSI ||
+ qp->ib_qp.qp_type == IB_QPT_RAW_ETHERTYPE) &&
+ qp->qplib_qp.wqe_cnt == BNXT_RE_UD_QP_HW_STALL) {
+ int qp_attr_mask;
+ struct ib_qp_attr qp_attr;
+
+ qp_attr_mask = IB_QP_STATE;
+ qp_attr.qp_state = IB_QPS_RTS;
+ bnxt_re_modify_qp(&qp->ib_qp, &qp_attr, qp_attr_mask, NULL);
+ qp->qplib_qp.wqe_cnt = 0;
+ }
+}
+
+static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev,
+ struct bnxt_re_qp *qp,
+ const struct ib_send_wr *wr)
+{
+ struct bnxt_qplib_swqe wqe;
+ unsigned long flags;
+ int rc = 0;
+
+ spin_lock_irqsave(&qp->sq_lock, flags);
+ while (wr) {
+ /* House keeping */
+ memset(&wqe, 0, sizeof(wqe));
+ /* Common */
+ if (wr->num_sge > qp->qplib_qp.sq.max_sge) {
+ dev_err(rdev_to_dev(rdev),
+ "Limit exceeded for Send SGEs\n");
+ rc = -EINVAL;
+ break;
+ }
+
+ bnxt_re_set_sg_list(wr, &wqe);
+ wqe.wr_id = wr->wr_id;
+ wqe.type = BNXT_QPLIB_SWQE_TYPE_SEND;
+ rc = bnxt_re_build_send_wqe(qp, wr, &wqe);
+ if (rc)
+ break;
+
+ rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev),
+ "bad_wr seen with opcode = 0x%x rc = %d\n",
+ wr->opcode, rc);
+ break;
+ }
+ wr = wr->next;
+ }
+ bnxt_qplib_post_send_db(&qp->qplib_qp);
+ bnxt_ud_qp_hw_stall_workaround(qp);
+ spin_unlock_irqrestore(&qp->sq_lock, flags);
+ return rc;
+}
+
+static void bnxt_re_legacy_set_uc_fence(struct bnxt_qplib_swqe *wqe)
+{
+ /* Need unconditional fence for non-wire memory opcode
+ * to work as expected.
+ */
+ if (wqe->type == BNXT_QPLIB_SWQE_TYPE_LOCAL_INV ||
+ wqe->type == BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR ||
+ wqe->type == BNXT_QPLIB_SWQE_TYPE_REG_MR ||
+ wqe->type == BNXT_QPLIB_SWQE_TYPE_BIND_MW)
+ wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
+}
+
+int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr,
+ const struct ib_send_wr **bad_wr)
+{
+ struct bnxt_re_qp *qp = to_bnxt_re(ib_qp, struct bnxt_re_qp, ib_qp);
+ struct bnxt_qplib_sge sge[6];
+ struct bnxt_qplib_swqe wqe;
+ struct bnxt_re_dev *rdev;
+ unsigned long flags;
+ int rc = 0;
+
+ rdev = qp->rdev;
+ spin_lock_irqsave(&qp->sq_lock, flags);
+ while (wr) {
+ /* House keeping */
+ memset(&wqe, 0, sizeof(wqe));
+ /* Common */
+ if (wr->num_sge > qp->qplib_qp.sq.max_sge) {
+ dev_err(rdev_to_dev(rdev),
+ "Limit exceeded for Send SGEs\n");
+ rc = -EINVAL;
+ goto bad;
+ }
+
+ bnxt_re_set_sg_list(wr, &wqe);
+ wqe.wr_id = wr->wr_id;
+
+ switch (wr->opcode) {
+ case IB_WR_SEND:
+ case IB_WR_SEND_WITH_IMM:
+ if (ib_qp->qp_type == IB_QPT_GSI &&
+ rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) {
+ memset(sge, 0, sizeof(sge));
+ wqe.sg_list = sge;
+ rc = bnxt_re_build_gsi_send(qp, wr, &wqe);
+ if (rc)
+ goto bad;
+ } else if (ib_qp->qp_type == IB_QPT_RAW_ETHERTYPE) {
+ bnxt_re_build_raw_send(wr, &wqe);
+ }
+ switch (wr->send_flags) {
+ case IB_SEND_IP_CSUM:
+ wqe.rawqp1.lflags |=
+ SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM;
+ break;
+ default:
+ break;
+ }
+ fallthrough;
+ case IB_WR_SEND_WITH_INV:
+ rc = bnxt_re_build_send_wqe(qp, wr, &wqe);
+ break;
+ case IB_WR_RDMA_WRITE:
+ case IB_WR_RDMA_WRITE_WITH_IMM:
+ case IB_WR_RDMA_READ:
+ rc = bnxt_re_build_rdma_wqe(wr, &wqe);
+ break;
+ case IB_WR_ATOMIC_CMP_AND_SWP:
+ case IB_WR_ATOMIC_FETCH_AND_ADD:
+ rc = bnxt_re_build_atomic_wqe(wr, &wqe);
+ break;
+ case IB_WR_RDMA_READ_WITH_INV:
+ dev_err(rdev_to_dev(rdev),
+ "RDMA Read with Invalidate is not supported\n");
+ rc = -EINVAL;
+ goto bad;
+ case IB_WR_LOCAL_INV:
+ rc = bnxt_re_build_inv_wqe(wr, &wqe);
+ break;
+ case IB_WR_REG_MR:
+ rc = bnxt_re_build_reg_wqe(reg_wr(wr), &wqe);
+ break;
+ default:
+ /* Unsupported WRs */
+ dev_err(rdev_to_dev(rdev),
+ "WR (0x%x) is not supported\n", wr->opcode);
+ rc = -EINVAL;
+ goto bad;
+ }
+
+ if (likely(!rc)) {
+ if (!_is_chip_gen_p5_p7(rdev->chip_ctx))
+ bnxt_re_legacy_set_uc_fence(&wqe);
+ rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
+ }
+bad:
+ if (unlikely(rc)) {
+ dev_err(rdev_to_dev(rdev),
+ "bad_wr seen with opcode = 0x%x\n", wr->opcode);
+ *bad_wr = wr;
+ break;
+ }
+ wr = wr->next;
+ }
+ bnxt_qplib_post_send_db(&qp->qplib_qp);
+ if (!_is_chip_gen_p5_p7(rdev->chip_ctx))
+ bnxt_ud_qp_hw_stall_workaround(qp);
+ spin_unlock_irqrestore(&qp->sq_lock, flags);
+
+ return rc;
+}
+
+static int bnxt_re_post_recv_shadow_qp(struct bnxt_re_dev *rdev,
+ struct bnxt_re_qp *qp,
+ struct ib_recv_wr *wr)
+{
+ struct bnxt_qplib_swqe wqe;
+ int rc = 0;
+
+ /* rq lock can be pardoned here. */
+ while (wr) {
+ /* House keeping */
+ memset(&wqe, 0, sizeof(wqe));
+ /* Common */
+ if (wr->num_sge > qp->qplib_qp.rq.max_sge) {
+ dev_err(rdev_to_dev(rdev),
+ "Limit exceeded for Receive SGEs\n");
+ rc = -EINVAL;
+ goto bad;
+ }
+
+ wqe.sg_list = (struct bnxt_qplib_sge *)wr->sg_list;
+ wqe.num_sge = wr->num_sge;
+ wqe.wr_id = wr->wr_id;
+ wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
+ rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe);
+bad:
+ if (rc) {
+ dev_err(rdev_to_dev(rdev),
+ "bad_wr seen with RQ post\n");
+ break;
+ }
+ wr = wr->next;
+ }
+ bnxt_qplib_post_recv_db(&qp->qplib_qp);
+ return rc;
+}
+
+static int bnxt_re_build_gsi_recv(struct bnxt_re_qp *qp,
+ const struct ib_recv_wr *wr,
+ struct bnxt_qplib_swqe *wqe)
+{
+ struct bnxt_re_dev *rdev = qp->rdev;
+ int rc = 0;
+
+ if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL)
+ rc = bnxt_re_build_qp1_shadow_qp_recv(qp, wr, wqe);
+ else
+ rc = bnxt_re_build_qp1_recv(qp, wr, wqe);
+
+ return rc;
+}
+
+int bnxt_re_post_recv(struct ib_qp *ib_qp, const struct ib_recv_wr *wr,
+ const struct ib_recv_wr **bad_wr)
+{
+ struct bnxt_re_qp *qp = to_bnxt_re(ib_qp, struct bnxt_re_qp, ib_qp);
+ struct bnxt_qplib_sge sge[6];
+ struct bnxt_qplib_swqe wqe;
+ unsigned long flags;
+ u32 count = 0;
+ int rc = 0;
+
+ spin_lock_irqsave(&qp->rq_lock, flags);
+ while (wr) {
+ memset(&wqe, 0, sizeof(wqe));
+ if (wr->num_sge > qp->qplib_qp.rq.max_sge) {
+ dev_err(rdev_to_dev(qp->rdev),
+ "Limit exceeded for Receive SGEs\n");
+ rc = -EINVAL;
+ goto bad;
+ }
+ wqe.num_sge = wr->num_sge;
+ wqe.sg_list = (struct bnxt_qplib_sge *)wr->sg_list;
+ wqe.wr_id = wr->wr_id;
+ wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
+
+ if (ib_qp->qp_type == IB_QPT_GSI &&
+ qp->rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) {
+ memset(sge, 0, sizeof(sge));
+ wqe.sg_list = sge;
+ rc = bnxt_re_build_gsi_recv(qp, wr, &wqe);
+ if (rc)
+ goto bad;
+ }
+ rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe);
+bad:
+ if (rc) {
+ dev_err(rdev_to_dev(qp->rdev),
+ "bad_wr seen with RQ post\n");
+ *bad_wr = wr;
+ break;
+ }
+ /* Ring DB if the RQEs posted reaches a threshold value */
+ if (++count >= BNXT_RE_RQ_WQE_THRESHOLD) {
+ bnxt_qplib_post_recv_db(&qp->qplib_qp);
+ count = 0;
+ }
+ wr = wr->next;
+ }
+
+ if (count)
+ bnxt_qplib_post_recv_db(&qp->qplib_qp);
+ spin_unlock_irqrestore(&qp->rq_lock, flags);
+
+ return rc;
+}
+
+/* Completion Queues */
+void bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
+{
+ struct bnxt_re_cq *cq = to_bnxt_re(ib_cq, struct bnxt_re_cq, ibcq);
+ struct bnxt_re_dev *rdev = cq->rdev;
+ int rc = 0;
+
+ if (cq->uctx_cq_page) {
+ BNXT_RE_CQ_PAGE_LIST_DEL(cq->uctx, cq);
+ free_page((u64)cq->uctx_cq_page);
+ cq->uctx_cq_page = NULL;
+ }
+ if (cq->is_dbr_soft_cq && cq->uctx) {
+ void *dbr_page;
+
+ if (cq->uctx->dbr_recov_cq) {
+ dbr_page = cq->uctx->dbr_recov_cq_page;
+ cq->uctx->dbr_recov_cq_page = NULL;
+ cq->uctx->dbr_recov_cq = NULL;
+ free_page((unsigned long)dbr_page);
+ }
+ goto end;
+ }
+ /* CQ getting destroyed. Set this state for cqn handler */
+ spin_lock_bh(&cq->qplib_cq.compl_lock);
+ cq->qplib_cq.destroyed = true;
+ spin_unlock_bh(&cq->qplib_cq.compl_lock);
+ if (ib_cq->poll_ctx == IB_POLL_WORKQUEUE ||
+ ib_cq->poll_ctx == IB_POLL_UNBOUND_WORKQUEUE)
+ cancel_work_sync(&ib_cq->work);
+
+ rc = bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq);
+ if (rc)
+ dev_err_ratelimited(rdev_to_dev(rdev),
+ "%s id = %d failed rc = %d\n",
+ __func__, cq->qplib_cq.id, rc);
+
+ bnxt_re_put_nq(rdev, cq->qplib_cq.nq);
+ if (cq->umem && !IS_ERR(cq->umem))
+ ib_umem_release(cq->umem);
+
+ kfree(cq->cql);
+ atomic_dec(&rdev->stats.rsors.cq_count);
+end:
+ return;
+}
+
+static inline struct
+bnxt_re_cq *__get_cq_from_cq_in(struct ib_cq *cq_in,
+ struct bnxt_re_dev *rdev)
+{
+ struct bnxt_re_cq *cq;
+ cq = container_of(cq_in, struct bnxt_re_cq, ibcq);
+ return cq;
+}
+
+int bnxt_re_create_cq(struct ib_cq *cq_in,
+ const struct ib_cq_init_attr *attr,
+ struct ib_udata *udata)
+{
+ struct bnxt_qplib_dev_attr *dev_attr;
+ struct bnxt_re_ucontext *uctx = NULL;
+ struct ib_ucontext *context = NULL;
+ struct bnxt_qplib_cq *qplcq;
+ struct bnxt_re_cq_req ureq;
+ struct bnxt_re_dev *rdev;
+ int rc, entries;
+ struct bnxt_re_cq *cq;
+ u32 max_active_cqs;
+ int cqe = attr->cqe;
+
+ if (attr->flags)
+ return -EOPNOTSUPP;
+
+ rdev = rdev_from_cq_in(cq_in);
+ if (rdev->mod_exit) {
+ rc = -EIO;
+ dev_dbg(rdev_to_dev(rdev), "%s(): in mod_exit, just return!\n", __func__);
+ goto exit;
+ }
+ if (udata) {
+ uctx = rdma_udata_to_drv_context(udata,
+ struct bnxt_re_ucontext,
+ ibucontext);
+ context = &uctx->ibucontext;
+ }
+ dev_attr = rdev->dev_attr;
+
+ if (atomic_read(&rdev->stats.rsors.cq_count) >= dev_attr->max_cq) {
+ dev_err(rdev_to_dev(rdev), "Create CQ failed - max exceeded(CQs)\n");
+ rc = -EINVAL;
+ goto exit;
+ }
+ /* Validate CQ fields */
+ if (cqe < 1 || cqe > dev_attr->max_cq_wqes) {
+ dev_err(rdev_to_dev(rdev), "Create CQ failed - max exceeded(CQ_WQs)\n");
+ rc = -EINVAL;
+ goto exit;
+ }
+
+ cq = __get_cq_from_cq_in(cq_in, rdev);
+ if (!cq) {
+ rc = -ENOMEM;
+ goto exit;
+ }
+ cq->rdev = rdev;
+ cq->uctx = uctx;
+ qplcq = &cq->qplib_cq;
+ qplcq->cq_handle = (u64)qplcq;
+ /*
+ * Since CQ is for QP1 is shared with Shadow CQ, the size
+ * should be double the size. There is no way to identify
+ * whether this CQ is for GSI QP. So assuming that the first
+ * CQ created is for QP1
+ */
+ if (!udata && !rdev->gsi_ctx.first_cq_created &&
+ rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL) {
+ rdev->gsi_ctx.first_cq_created = true;
+ /*
+ * Total CQE required for the CQ = CQE for QP1 RQ +
+ * CQE for Shadow QP SQEs + CQE for Shadow QP RQEs.
+ * Max entries of shadow QP SQ and RQ = QP1 RQEs = cqe
+ */
+ cqe *= 3;
+ }
+
+ entries = bnxt_re_init_depth(cqe + 1, uctx);
+ if (entries > dev_attr->max_cq_wqes + 1)
+ entries = dev_attr->max_cq_wqes + 1;
+
+ qplcq->sginfo.pgshft = PAGE_SHIFT;
+ qplcq->sginfo.pgsize = PAGE_SIZE;
+ if (udata) {
+ if (udata->inlen < sizeof(ureq))
+ dev_warn(rdev_to_dev(rdev),
+ "Update the library ulen %d klen %d\n",
+ (unsigned int)udata->inlen,
+ (unsigned int)sizeof(ureq));
+
+ rc = ib_copy_from_udata(&ureq, udata,
+ min(udata->inlen, sizeof(ureq)));
+ if (rc)
+ goto fail;
+
+ if (BNXT_RE_IS_DBR_PACING_NOTIFY_CQ(ureq)) {
+ cq->is_dbr_soft_cq = true;
+ goto success;
+ }
+
+ if (BNXT_RE_IS_DBR_RECOV_CQ(ureq)) {
+ void *dbr_page;
+ u32 *epoch;
+
+ dbr_page = (void *)__get_free_page(GFP_KERNEL);
+ if (!dbr_page) {
+ dev_err(rdev_to_dev(rdev),
+ "DBR recov CQ page allocation failed!");
+ rc = -ENOMEM;
+ goto fail;
+ }
+
+ /* memset the epoch and epoch_ack to 0 */
+ epoch = dbr_page;
+ epoch[0] = 0x0;
+ epoch[1] = 0x0;
+
+ uctx->dbr_recov_cq = cq;
+ uctx->dbr_recov_cq_page = dbr_page;
+
+ cq->is_dbr_soft_cq = true;
+ goto success;
+ }
+
+ cq->umem = ib_umem_get_compat
+ (rdev, context, udata, ureq.cq_va,
+ entries * sizeof(struct cq_base),
+ IB_ACCESS_LOCAL_WRITE, 1);
+ if (IS_ERR(cq->umem)) {
+ rc = PTR_ERR(cq->umem);
+ dev_err(rdev_to_dev(rdev),
+ "%s: ib_umem_get failed! rc = %d\n",
+ __func__, rc);
+ goto fail;
+ }
+ qplcq->sginfo.sghead = get_ib_umem_sgl(cq->umem,
+ &qplcq->sginfo.nmap);
+ qplcq->sginfo.npages = ib_umem_num_pages_compat(cq->umem);
+ if (!uctx->dpi.dbr) {
+ rc = bnxt_re_get_user_dpi(rdev, uctx);
+ if (rc)
+ goto c2fail;
+ }
+ qplcq->dpi = &uctx->dpi;
+ } else {
+ cq->max_cql = entries > MAX_CQL_PER_POLL ? MAX_CQL_PER_POLL : entries;
+ cq->cql = kcalloc(cq->max_cql, sizeof(struct bnxt_qplib_cqe),
+ GFP_KERNEL);
+ if (!cq->cql) {
+ dev_err(rdev_to_dev(rdev),
+ "Allocate CQL for %d failed!\n", cq->max_cql);
+ rc = -ENOMEM;
+ goto fail;
+ }
+ qplcq->dpi = &rdev->dpi_privileged;
+ }
+ /*
+ * Allocating the NQ in a round robin fashion. nq_alloc_cnt is a
+ * used for getting the NQ index.
+ */
+ qplcq->max_wqe = entries;
+ qplcq->nq = bnxt_re_get_nq(rdev);
+ qplcq->cnq_hw_ring_id = qplcq->nq->ring_id;
+
+ rc = bnxt_qplib_create_cq(&rdev->qplib_res, qplcq);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "Create HW CQ failed!\n");
+ goto fail;
+ }
+
+ INIT_LIST_HEAD(&cq->cq_list);
+ cq->ibcq.cqe = entries;
+ cq->cq_period = qplcq->period;
+
+ atomic_inc(&rdev->stats.rsors.cq_count);
+ max_active_cqs = atomic_read(&rdev->stats.rsors.cq_count);
+ if (max_active_cqs > atomic_read(&rdev->stats.rsors.max_cq_count))
+ atomic_set(&rdev->stats.rsors.max_cq_count, max_active_cqs);
+ spin_lock_init(&cq->cq_lock);
+
+ if (udata) {
+ struct bnxt_re_cq_resp resp;
+
+ resp.cqid = qplcq->id;
+ resp.tail = qplcq->hwq.cons;
+ resp.phase = qplcq->period;
+ resp.comp_mask = 0;
+ resp.dbr = (u64)uctx->dpi.umdbr;
+ resp.dpi = uctx->dpi.dpi;
+ resp.comp_mask |= BNXT_RE_COMP_MASK_CQ_HAS_DB_INFO;
+ /* Copy only on a valid wcpdi */
+ if (uctx->wcdpi.dpi) {
+ resp.wcdpi = uctx->wcdpi.dpi;
+ resp.comp_mask |= BNXT_RE_COMP_MASK_CQ_HAS_WC_DPI;
+ }
+
+ if (_is_chip_p7(rdev->chip_ctx)) {
+ cq->uctx_cq_page = (void *)__get_free_page(GFP_KERNEL);
+
+ if (!cq->uctx_cq_page) {
+ dev_err(rdev_to_dev(rdev),
+ "CQ page allocation failed!\n");
+ bnxt_qplib_destroy_cq(&rdev->qplib_res, qplcq);
+ rc = -ENOMEM;
+ goto c2fail;
+ }
+
+ resp.uctx_cq_page = (u64)cq->uctx_cq_page;
+ resp.comp_mask |= BNXT_RE_COMP_MASK_CQ_HAS_CQ_PAGE;
+ }
+
+ rc = bnxt_re_copy_to_udata(rdev, &resp,
+ min(udata->outlen, sizeof(resp)),
+ udata);
+ if (rc) {
+ free_page((u64)cq->uctx_cq_page);
+ cq->uctx_cq_page = NULL;
+ bnxt_qplib_destroy_cq(&rdev->qplib_res, qplcq);
+ goto c2fail;
+ }
+
+ if (cq->uctx_cq_page)
+ BNXT_RE_CQ_PAGE_LIST_ADD(uctx, cq);
+ }
+
+success:
+ return 0;
+c2fail:
+ if (udata && cq->umem && !IS_ERR(cq->umem))
+ ib_umem_release(cq->umem);
+fail:
+ if (cq) {
+ if (cq->cql)
+ kfree(cq->cql);
+ }
+exit:
+ return rc;
+}
+
+int bnxt_re_modify_cq(struct ib_cq *ib_cq, u16 cq_count, u16 cq_period)
+{
+ struct bnxt_re_cq *cq = to_bnxt_re(ib_cq, struct bnxt_re_cq, ibcq);
+ struct bnxt_re_dev *rdev = cq->rdev;
+ int rc;
+
+ if ((cq->cq_count != cq_count) || (cq->cq_period != cq_period)) {
+ cq->qplib_cq.count = cq_count;
+ cq->qplib_cq.period = cq_period;
+ rc = bnxt_qplib_modify_cq(&rdev->qplib_res, &cq->qplib_cq);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "Modify HW CQ %#x failed!\n",
+ cq->qplib_cq.id);
+ return rc;
+ }
+ /* On success, update the shadow */
+ cq->cq_count = cq_count;
+ cq->cq_period = cq_period;
+ }
+ return 0;
+}
+
+static void bnxt_re_resize_cq_complete(struct bnxt_re_cq *cq)
+{
+ struct bnxt_re_dev *rdev = cq->rdev;
+
+ bnxt_qplib_resize_cq_complete(&rdev->qplib_res, &cq->qplib_cq);
+
+ cq->qplib_cq.max_wqe = cq->resize_cqe;
+ if (cq->resize_umem) {
+ ib_umem_release(cq->umem);
+ cq->umem = cq->resize_umem;
+ cq->resize_umem = NULL;
+ cq->resize_cqe = 0;
+ }
+}
+
+int bnxt_re_resize_cq(struct ib_cq *ib_cq, int cqe, struct ib_udata *udata)
+{
+ struct bnxt_qplib_sg_info sginfo = {};
+ struct bnxt_qplib_dpi *orig_dpi = NULL;
+ struct bnxt_qplib_dev_attr *dev_attr;
+ struct bnxt_re_ucontext *uctx = NULL;
+ struct bnxt_re_resize_cq_req ureq;
+ struct ib_ucontext *context = NULL;
+ struct bnxt_re_dev *rdev;
+ struct bnxt_re_cq *cq;
+ int rc, entries;
+
+ /* Don't allow more than one resize request at the same time.
+ * TODO: need a mutex here when we support kernel consumers of resize.
+ */
+ cq = to_bnxt_re(ib_cq, struct bnxt_re_cq, ibcq);
+ rdev = cq->rdev;
+ dev_attr = rdev->dev_attr;
+ if (ib_cq->uobject) {
+ uctx = rdma_udata_to_drv_context(udata,
+ struct bnxt_re_ucontext,
+ ibucontext);
+ context = &uctx->ibucontext;
+ }
+
+ if (cq->resize_umem) {
+ dev_err(rdev_to_dev(rdev), "Resize CQ %#x failed - Busy\n",
+ cq->qplib_cq.id);
+ return -EBUSY;
+ }
+
+ /* Check the requested cq depth out of supported depth */
+ if (cqe < 1 || cqe > dev_attr->max_cq_wqes) {
+ dev_err(rdev_to_dev(rdev), "Resize CQ %#x failed - max exceeded\n",
+ cq->qplib_cq.id);
+ return -EINVAL;
+ }
+
+ entries = bnxt_re_init_depth(cqe + 1, uctx);
+ entries = min_t(u32, (u32)entries, dev_attr->max_cq_wqes + 1);
+
+ /* Check to see if the new requested size can be handled by already
+ * existing CQ
+ */
+ if (entries == cq->ibcq.cqe) {
+ dev_info(rdev_to_dev(rdev), "CQ is already at size %d\n", cqe);
+ return 0;
+ }
+
+ if (ib_cq->uobject && udata) {
+ if (udata->inlen < sizeof(ureq))
+ dev_warn(rdev_to_dev(rdev),
+ "Update the library ulen %d klen %d\n",
+ (unsigned int)udata->inlen,
+ (unsigned int)sizeof(ureq));
+
+ rc = ib_copy_from_udata(&ureq, udata,
+ min(udata->inlen, sizeof(ureq)));
+ if (rc)
+ goto fail;
+
+ dev_dbg(rdev_to_dev(rdev), "%s: va %p\n", __func__,
+ (void *)ureq.cq_va);
+ cq->resize_umem = ib_umem_get_compat
+ (rdev,
+ context, udata, ureq.cq_va,
+ entries * sizeof(struct cq_base),
+ IB_ACCESS_LOCAL_WRITE, 1);
+ if (IS_ERR(cq->resize_umem)) {
+ rc = PTR_ERR(cq->resize_umem);
+ cq->resize_umem = NULL;
+ dev_err(rdev_to_dev(rdev), "%s: ib_umem_get failed! rc = %d\n",
+ __func__, rc);
+ goto fail;
+ }
+ cq->resize_cqe = entries;
+ dev_dbg(rdev_to_dev(rdev), "%s: ib_umem_get() success\n",
+ __func__);
+ memcpy(&sginfo, &cq->qplib_cq.sginfo, sizeof(sginfo));
+ orig_dpi = cq->qplib_cq.dpi;
+
+ cq->qplib_cq.sginfo.sghead = get_ib_umem_sgl(cq->resize_umem,
+ &cq->qplib_cq.sginfo.nmap);
+ cq->qplib_cq.sginfo.npages =
+ ib_umem_num_pages_compat(cq->resize_umem);
+ cq->qplib_cq.sginfo.pgsize = PAGE_SIZE;
+ cq->qplib_cq.sginfo.pgshft = PAGE_SHIFT;
+ cq->qplib_cq.dpi = &uctx->dpi;
+ } else {
+ /* TODO: kernel consumer */
+ }
+
+ rc = bnxt_qplib_resize_cq(&rdev->qplib_res, &cq->qplib_cq, entries);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "Resize HW CQ %#x failed!\n",
+ cq->qplib_cq.id);
+ goto fail;
+ }
+
+ cq->ibcq.cqe = cq->resize_cqe;
+ /* For kernel consumers complete resize here. For uverbs consumers,
+ * we complete it in the context of ibv_poll_cq().
+ */
+ if (!cq->resize_umem)
+ bnxt_qplib_resize_cq_complete(&rdev->qplib_res, &cq->qplib_cq);
+
+ atomic_inc(&rdev->stats.rsors.resize_count);
+ return 0;
+
+fail:
+ if (cq->resize_umem) {
+ ib_umem_release(cq->resize_umem);
+ cq->resize_umem = NULL;
+ cq->resize_cqe = 0;
+ memcpy(&cq->qplib_cq.sginfo, &sginfo, sizeof(sginfo));
+ cq->qplib_cq.dpi = orig_dpi;
+ }
+ return rc;
+}
+
+static enum ib_wc_status __req_to_ib_wc_status(u8 qstatus)
+{
+ switch(qstatus) {
+ case CQ_REQ_STATUS_OK:
+ return IB_WC_SUCCESS;
+ case CQ_REQ_STATUS_BAD_RESPONSE_ERR:
+ return IB_WC_BAD_RESP_ERR;
+ case CQ_REQ_STATUS_LOCAL_LENGTH_ERR:
+ return IB_WC_LOC_LEN_ERR;
+ case CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR:
+ return IB_WC_LOC_QP_OP_ERR;
+ case CQ_REQ_STATUS_LOCAL_PROTECTION_ERR:
+ return IB_WC_LOC_PROT_ERR;
+ case CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR:
+ return IB_WC_GENERAL_ERR;
+ case CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR:
+ return IB_WC_REM_INV_REQ_ERR;
+ case CQ_REQ_STATUS_REMOTE_ACCESS_ERR:
+ return IB_WC_REM_ACCESS_ERR;
+ case CQ_REQ_STATUS_REMOTE_OPERATION_ERR:
+ return IB_WC_REM_OP_ERR;
+ case CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR:
+ return IB_WC_RNR_RETRY_EXC_ERR;
+ case CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR:
+ return IB_WC_RETRY_EXC_ERR;
+ case CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR:
+ return IB_WC_WR_FLUSH_ERR;
+ default:
+ return IB_WC_GENERAL_ERR;
+ }
+ return 0;
+}
+
+static enum ib_wc_status __rawqp1_to_ib_wc_status(u8 qstatus)
+{
+ switch(qstatus) {
+ case CQ_RES_RAWETH_QP1_STATUS_OK:
+ return IB_WC_SUCCESS;
+ case CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR:
+ return IB_WC_LOC_ACCESS_ERR;
+ case CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR:
+ return IB_WC_LOC_LEN_ERR;
+ case CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR:
+ return IB_WC_LOC_PROT_ERR;
+ case CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR:
+ return IB_WC_LOC_QP_OP_ERR;
+ case CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR:
+ return IB_WC_GENERAL_ERR;
+ case CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR:
+ return IB_WC_WR_FLUSH_ERR;
+ case CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR:
+ return IB_WC_WR_FLUSH_ERR;
+ default:
+ return IB_WC_GENERAL_ERR;
+ }
+}
+
+static enum ib_wc_status __rc_to_ib_wc_status(u8 qstatus)
+{
+ switch(qstatus) {
+ case CQ_RES_RC_STATUS_OK:
+ return IB_WC_SUCCESS;
+ case CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR:
+ return IB_WC_LOC_ACCESS_ERR;
+ case CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR:
+ return IB_WC_LOC_LEN_ERR;
+ case CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR:
+ return IB_WC_LOC_PROT_ERR;
+ case CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR:
+ return IB_WC_LOC_QP_OP_ERR;
+ case CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR:
+ return IB_WC_GENERAL_ERR;
+ case CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR:
+ return IB_WC_REM_INV_REQ_ERR;
+ case CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR:
+ return IB_WC_WR_FLUSH_ERR;
+ case CQ_RES_RC_STATUS_HW_FLUSH_ERR:
+ return IB_WC_WR_FLUSH_ERR;
+ default:
+ return IB_WC_GENERAL_ERR;
+ }
+}
+
+static void bnxt_re_process_req_wc(struct ib_wc *wc, struct bnxt_qplib_cqe *cqe)
+{
+ switch (cqe->type) {
+ case BNXT_QPLIB_SWQE_TYPE_SEND:
+ wc->opcode = IB_WC_SEND;
+ break;
+ case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM:
+ wc->opcode = IB_WC_SEND;
+ wc->wc_flags |= IB_WC_WITH_IMM;
+ break;
+ case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV:
+ wc->opcode = IB_WC_SEND;
+ wc->wc_flags |= IB_WC_WITH_INVALIDATE;
+ break;
+ case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE:
+ wc->opcode = IB_WC_RDMA_WRITE;
+ break;
+ case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM:
+ wc->opcode = IB_WC_RDMA_WRITE;
+ wc->wc_flags |= IB_WC_WITH_IMM;
+ break;
+ case BNXT_QPLIB_SWQE_TYPE_RDMA_READ:
+ wc->opcode = IB_WC_RDMA_READ;
+ break;
+ case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP:
+ wc->opcode = IB_WC_COMP_SWAP;
+ break;
+ case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD:
+ wc->opcode = IB_WC_FETCH_ADD;
+ break;
+ case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV:
+ wc->opcode = IB_WC_LOCAL_INV;
+ break;
+ case BNXT_QPLIB_SWQE_TYPE_REG_MR:
+ wc->opcode = IB_WC_REG_MR;
+ break;
+ default:
+ wc->opcode = IB_WC_SEND;
+ break;
+ }
+
+ wc->status = __req_to_ib_wc_status(cqe->status);
+}
+
+static int bnxt_re_check_packet_type(u16 raweth_qp1_flags, u16 raweth_qp1_flags2)
+{
+ bool is_ipv6 = false, is_ipv4 = false;
+
+ /* raweth_qp1_flags Bit 9-6 indicates itype */
+
+ if ((raweth_qp1_flags & CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE)
+ != CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE)
+ return -1;
+
+ if (raweth_qp1_flags2 &
+ CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC &&
+ raweth_qp1_flags2 &
+ CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_L4_CS_CALC) {
+ /* raweth_qp1_flags2 Bit 8 indicates ip_type. 0-v4 1 - v6 */
+ (raweth_qp1_flags2 &
+ CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_TYPE) ?
+ (is_ipv6 = true) : (is_ipv4 = true);
+ return ((is_ipv6) ?
+ BNXT_RE_ROCEV2_IPV6_PACKET :
+ BNXT_RE_ROCEV2_IPV4_PACKET);
+ } else {
+ return BNXT_RE_ROCE_V1_PACKET;
+ }
+}
+
+static bool bnxt_re_is_loopback_packet(struct bnxt_re_dev *rdev,
+ void *rq_hdr_buf)
+{
+ u8 *tmp_buf = NULL;
+ struct ethhdr *eth_hdr;
+ u16 eth_type;
+ bool rc = false;
+
+ tmp_buf = (u8 *)rq_hdr_buf;
+ /*
+ * If dest mac is not same as I/F mac, this could be a
+ * loopback address or multicast address, check whether
+ * it is a loopback packet
+ */
+ if (!ether_addr_equal(tmp_buf, rdev->dev_addr)) {
+ tmp_buf += 4;
+ /* Check the ether type */
+ eth_hdr = (struct ethhdr *)tmp_buf;
+ eth_type = ntohs(eth_hdr->h_proto);
+ switch (eth_type) {
+ case BNXT_QPLIB_ETHTYPE_ROCEV1:
+ rc = true;
+ break;
+ default:
+ break;
+ }
+ }
+
+ return rc;
+}
+
+static bool bnxt_re_is_vlan_in_packet(struct bnxt_re_dev *rdev,
+ void *rq_hdr_buf,
+ struct bnxt_qplib_cqe *cqe)
+{
+ struct vlan_hdr *vlan_hdr;
+ struct ethhdr *eth_hdr;
+ u8 *tmp_buf = NULL;
+ u16 eth_type;
+
+ tmp_buf = (u8 *)rq_hdr_buf;
+ /* Check the ether type */
+ eth_hdr = (struct ethhdr *)tmp_buf;
+ eth_type = ntohs(eth_hdr->h_proto);
+ if (eth_type == ETH_P_8021Q) {
+ tmp_buf += sizeof(struct ethhdr);
+ vlan_hdr = (struct vlan_hdr *)tmp_buf;
+ cqe->raweth_qp1_metadata =
+ ntohs(vlan_hdr->h_vlan_TCI) |
+ (eth_type <<
+ CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_SFT);
+ cqe->raweth_qp1_flags2 |=
+ CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN;
+ return true;
+ }
+
+ return false;
+}
+
+static int bnxt_re_process_raw_qp_packet_receive(struct bnxt_re_qp *gsi_qp,
+ struct bnxt_qplib_cqe *cqe)
+{
+ struct bnxt_re_sqp_entries *sqp_entry = NULL;
+ struct bnxt_qplib_hdrbuf *hdr_buf;
+ dma_addr_t shrq_hdr_buf_map;
+ struct ib_sge s_sge[2] = {};
+ struct ib_sge r_sge[2] = {};
+ struct ib_recv_wr rwr = {};
+ struct bnxt_re_ah *gsi_sah;
+ struct bnxt_re_qp *gsi_sqp;
+ dma_addr_t rq_hdr_buf_map;
+ struct bnxt_re_dev *rdev;
+ struct ib_send_wr *swr;
+ u32 skip_bytes = 0;
+ void *rq_hdr_buf;
+ int pkt_type = 0;
+ u32 offset = 0;
+ u32 tbl_idx;
+ int rc;
+ struct ib_ud_wr udwr = {};
+
+ swr = &udwr.wr;
+ rdev = gsi_qp->rdev;
+ gsi_sqp = rdev->gsi_ctx.gsi_sqp;
+ tbl_idx = cqe->wr_id;
+
+ hdr_buf = gsi_qp->qplib_qp.rq_hdr_buf;
+ rq_hdr_buf = (u8 *) hdr_buf->va + tbl_idx * hdr_buf->step;
+ rq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&gsi_qp->qplib_qp,
+ tbl_idx);
+ /* Shadow QP header buffer */
+ shrq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&gsi_sqp->qplib_qp,
+ tbl_idx);
+ sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx];
+
+ /* Find packet type from the cqe */
+ pkt_type = bnxt_re_check_packet_type(cqe->raweth_qp1_flags,
+ cqe->raweth_qp1_flags2);
+ if (pkt_type < 0) {
+ dev_err(rdev_to_dev(rdev), "Not handling this packet\n");
+ return -EINVAL;
+ }
+
+ /* Adjust the offset for the user buffer and post in the rq */
+
+ if (pkt_type == BNXT_RE_ROCEV2_IPV4_PACKET)
+ offset = 20;
+
+ /*
+ * QP1 loopback packet has 4 bytes of internal header before
+ * ether header. Skip these four bytes.
+ */
+ if (bnxt_re_is_loopback_packet(rdev, rq_hdr_buf))
+ skip_bytes = 4;
+
+ if (bnxt_re_is_vlan_in_packet(rdev, rq_hdr_buf, cqe))
+ skip_bytes += VLAN_HLEN;
+
+ /* Store this cqe */
+ memcpy(&sqp_entry->cqe, cqe, sizeof(struct bnxt_qplib_cqe));
+ sqp_entry->qp1_qp = gsi_qp;
+
+ /* First send SGE . Skip the ether header*/
+ s_sge[0].addr = rq_hdr_buf_map + BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE
+ + skip_bytes;
+ s_sge[0].lkey = 0xFFFFFFFF;
+ s_sge[0].length = offset ? BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 :
+ BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6;
+
+ /* Second Send SGE */
+ s_sge[1].addr = s_sge[0].addr + s_sge[0].length +
+ BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE;
+ if (pkt_type != BNXT_RE_ROCE_V1_PACKET)
+ s_sge[1].addr += 8;
+ s_sge[1].lkey = 0xFFFFFFFF;
+ s_sge[1].length = 256;
+
+ /* First recv SGE */
+ r_sge[0].addr = shrq_hdr_buf_map;
+ r_sge[0].lkey = 0xFFFFFFFF;
+ r_sge[0].length = 40;
+
+ r_sge[1].addr = sqp_entry->sge.addr + offset;
+ r_sge[1].lkey = sqp_entry->sge.lkey;
+ r_sge[1].length = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 + 256 - offset;
+
+ /* Create receive work request */
+ rwr.num_sge = 2;
+ rwr.sg_list = r_sge;
+ rwr.wr_id = tbl_idx;
+ rwr.next = NULL;
+
+ rc = bnxt_re_post_recv_shadow_qp(rdev, gsi_sqp, &rwr);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev),
+ "Failed to post Rx buffers to shadow QP\n");
+ return -ENOMEM;
+ }
+
+ swr->num_sge = 2;
+ swr->sg_list = s_sge;
+ swr->wr_id = tbl_idx;
+ swr->opcode = IB_WR_SEND;
+ swr->next = NULL;
+
+ gsi_sah = rdev->gsi_ctx.gsi_sah;
+ udwr.ah = &gsi_sah->ibah;
+ udwr.remote_qpn = gsi_sqp->qplib_qp.id;
+ udwr.remote_qkey = gsi_sqp->qplib_qp.qkey;
+ /* post data received in the send queue */
+ rc = bnxt_re_post_send_shadow_qp(rdev, gsi_sqp, swr);
+
+ return rc;
+}
+
+static void bnxt_re_process_res_rawqp1_wc(struct ib_wc *wc,
+ struct bnxt_qplib_cqe *cqe)
+{
+ wc->opcode = IB_WC_RECV;
+ wc->status = __rawqp1_to_ib_wc_status(cqe->status);
+ wc->wc_flags |= IB_WC_GRH;
+}
+
+static void bnxt_re_process_res_rc_wc(struct ib_wc *wc,
+ struct bnxt_qplib_cqe *cqe)
+{
+ wc->opcode = IB_WC_RECV;
+ wc->status = __rc_to_ib_wc_status(cqe->status);
+
+ if (cqe->flags & CQ_RES_RC_FLAGS_IMM)
+ wc->wc_flags |= IB_WC_WITH_IMM;
+ if (cqe->flags & CQ_RES_RC_FLAGS_INV)
+ wc->wc_flags |= IB_WC_WITH_INVALIDATE;
+ if ((cqe->flags & (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) ==
+ (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM))
+ wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
+}
+
+/* Returns TRUE if pkt has valid VLAN and if VLAN id is non-zero */
+static bool bnxt_re_is_nonzero_vlanid_pkt(struct bnxt_qplib_cqe *orig_cqe,
+ u16 *vid, u8 *sl)
+{
+ u32 metadata;
+ u16 tpid;
+ bool ret = false;
+ metadata = orig_cqe->raweth_qp1_metadata;
+ if (orig_cqe->raweth_qp1_flags2 &
+ CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN) {
+ tpid = ((metadata &
+ CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_MASK) >>
+ CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_SFT);
+ if (tpid == ETH_P_8021Q) {
+ *vid = metadata &
+ CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_MASK;
+ *sl = (metadata &
+ CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_MASK) >>
+ CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_SFT;
+ ret = !!(*vid);
+ }
+ }
+
+ return ret;
+}
+
+static void bnxt_re_process_res_shadow_qp_wc(struct bnxt_re_qp *gsi_sqp,
+ struct ib_wc *wc,
+ struct bnxt_qplib_cqe *cqe)
+{
+ u32 tbl_idx;
+ struct bnxt_re_dev *rdev = gsi_sqp->rdev;
+ struct bnxt_re_qp *gsi_qp = NULL;
+ struct bnxt_qplib_cqe *orig_cqe = NULL;
+ struct bnxt_re_sqp_entries *sqp_entry = NULL;
+ int nw_type;
+ u16 vlan_id;
+ u8 sl;
+
+ tbl_idx = cqe->wr_id;
+
+ sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx];
+ gsi_qp = sqp_entry->qp1_qp;
+ orig_cqe = &sqp_entry->cqe;
+
+ wc->wr_id = sqp_entry->wrid;
+ wc->byte_len = orig_cqe->length;
+ wc->qp = &gsi_qp->ib_qp;
+
+ wc->ex.imm_data = orig_cqe->immdata;
+ wc->src_qp = orig_cqe->src_qp;
+ memcpy(wc->smac, orig_cqe->smac, ETH_ALEN);
+ if (bnxt_re_is_nonzero_vlanid_pkt(orig_cqe, &vlan_id, &sl)) {
+ if (bnxt_re_check_if_vlan_valid(rdev, vlan_id)) {
+ wc->sl = sl;
+ wc->vlan_id = vlan_id;
+ wc->wc_flags |= IB_WC_WITH_VLAN;
+ }
+ }
+ wc->port_num = 1;
+ wc->vendor_err = orig_cqe->status;
+
+ wc->opcode = IB_WC_RECV;
+ wc->status = __rawqp1_to_ib_wc_status(orig_cqe->status);
+ wc->wc_flags |= IB_WC_GRH;
+
+ nw_type = bnxt_re_check_packet_type(orig_cqe->raweth_qp1_flags,
+ orig_cqe->raweth_qp1_flags2);
+ if(nw_type >= 0)
+ dev_dbg(rdev_to_dev(rdev), "%s nw_type = %d\n", __func__, nw_type);
+}
+
+static void bnxt_re_process_res_ud_wc(struct bnxt_re_dev *rdev,
+ struct bnxt_re_qp *qp, struct ib_wc *wc,
+ struct bnxt_qplib_cqe *cqe)
+{
+ u16 vlan_id = 0;
+
+ wc->opcode = IB_WC_RECV;
+ wc->status = __rc_to_ib_wc_status(cqe->status);
+ if (cqe->flags & CQ_RES_UD_FLAGS_IMM)
+ wc->wc_flags |= IB_WC_WITH_IMM;
+ if (cqe->flags & CQ_RES_RC_FLAGS_INV)
+ wc->wc_flags |= IB_WC_WITH_INVALIDATE;
+ /* report only on GSI QP for Thor */
+ if (rdev->gsi_ctx.gsi_qp->qplib_qp.id == qp->qplib_qp.id &&
+ rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_UD) {
+ wc->wc_flags |= IB_WC_GRH;
+ memcpy(wc->smac, cqe->smac, ETH_ALEN);
+ wc->wc_flags |= IB_WC_WITH_SMAC;
+ if (_is_cqe_v2_supported(rdev->dev_attr->dev_cap_flags)) {
+ if (cqe->flags & CQ_RES_UD_V2_FLAGS_META_FORMAT_MASK) {
+ if (cqe->cfa_meta &
+ BNXT_QPLIB_CQE_CFA_META1_VALID)
+ vlan_id = (cqe->cfa_meta & 0xFFF);
+ }
+ } else if (cqe->flags & CQ_RES_UD_FLAGS_META_FORMAT_VLAN) {
+ vlan_id = (cqe->cfa_meta & 0xFFF);
+ }
+ /* Mark only if vlan_id is non zero */
+ if (vlan_id && bnxt_re_check_if_vlan_valid(rdev, vlan_id)) {
+ wc->vlan_id = vlan_id;
+ wc->wc_flags |= IB_WC_WITH_VLAN;
+ }
+ }
+}
+
+static int bnxt_re_legacy_send_phantom_wqe(struct bnxt_re_qp *qp)
+{
+ struct bnxt_qplib_qp *lib_qp = &qp->qplib_qp;
+ unsigned long flags;
+ int rc = 0;
+
+ spin_lock_irqsave(&qp->sq_lock, flags);
+
+ rc = bnxt_re_legacy_bind_fence_mw(lib_qp);
+ if (!rc) {
+ lib_qp->sq.phantom_wqe_cnt++;
+ dev_dbg(&lib_qp->sq.hwq.pdev->dev,
+ "qp %#x sq->prod %#x sw_prod %#x phantom_wqe_cnt %d\n",
+ lib_qp->id, lib_qp->sq.hwq.prod,
+ HWQ_CMP(lib_qp->sq.hwq.prod, &lib_qp->sq.hwq),
+ lib_qp->sq.phantom_wqe_cnt);
+ }
+
+ spin_unlock_irqrestore(&qp->sq_lock, flags);
+ return rc;
+}
+
+int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc)
+{
+ struct bnxt_re_cq *cq = to_bnxt_re(ib_cq, struct bnxt_re_cq, ibcq);
+ struct bnxt_re_dev *rdev = cq->rdev;
+ struct bnxt_re_qp *qp;
+ struct bnxt_qplib_cqe *cqe;
+ int i, ncqe, budget, init_budget;
+ struct bnxt_qplib_q *sq;
+ struct bnxt_qplib_qp *lib_qp;
+ u32 tbl_idx;
+ struct bnxt_re_sqp_entries *sqp_entry = NULL;
+ unsigned long flags;
+ u8 gsi_mode;
+
+ /*
+ * DB recovery CQ; only process the door bell pacing alert from
+ * the user lib
+ */
+ if (cq->is_dbr_soft_cq) {
+ bnxt_re_pacing_alert(rdev);
+ return 0;
+ }
+
+ /* User CQ; the only processing we do is to
+ * complete any pending CQ resize operation.
+ */
+ if (cq->umem) {
+ if (cq->resize_umem)
+ bnxt_re_resize_cq_complete(cq);
+ return 0;
+ }
+
+ spin_lock_irqsave(&cq->cq_lock, flags);
+
+ budget = min_t(u32, num_entries, cq->max_cql);
+ init_budget = budget;
+ if (!cq->cql) {
+ dev_err(rdev_to_dev(rdev), "POLL CQ no CQL to use\n");
+ goto exit;
+ }
+ cqe = &cq->cql[0];
+ gsi_mode = rdev->gsi_ctx.gsi_qp_mode;
+ while (budget) {
+ lib_qp = NULL;
+ ncqe = bnxt_qplib_poll_cq(&cq->qplib_cq, cqe, budget, &lib_qp);
+ if (lib_qp) {
+ sq = &lib_qp->sq;
+ if (sq->legacy_send_phantom == true) {
+ qp = container_of(lib_qp, struct bnxt_re_qp, qplib_qp);
+ if (bnxt_re_legacy_send_phantom_wqe(qp) == -ENOMEM)
+ dev_err(rdev_to_dev(rdev),
+ "Phantom failed! Scheduled to send again\n");
+ else
+ sq->legacy_send_phantom = false;
+ }
+ }
+ if (ncqe < budget)
+ ncqe += bnxt_qplib_process_flush_list(&cq->qplib_cq,
+ cqe + ncqe,
+ budget - ncqe);
+
+ if (!ncqe)
+ break;
+
+ for (i = 0; i < ncqe; i++, cqe++) {
+ /* Transcribe each qplib_wqe back to ib_wc */
+ memset(wc, 0, sizeof(*wc));
+
+ wc->wr_id = cqe->wr_id;
+ wc->byte_len = cqe->length;
+ qp = to_bnxt_re((struct bnxt_qplib_qp *)cqe->qp_handle,
+ struct bnxt_re_qp, qplib_qp);
+ if (!qp) {
+ dev_err(rdev_to_dev(rdev),
+ "POLL CQ bad QP handle\n");
+ continue;
+ }
+ wc->qp = &qp->ib_qp;
+ wc->ex.imm_data = cqe->immdata;
+ wc->src_qp = cqe->src_qp;
+ memcpy(wc->smac, cqe->smac, ETH_ALEN);
+ wc->port_num = 1;
+ wc->vendor_err = cqe->status;
+
+ switch(cqe->opcode) {
+ case CQ_BASE_CQE_TYPE_REQ:
+ if (gsi_mode == BNXT_RE_GSI_MODE_ALL &&
+ qp->qplib_qp.id ==
+ rdev->gsi_ctx.gsi_sqp->qplib_qp.id) {
+ /* Handle this completion with
+ * the stored completion */
+ dev_dbg(rdev_to_dev(rdev),
+ "Skipping this UD Send CQ\n");
+ memset(wc, 0, sizeof(*wc));
+ continue;
+ }
+ bnxt_re_process_req_wc(wc, cqe);
+ break;
+ case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
+ if (gsi_mode == BNXT_RE_GSI_MODE_ALL) {
+ if (!cqe->status) {
+ int rc = 0;
+ rc = bnxt_re_process_raw_qp_packet_receive(qp, cqe);
+ if (!rc) {
+ memset(wc, 0,
+ sizeof(*wc));
+ continue;
+ }
+ cqe->status = -1;
+ }
+ /* Errors need not be looped back.
+ * But change the wr_id to the one
+ * stored in the table
+ */
+ tbl_idx = cqe->wr_id;
+ sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx];
+ wc->wr_id = sqp_entry->wrid;
+ }
+
+ bnxt_re_process_res_rawqp1_wc(wc, cqe);
+ break;
+ case CQ_BASE_CQE_TYPE_RES_RC:
+ bnxt_re_process_res_rc_wc(wc, cqe);
+ break;
+ case CQ_BASE_CQE_TYPE_RES_UD:
+ if (gsi_mode == BNXT_RE_GSI_MODE_ALL &&
+ qp->qplib_qp.id ==
+ rdev->gsi_ctx.gsi_sqp->qplib_qp.id) {
+ /* Handle this completion with
+ * the stored completion
+ */
+ dev_dbg(rdev_to_dev(rdev),
+ "Handling the UD receive CQ\n");
+ if (cqe->status) {
+ /* TODO handle this completion as a failure in
+ * loopback porocedure
+ */
+ continue;
+ } else {
+ bnxt_re_process_res_shadow_qp_wc(qp, wc, cqe);
+ break;
+ }
+ }
+ bnxt_re_process_res_ud_wc(rdev, qp, wc, cqe);
+ break;
+ default:
+ dev_err(rdev_to_dev(cq->rdev),
+ "POLL CQ type 0x%x not handled, skip!\n",
+ cqe->opcode);
+ continue;
+ }
+ wc++;
+ budget--;
+ }
+ }
+exit:
+ spin_unlock_irqrestore(&cq->cq_lock, flags);
+ return init_budget - budget;
+}
+
+int bnxt_re_req_notify_cq(struct ib_cq *ib_cq,
+ enum ib_cq_notify_flags ib_cqn_flags)
+{
+ struct bnxt_re_cq *cq = to_bnxt_re(ib_cq, struct bnxt_re_cq, ibcq);
+ int type = 0, rc = 0;
+ unsigned long flags;
+
+ spin_lock_irqsave(&cq->cq_lock, flags);
+ /* Trigger on the very next completion */
+ if (ib_cqn_flags & IB_CQ_NEXT_COMP)
+ type = DBC_DBC_TYPE_CQ_ARMALL;
+ /* Trigger on the next solicited completion */
+ else if (ib_cqn_flags & IB_CQ_SOLICITED)
+ type = DBC_DBC_TYPE_CQ_ARMSE;
+
+ bnxt_qplib_req_notify_cq(&cq->qplib_cq, type);
+
+ /* Poll to see if there are missed events */
+ if ((ib_cqn_flags & IB_CQ_REPORT_MISSED_EVENTS) &&
+ !(bnxt_qplib_is_cq_empty(&cq->qplib_cq)))
+ rc = 1;
+
+ spin_unlock_irqrestore(&cq->cq_lock, flags);
+
+ return rc;
+}
+
+/* Memory Regions */
+struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags)
+{
+ struct bnxt_qplib_mrinfo mrinfo;
+ struct bnxt_re_dev *rdev;
+ struct bnxt_re_mr *mr;
+ struct bnxt_re_pd *pd;
+ u32 max_mr_count;
+ u64 pbl = 0;
+ int rc;
+
+ memset(&mrinfo, 0, sizeof(mrinfo));
+ pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd);
+ rdev = pd->rdev;
+
+ mr = kzalloc(sizeof(*mr), GFP_KERNEL);
+ if (!mr) {
+ dev_err(rdev_to_dev(rdev),
+ "Allocate memory for DMA MR failed!\n");
+ return ERR_PTR(-ENOMEM);
+ }
+ mr->rdev = rdev;
+ mr->qplib_mr.pd = &pd->qplib_pd;
+ mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags);
+ mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
+
+ /* Allocate and register 0 as the address */
+ rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "Allocate DMA MR failed!\n");
+ goto fail;
+ }
+ mr->qplib_mr.total_size = -1; /* Infinite length */
+ mrinfo.ptes = &pbl;
+ mrinfo.sg.npages = 0;
+ mrinfo.sg.pgsize = PAGE_SIZE;
+ mrinfo.sg.pgshft = PAGE_SHIFT;
+ mrinfo.sg.pgsize = PAGE_SIZE;
+ mrinfo.mrw = &mr->qplib_mr;
+ mrinfo.is_dma = true;
+ rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mrinfo, false);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "Register DMA MR failed!\n");
+ goto fail_mr;
+ }
+ mr->ib_mr.lkey = mr->qplib_mr.lkey;
+ if (mr_access_flags & (IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ |
+ IB_ACCESS_REMOTE_ATOMIC))
+ mr->ib_mr.rkey = mr->ib_mr.lkey;
+ atomic_inc(&rdev->stats.rsors.mr_count);
+ max_mr_count = atomic_read(&rdev->stats.rsors.mr_count);
+ if (max_mr_count > atomic_read(&rdev->stats.rsors.max_mr_count))
+ atomic_set(&rdev->stats.rsors.max_mr_count, max_mr_count);
+
+ return &mr->ib_mr;
+
+fail_mr:
+ bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
+fail:
+ kfree(mr);
+ return ERR_PTR(rc);
+}
+
+int bnxt_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
+{
+ struct bnxt_re_mr *mr = to_bnxt_re(ib_mr, struct bnxt_re_mr, ib_mr);
+ struct bnxt_re_dev *rdev = mr->rdev;
+ int rc = 0;
+
+ rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
+ if (rc)
+ dev_err(rdev_to_dev(rdev), "Dereg MR failed (%d): rc - %#x\n",
+ mr->qplib_mr.lkey, rc);
+
+ if (mr->pages) {
+ bnxt_qplib_free_fast_reg_page_list(&rdev->qplib_res,
+ &mr->qplib_frpl);
+ kfree(mr->pages);
+ mr->npages = 0;
+ mr->pages = NULL;
+ }
+ if (!IS_ERR(mr->ib_umem) && mr->ib_umem) {
+ mr->is_invalcb_active = false;
+ bnxt_re_peer_mem_release(mr->ib_umem);
+ }
+ kfree(mr);
+ atomic_dec(&rdev->stats.rsors.mr_count);
+ return 0;
+}
+
+static int bnxt_re_set_page(struct ib_mr *ib_mr, u64 addr)
+{
+ struct bnxt_re_mr *mr = to_bnxt_re(ib_mr, struct bnxt_re_mr, ib_mr);
+
+ if (unlikely(mr->npages == mr->qplib_frpl.max_pg_ptrs))
+ return -ENOMEM;
+
+ mr->pages[mr->npages++] = addr;
+ dev_dbg(NULL, "%s: ibdev %p Set MR pages[%d] = 0x%lx\n",
+ ROCE_DRV_MODULE_NAME, ib_mr->device, mr->npages - 1,
+ mr->pages[mr->npages - 1]);
+ return 0;
+}
+
+int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg,
+ int sg_nents, unsigned int *sg_offset)
+{
+ struct bnxt_re_mr *mr = to_bnxt_re(ib_mr, struct bnxt_re_mr, ib_mr);
+
+ mr->npages = 0;
+ return ib_sg_to_pages(ib_mr, sg, sg_nents,
+ sg_offset, bnxt_re_set_page);
+}
+
+struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type,
+ u32 max_num_sg, struct ib_udata *udata)
+{
+ struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd);
+ struct bnxt_re_dev *rdev = pd->rdev;
+ struct bnxt_re_mr *mr;
+ u32 max_mr_count;
+ int rc;
+
+ dev_dbg(rdev_to_dev(rdev), "Alloc MR\n");
+ if (type != IB_MR_TYPE_MEM_REG) {
+ dev_dbg(rdev_to_dev(rdev), "MR type 0x%x not supported\n", type);
+ return ERR_PTR(-EINVAL);
+ }
+ if (max_num_sg > MAX_PBL_LVL_1_PGS) {
+ dev_dbg(rdev_to_dev(rdev), "Max SG exceeded\n");
+ return ERR_PTR(-EINVAL);
+ }
+ mr = kzalloc(sizeof(*mr), GFP_KERNEL);
+ if (!mr) {
+ dev_err(rdev_to_dev(rdev), "Allocate MR mem failed!\n");
+ return ERR_PTR(-ENOMEM);
+ }
+ mr->rdev = rdev;
+ mr->qplib_mr.pd = &pd->qplib_pd;
+ mr->qplib_mr.flags = BNXT_QPLIB_FR_PMR;
+ mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
+
+ rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "Allocate MR failed!\n");
+ goto fail;
+ }
+ mr->ib_mr.lkey = mr->qplib_mr.lkey;
+ mr->ib_mr.rkey = mr->ib_mr.lkey;
+ mr->pages = kzalloc(sizeof(u64) * max_num_sg, GFP_KERNEL);
+ if (!mr->pages) {
+ dev_err(rdev_to_dev(rdev),
+ "Allocate MR page list mem failed!\n");
+ rc = -ENOMEM;
+ goto fail_mr;
+ }
+ rc = bnxt_qplib_alloc_fast_reg_page_list(&rdev->qplib_res,
+ &mr->qplib_frpl, max_num_sg);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev),
+ "Allocate HW Fast reg page list failed!\n");
+ goto free_page;
+ }
+ dev_dbg(rdev_to_dev(rdev), "Alloc MR pages = 0x%p\n", mr->pages);
+
+ atomic_inc(&rdev->stats.rsors.mr_count);
+ max_mr_count = atomic_read(&rdev->stats.rsors.mr_count);
+ if (max_mr_count > atomic_read(&rdev->stats.rsors.max_mr_count))
+ atomic_set(&rdev->stats.rsors.max_mr_count, max_mr_count);
+ return &mr->ib_mr;
+
+free_page:
+ kfree(mr->pages);
+fail_mr:
+ bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
+fail:
+ kfree(mr);
+ return ERR_PTR(rc);
+}
+
+/* Memory Windows */
+struct ib_mw *bnxt_re_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type,
+ struct ib_udata *udata)
+{
+ struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd);
+ struct bnxt_re_dev *rdev = pd->rdev;
+ struct bnxt_re_mw *mw;
+ u32 max_mw_count;
+ int rc;
+
+ mw = kzalloc(sizeof(*mw), GFP_KERNEL);
+ if (!mw) {
+ dev_err(rdev_to_dev(rdev), "Allocate MW failed!\n");
+ rc = -ENOMEM;
+ goto exit;
+ }
+ mw->rdev = rdev;
+ mw->qplib_mw.pd = &pd->qplib_pd;
+
+ mw->qplib_mw.type = (type == IB_MW_TYPE_1 ?
+ CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1 :
+ CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B);
+ rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mw->qplib_mw);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "Allocate MW failed!\n");
+ goto fail;
+ }
+ mw->ib_mw.rkey = mw->qplib_mw.rkey;
+ atomic_inc(&rdev->stats.rsors.mw_count);
+ max_mw_count = atomic_read(&rdev->stats.rsors.mw_count);
+ if (max_mw_count > atomic_read(&rdev->stats.rsors.max_mw_count))
+ atomic_set(&rdev->stats.rsors.max_mw_count, max_mw_count);
+
+ return &mw->ib_mw;
+fail:
+ kfree(mw);
+exit:
+ return ERR_PTR(rc);
+}
+
+int bnxt_re_dealloc_mw(struct ib_mw *ib_mw)
+{
+ struct bnxt_re_mw *mw = to_bnxt_re(ib_mw, struct bnxt_re_mw, ib_mw);
+ struct bnxt_re_dev *rdev = mw->rdev;
+ int rc;
+
+ rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mw->qplib_mw);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "Free MW failed: %#x\n", rc);
+ return rc;
+ }
+
+ kfree(mw);
+ atomic_dec(&rdev->stats.rsors.mw_count);
+ return rc;
+}
+
+static int bnxt_re_page_size_ok(int page_shift)
+{
+ switch (page_shift) {
+ case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_4K:
+ case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_8K:
+ case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_64K:
+ case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_2M:
+ case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_256K:
+ case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1M:
+ case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_4M:
+ case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_256MB:
+ case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1G:
+ return 1;
+ default:
+ return 0;
+ }
+}
+
+static int bnxt_re_get_page_shift(struct ib_umem *umem,
+ u64 va, u64 st, u64 cmask)
+{
+ int pgshft;
+
+ pgshft = ilog2(umem->page_size);
+
+ return pgshft;
+}
+
+static int bnxt_re_get_num_pages(struct ib_umem *umem, u64 start, u64 length, int page_shift)
+{
+ int npages = 0;
+
+ if (page_shift == PAGE_SHIFT) {
+ npages = ib_umem_num_pages_compat(umem);
+ } else {
+ npages = ALIGN(length, BIT(page_shift)) / BIT(page_shift);
+ if (start % BIT(page_shift))
+ npages++;
+ }
+ return npages;
+}
+
+/* uverbs */
+struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length,
+ u64 virt_addr, int mr_access_flags,
+ struct ib_udata *udata)
+{
+ struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd);
+ struct bnxt_re_dev *rdev = pd->rdev;
+ struct bnxt_qplib_mrinfo mrinfo;
+ int umem_pgs, page_shift, rc;
+ struct bnxt_re_mr *mr;
+ struct ib_umem *umem;
+ u32 max_mr_count;
+ int npages;
+
+ dev_dbg(rdev_to_dev(rdev), "Reg user MR\n");
+
+ if (bnxt_re_get_total_mr_mw_count(rdev) >= rdev->dev_attr->max_mr)
+ return ERR_PTR(-ENOMEM);
+
+ if (rdev->mod_exit) {
+ dev_dbg(rdev_to_dev(rdev), "%s(): in mod_exit, just return!\n", __func__);
+ return ERR_PTR(-EIO);
+ }
+ memset(&mrinfo, 0, sizeof(mrinfo));
+ if (length > BNXT_RE_MAX_MR_SIZE) {
+ dev_err(rdev_to_dev(rdev), "Requested MR Size: %lu "
+ "> Max supported: %ld\n", length, BNXT_RE_MAX_MR_SIZE);
+ return ERR_PTR(-ENOMEM);
+ }
+ mr = kzalloc(sizeof(*mr), GFP_KERNEL);
+ if (!mr) {
+ dev_err(rdev_to_dev(rdev), "Allocate MR failed!\n");
+ return ERR_PTR (-ENOMEM);
+ }
+ mr->rdev = rdev;
+ mr->qplib_mr.pd = &pd->qplib_pd;
+ mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags);
+ mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR;
+
+ if (!_is_alloc_mr_unified(rdev->qplib_res.dattr)) {
+ rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "Alloc MR failed!\n");
+ goto fail;
+ }
+ /* The fixed portion of the rkey is the same as the lkey */
+ mr->ib_mr.rkey = mr->qplib_mr.rkey;
+ }
+
+ umem = ib_umem_get_flags_compat(rdev, ib_pd->uobject->context,
+ udata, start, length,
+ mr_access_flags, 0);
+ if (IS_ERR(umem)) {
+ rc = PTR_ERR(umem);
+ dev_err(rdev_to_dev(rdev), "%s: ib_umem_get failed! rc = %d\n",
+ __func__, rc);
+ goto free_mr;
+ }
+ mr->ib_umem = umem;
+
+ mr->qplib_mr.va = virt_addr;
+ umem_pgs = ib_umem_num_pages_compat(umem);
+ if (!umem_pgs) {
+ dev_err(rdev_to_dev(rdev), "umem is invalid!\n");
+ rc = -EINVAL;
+ goto free_umem;
+ }
+ mr->qplib_mr.total_size = length;
+ page_shift = bnxt_re_get_page_shift(umem, virt_addr, start,
+ rdev->dev_attr->page_size_cap);
+ if (!bnxt_re_page_size_ok(page_shift)) {
+ dev_err(rdev_to_dev(rdev), "umem page size unsupported!\n");
+ rc = -EFAULT;
+ goto free_umem;
+ }
+ npages = bnxt_re_get_num_pages(umem, start, length, page_shift);
+
+ /* Map umem buf ptrs to the PBL */
+ mrinfo.sg.npages = npages;
+ mrinfo.sg.sghead = get_ib_umem_sgl(umem, &mrinfo.sg.nmap);
+ mrinfo.sg.pgshft = page_shift;
+ mrinfo.sg.pgsize = BIT(page_shift);
+
+ mrinfo.mrw = &mr->qplib_mr;
+
+ rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mrinfo, false);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "Reg user MR failed!\n");
+ goto free_umem;
+ }
+
+ mr->ib_mr.lkey = mr->ib_mr.rkey = mr->qplib_mr.lkey;
+ atomic_inc(&rdev->stats.rsors.mr_count);
+ max_mr_count = atomic_read(&rdev->stats.rsors.mr_count);
+ if (max_mr_count > atomic_read(&rdev->stats.rsors.max_mr_count))
+ atomic_set(&rdev->stats.rsors.max_mr_count, max_mr_count);
+
+ return &mr->ib_mr;
+
+free_umem:
+ bnxt_re_peer_mem_release(mr->ib_umem);
+free_mr:
+ if (!_is_alloc_mr_unified(rdev->qplib_res.dattr))
+ bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
+fail:
+ kfree(mr);
+ return ERR_PTR(rc);
+}
+
+int
+bnxt_re_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, u64 length,
+ u64 virt_addr, int mr_access_flags,
+ struct ib_pd *ib_pd, struct ib_udata *udata)
+{
+ struct bnxt_re_mr *mr = to_bnxt_re(ib_mr, struct bnxt_re_mr, ib_mr);
+ struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd);
+ int umem_pgs = 0, page_shift = PAGE_SHIFT, rc;
+ struct bnxt_re_dev *rdev = mr->rdev;
+ struct bnxt_qplib_mrinfo mrinfo;
+ struct ib_umem *umem;
+ u32 npages;
+
+ /* TODO: Must decipher what to modify based on the flags */
+ memset(&mrinfo, 0, sizeof(mrinfo));
+ if (flags & IB_MR_REREG_TRANS) {
+ umem = ib_umem_get_flags_compat(rdev, ib_pd->uobject->context,
+ udata, start, length,
+ mr_access_flags, 0);
+ if (IS_ERR(umem)) {
+ rc = PTR_ERR(umem);
+ dev_err(rdev_to_dev(rdev),
+ "%s: ib_umem_get failed! ret = %d\n",
+ __func__, rc);
+ goto fail;
+ }
+ mr->ib_umem = umem;
+
+ mr->qplib_mr.va = virt_addr;
+ umem_pgs = ib_umem_num_pages_compat(umem);
+ if (!umem_pgs) {
+ dev_err(rdev_to_dev(rdev), "umem is invalid!\n");
+ rc = -EINVAL;
+ goto fail_free_umem;
+ }
+ mr->qplib_mr.total_size = length;
+ page_shift = bnxt_re_get_page_shift(umem, virt_addr, start,
+ rdev->dev_attr->page_size_cap);
+ if (!bnxt_re_page_size_ok(page_shift)) {
+ dev_err(rdev_to_dev(rdev),
+ "umem page size unsupported!\n");
+ rc = -EFAULT;
+ goto fail_free_umem;
+ }
+ npages = bnxt_re_get_num_pages(umem, start, length, page_shift);
+ /* Map umem buf ptrs to the PBL */
+ mrinfo.sg.npages = npages;
+ mrinfo.sg.sghead = get_ib_umem_sgl(umem, &mrinfo.sg.nmap);
+ mrinfo.sg.pgshft = page_shift;
+ mrinfo.sg.pgsize = BIT(page_shift);
+ }
+
+ mrinfo.mrw = &mr->qplib_mr;
+ if (flags & IB_MR_REREG_PD)
+ mr->qplib_mr.pd = &pd->qplib_pd;
+
+ if (flags & IB_MR_REREG_ACCESS)
+ mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags);
+
+ rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mrinfo, false);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "Rereg user MR failed!\n");
+ goto fail_free_umem;
+ }
+ mr->ib_mr.rkey = mr->qplib_mr.rkey;
+
+ return 0;
+
+fail_free_umem:
+ bnxt_re_peer_mem_release(mr->ib_umem);
+fail:
+ return rc;
+}
+
+static int bnxt_re_check_abi_version(struct bnxt_re_dev *rdev)
+{
+ struct ib_device *ibdev = &rdev->ibdev;
+ u32 uverbs_abi_ver;
+
+ uverbs_abi_ver = GET_UVERBS_ABI_VERSION(ibdev);
+ dev_dbg(rdev_to_dev(rdev), "ABI version requested %d\n",
+ uverbs_abi_ver);
+ if (uverbs_abi_ver != BNXT_RE_ABI_VERSION) {
+ dev_dbg(rdev_to_dev(rdev), " is different from the device %d \n",
+ BNXT_RE_ABI_VERSION);
+ return -EPERM;
+ }
+ return 0;
+}
+
+int bnxt_re_alloc_ucontext(struct ib_ucontext *uctx_in,
+ struct ib_udata *udata)
+{
+ struct ib_ucontext *ctx = uctx_in;
+ struct ib_device *ibdev = ctx->device;
+ struct bnxt_re_ucontext *uctx =
+ container_of(ctx, struct bnxt_re_ucontext, ibucontext);
+
+ struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
+ struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr;
+ struct bnxt_re_uctx_resp resp = {};
+ struct bnxt_re_uctx_req ureq = {};
+ struct bnxt_qplib_chip_ctx *cctx;
+ u32 chip_met_rev_num;
+ bool genp5 = false;
+ int rc;
+
+ cctx = rdev->chip_ctx;
+ rc = bnxt_re_check_abi_version(rdev);
+ if (rc)
+ goto fail;
+
+ uctx->rdev = rdev;
+ uctx->shpg = (void *)__get_free_page(GFP_KERNEL);
+ if (!uctx->shpg) {
+ dev_err(rdev_to_dev(rdev), "shared memory allocation failed!\n");
+ rc = -ENOMEM;
+ goto fail;
+ }
+ spin_lock_init(&uctx->sh_lock);
+ if (BNXT_RE_ABI_VERSION >= 4) {
+ chip_met_rev_num = cctx->chip_num;
+ chip_met_rev_num |= ((u32)cctx->chip_rev & 0xFF) <<
+ BNXT_RE_CHIP_ID0_CHIP_REV_SFT;
+ chip_met_rev_num |= ((u32)cctx->chip_metal & 0xFF) <<
+ BNXT_RE_CHIP_ID0_CHIP_MET_SFT;
+ resp.chip_id0 = chip_met_rev_num;
+ resp.chip_id1 = 0; /* future extension of chip info */
+ }
+
+ if (BNXT_RE_ABI_VERSION != 4) {
+ /*Temp, Use idr_alloc instead*/
+ resp.dev_id = rdev->en_dev->pdev->devfn;
+ resp.max_qp = rdev->qplib_res.hctx->qp_ctx.max;
+ }
+
+ genp5 = _is_chip_gen_p5_p7(cctx);
+ if (BNXT_RE_ABI_VERSION > 5) {
+ resp.modes = genp5 ? cctx->modes.wqe_mode : 0;
+ if (rdev->dev_attr && BNXT_RE_HW_RETX(rdev->dev_attr->dev_cap_flags))
+ resp.comp_mask = BNXT_RE_COMP_MASK_UCNTX_HW_RETX_ENABLED;
+ }
+
+ resp.pg_size = PAGE_SIZE;
+ resp.cqe_sz = sizeof(struct cq_base);
+ resp.max_cqd = dev_attr->max_cq_wqes;
+ if (genp5 && cctx->modes.db_push) {
+ resp.comp_mask |= BNXT_RE_COMP_MASK_UCNTX_WC_DPI_ENABLED;
+ if (_is_chip_p7(cctx) &&
+ !(dev_attr->dev_cap_flags &
+ CREQ_QUERY_FUNC_RESP_SB_PINGPONG_PUSH_MODE))
+ resp.comp_mask &=
+ ~BNXT_RE_COMP_MASK_UCNTX_WC_DPI_ENABLED;
+ }
+
+ resp.comp_mask |= BNXT_RE_COMP_MASK_UCNTX_MQP_EX_SUPPORTED;
+
+ if (rdev->dbr_pacing)
+ resp.comp_mask |= BNXT_RE_COMP_MASK_UCNTX_DBR_PACING_ENABLED;
+
+ if (rdev->dbr_drop_recov && rdev->user_dbr_drop_recov)
+ resp.comp_mask |= BNXT_RE_COMP_MASK_UCNTX_DBR_RECOVERY_ENABLED;
+
+ if (udata->inlen >= sizeof(ureq)) {
+ rc = ib_copy_from_udata(&ureq, udata,
+ min(udata->inlen, sizeof(ureq)));
+ if (rc)
+ goto cfail;
+ if (bnxt_re_init_pow2_flag(&ureq, &resp))
+ dev_warn(rdev_to_dev(rdev),
+ "Enabled roundup logic. Library bug?\n");
+ if (bnxt_re_init_rsvd_wqe_flag(&ureq, &resp, genp5))
+ dev_warn(rdev_to_dev(rdev),
+ "Rsvd wqe in use! Try the updated library.\n");
+ } else {
+ dev_warn(rdev_to_dev(rdev),
+ "Enabled roundup logic. Update the library!\n");
+ resp.comp_mask &= ~BNXT_RE_COMP_MASK_UCNTX_POW2_DISABLED;
+
+ dev_warn(rdev_to_dev(rdev),
+ "Rsvd wqe in use. Update the library!\n");
+ resp.comp_mask &= ~BNXT_RE_COMP_MASK_UCNTX_RSVD_WQE_DISABLED;
+ }
+
+ uctx->cmask = (uint64_t)resp.comp_mask;
+ rc = bnxt_re_copy_to_udata(rdev, &resp,
+ min(udata->outlen, sizeof(resp)),
+ udata);
+ if (rc)
+ goto cfail;
+
+ INIT_LIST_HEAD(&uctx->cq_list);
+ mutex_init(&uctx->cq_lock);
+
+ return 0;
+cfail:
+ free_page((u64)uctx->shpg);
+ uctx->shpg = NULL;
+fail:
+ return rc;
+}
+
+void bnxt_re_dealloc_ucontext(struct ib_ucontext *ib_uctx)
+{
+ struct bnxt_re_ucontext *uctx = to_bnxt_re(ib_uctx,
+ struct bnxt_re_ucontext,
+ ibucontext);
+ struct bnxt_re_dev *rdev = uctx->rdev;
+ int rc = 0;
+
+ if (uctx->shpg)
+ free_page((u64)uctx->shpg);
+
+ if (uctx->dpi.dbr) {
+ /* Free DPI only if this is the first PD allocated by the
+ * application and mark the context dpi as NULL
+ */
+ if (_is_chip_gen_p5_p7(rdev->chip_ctx) && uctx->wcdpi.dbr) {
+ rc = bnxt_qplib_dealloc_dpi(&rdev->qplib_res,
+ &uctx->wcdpi);
+ if (rc)
+ dev_err(rdev_to_dev(rdev),
+ "dealloc push dp failed\n");
+ uctx->wcdpi.dbr = NULL;
+ }
+
+ rc = bnxt_qplib_dealloc_dpi(&rdev->qplib_res,
+ &uctx->dpi);
+ if (rc)
+ dev_err(rdev_to_dev(rdev), "Deallocte HW DPI failed!\n");
+ /* Don't fail, continue*/
+ uctx->dpi.dbr = NULL;
+ }
+ return;
+}
+
+static struct bnxt_re_cq *is_bnxt_re_cq_page(struct bnxt_re_ucontext *uctx,
+ u64 pg_off)
+{
+ struct bnxt_re_cq *cq = NULL, *tmp_cq;
+
+ if (!_is_chip_p7(uctx->rdev->chip_ctx))
+ return NULL;
+
+ mutex_lock(&uctx->cq_lock);
+ list_for_each_entry(tmp_cq, &uctx->cq_list, cq_list) {
+ if (((u64)tmp_cq->uctx_cq_page >> PAGE_SHIFT) == pg_off) {
+ cq = tmp_cq;
+ break;
+ }
+ }
+ mutex_unlock(&uctx->cq_lock);
+ return cq;
+}
+
+/* Helper function to mmap the virtual memory from user app */
+int bnxt_re_mmap(struct ib_ucontext *ib_uctx, struct vm_area_struct *vma)
+{
+ struct bnxt_re_ucontext *uctx = to_bnxt_re(ib_uctx,
+ struct bnxt_re_ucontext,
+ ibucontext);
+ struct bnxt_re_dev *rdev = uctx->rdev;
+ struct bnxt_re_cq *cq = NULL;
+ int rc = 0;
+ u64 pfn;
+
+ switch (vma->vm_pgoff) {
+ case BNXT_RE_MAP_SH_PAGE:
+ pfn = vtophys(uctx->shpg) >> PAGE_SHIFT;
+ return rdma_user_mmap_io(&uctx->ibucontext, vma, pfn, PAGE_SIZE, vma->vm_page_prot, NULL);
+ dev_dbg(rdev_to_dev(rdev), "%s:%d uctx->shpg 0x%lx, vtophys(uctx->shpg) 0x%lx, pfn = 0x%lx \n",
+ __func__, __LINE__, (u64) uctx->shpg, vtophys(uctx->shpg), pfn);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "Shared page mapping failed!\n");
+ rc = -EAGAIN;
+ }
+ return rc;
+ case BNXT_RE_MAP_WC:
+ vma->vm_page_prot =
+ pgprot_writecombine(vma->vm_page_prot);
+ pfn = (uctx->wcdpi.umdbr >> PAGE_SHIFT);
+ if (!pfn)
+ return -EFAULT;
+ break;
+ case BNXT_RE_DBR_PAGE:
+ /* Driver doesn't expect write access request */
+ if (vma->vm_flags & VM_WRITE)
+ return -EFAULT;
+
+ pfn = vtophys(rdev->dbr_page) >> PAGE_SHIFT;
+ if (!pfn)
+ return -EFAULT;
+ break;
+ case BNXT_RE_MAP_DB_RECOVERY_PAGE:
+ pfn = vtophys(uctx->dbr_recov_cq_page) >> PAGE_SHIFT;
+ if (!pfn)
+ return -EFAULT;
+ break;
+ default:
+ cq = is_bnxt_re_cq_page(uctx, vma->vm_pgoff);
+ if (cq) {
+ pfn = vtophys((void *)cq->uctx_cq_page) >> PAGE_SHIFT;
+ rc = rdma_user_mmap_io(&uctx->ibucontext, vma, pfn, PAGE_SIZE, vma->vm_page_prot, NULL);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev),
+ "CQ page mapping failed!\n");
+ rc = -EAGAIN;
+ }
+ goto out;
+ } else {
+ vma->vm_page_prot =
+ pgprot_noncached(vma->vm_page_prot);
+ pfn = vma->vm_pgoff;
+ }
+ break;
+ }
+
+ rc = rdma_user_mmap_io(&uctx->ibucontext, vma, pfn, PAGE_SIZE, vma->vm_page_prot, NULL);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "DPI mapping failed!\n");
+ return -EAGAIN;
+ }
+ rc = __bnxt_re_set_vma_data(uctx, vma);
+out:
+ return rc;
+}
+
+int bnxt_re_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
+ const struct ib_wc *wc, const struct ib_grh *grh,
+ const struct ib_mad_hdr *in_mad, size_t in_mad_size,
+ struct ib_mad_hdr *out_mad, size_t *out_mad_size,
+ u16 *out_mad_pkey_index)
+{
+ return IB_MAD_RESULT_SUCCESS;
+}
+
+void bnxt_re_disassociate_ucntx(struct ib_ucontext *ib_uctx)
+{
+}
diff --git a/sys/dev/bnxt/bnxt_re/ib_verbs.h b/sys/dev/bnxt/bnxt_re/ib_verbs.h
new file mode 100644
index 000000000000..cb9f7974e92d
--- /dev/null
+++ b/sys/dev/bnxt/bnxt_re/ib_verbs.h
@@ -0,0 +1,632 @@
+/*
+ * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term
+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Description: IB Verbs interpreter (header)
+ */
+
+#ifndef __BNXT_RE_IB_VERBS_H__
+#define __BNXT_RE_IB_VERBS_H__
+
+#include <rdma/ib_addr.h>
+#include "bnxt_re-abi.h"
+#include "qplib_res.h"
+#include "qplib_fp.h"
+
+struct bnxt_re_dev;
+
+#define BNXT_RE_ROCE_V2_UDP_SPORT 0x8CD1
+#define BNXT_RE_QP_RANDOM_QKEY 0x81818181
+
+#ifndef IB_MTU_8192
+#define IB_MTU_8192 8192
+#endif
+
+#ifndef SPEED_1000
+#define SPEED_1000 1000
+#endif
+
+#ifndef SPEED_10000
+#define SPEED_10000 10000
+#endif
+
+#ifndef SPEED_20000
+#define SPEED_20000 20000
+#endif
+
+#ifndef SPEED_25000
+#define SPEED_25000 25000
+#endif
+
+#ifndef SPEED_40000
+#define SPEED_40000 40000
+#endif
+
+#ifndef SPEED_50000
+#define SPEED_50000 50000
+#endif
+
+#ifndef SPEED_100000
+#define SPEED_100000 100000
+#endif
+
+#ifndef SPEED_200000
+#define SPEED_200000 200000
+#endif
+
+#ifndef IB_SPEED_HDR
+#define IB_SPEED_HDR 64
+#endif
+
+#define RDMA_NETWORK_IPV4 1
+#define RDMA_NETWORK_IPV6 2
+
+#define ROCE_DMAC(x) (x)->dmac
+
+#define dma_rmb() rmb()
+
+#define compat_ib_alloc_device(size) ib_alloc_device(size);
+
+#define rdev_from_cq_in(cq_in) to_bnxt_re_dev(cq_in->device, ibdev)
+
+#define GET_UVERBS_ABI_VERSION(ibdev) (ibdev->uverbs_abi_ver)
+
+#define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_256MB 0x1cUL
+
+#define IB_POLL_UNBOUND_WORKQUEUE IB_POLL_WORKQUEUE
+
+#define BNXT_RE_LEGACY_FENCE_BYTES 64
+#define BNXT_RE_LEGACY_FENCE_PBL_SIZE DIV_ROUND_UP(BNXT_RE_LEGACY_FENCE_BYTES, PAGE_SIZE)
+
+
+static inline struct
+bnxt_re_cq *__get_cq_from_cq_in(struct ib_cq *cq_in,
+ struct bnxt_re_dev *rdev);
+static inline struct
+bnxt_re_qp *__get_qp_from_qp_in(struct ib_pd *qp_in,
+ struct bnxt_re_dev *rdev);
+
+static inline bool
+bnxt_re_check_if_vlan_valid(struct bnxt_re_dev *rdev, u16 vlan_id);
+
+#define bnxt_re_compat_qfwstr(void) \
+ bnxt_re_query_fw_str(struct ib_device *ibdev, \
+ char *str, size_t str_len)
+
+static inline
+struct scatterlist *get_ib_umem_sgl(struct ib_umem *umem, u32 *nmap);
+
+struct bnxt_re_gid_ctx {
+ u32 idx;
+ u32 refcnt;
+};
+
+struct bnxt_re_legacy_fence_data {
+ u32 size;
+ void *va;
+ dma_addr_t dma_addr;
+ struct bnxt_re_mr *mr;
+ struct ib_mw *mw;
+ struct bnxt_qplib_swqe bind_wqe;
+ u32 bind_rkey;
+};
+
+struct bnxt_re_pd {
+ struct ib_pd ibpd;
+ struct bnxt_re_dev *rdev;
+ struct bnxt_qplib_pd qplib_pd;
+ struct bnxt_re_legacy_fence_data fence;
+};
+
+struct bnxt_re_ah {
+ struct ib_ah ibah;
+ struct bnxt_re_dev *rdev;
+ struct bnxt_qplib_ah qplib_ah;
+};
+
+struct bnxt_re_srq {
+ struct ib_srq ibsrq;
+ struct bnxt_re_dev *rdev;
+ u32 srq_limit;
+ struct bnxt_qplib_srq qplib_srq;
+ struct ib_umem *umem;
+ spinlock_t lock;
+};
+
+union ip_addr {
+ u32 ipv4_addr;
+ u8 ipv6_addr[16];
+};
+
+struct bnxt_re_qp_info_entry {
+ union ib_gid sgid;
+ union ib_gid dgid;
+ union ip_addr s_ip;
+ union ip_addr d_ip;
+ u16 s_port;
+#define BNXT_RE_QP_DEST_PORT 4791
+ u16 d_port;
+};
+
+struct bnxt_re_qp {
+ struct ib_qp ib_qp;
+ struct list_head list;
+ struct bnxt_re_dev *rdev;
+ spinlock_t sq_lock;
+ spinlock_t rq_lock;
+ struct bnxt_qplib_qp qplib_qp;
+ struct ib_umem *sumem;
+ struct ib_umem *rumem;
+ /* QP1 */
+ u32 send_psn;
+ struct ib_ud_header qp1_hdr;
+ struct bnxt_re_cq *scq;
+ struct bnxt_re_cq *rcq;
+ struct dentry *qp_info_pdev_dentry;
+ struct bnxt_re_qp_info_entry qp_info_entry;
+ void *qp_data;
+};
+
+struct bnxt_re_cq {
+ struct ib_cq ibcq;
+ struct list_head cq_list;
+ struct bnxt_re_dev *rdev;
+ struct bnxt_re_ucontext *uctx;
+ spinlock_t cq_lock;
+ u16 cq_count;
+ u16 cq_period;
+ struct bnxt_qplib_cq qplib_cq;
+ struct bnxt_qplib_cqe *cql;
+#define MAX_CQL_PER_POLL 1024
+ u32 max_cql;
+ struct ib_umem *umem;
+ struct ib_umem *resize_umem;
+ struct ib_ucontext *context;
+ int resize_cqe;
+ /* list of cq per uctx. Used only for Thor-2 */
+ void *uctx_cq_page;
+ void *dbr_recov_cq_page;
+ bool is_dbr_soft_cq;
+};
+
+struct bnxt_re_mr {
+ struct bnxt_re_dev *rdev;
+ struct ib_mr ib_mr;
+ struct ib_umem *ib_umem;
+ struct bnxt_qplib_mrw qplib_mr;
+ u32 npages;
+ u64 *pages;
+ struct bnxt_qplib_frpl qplib_frpl;
+ bool is_invalcb_active;
+};
+
+struct bnxt_re_frpl {
+ struct bnxt_re_dev *rdev;
+ struct bnxt_qplib_frpl qplib_frpl;
+ u64 *page_list;
+};
+
+struct bnxt_re_mw {
+ struct bnxt_re_dev *rdev;
+ struct ib_mw ib_mw;
+ struct bnxt_qplib_mrw qplib_mw;
+};
+
+struct bnxt_re_ucontext {
+ struct ib_ucontext ibucontext;
+ struct bnxt_re_dev *rdev;
+ struct list_head cq_list;
+ struct bnxt_qplib_dpi dpi;
+ struct bnxt_qplib_dpi wcdpi;
+ void *shpg;
+ spinlock_t sh_lock;
+ uint64_t cmask;
+ struct mutex cq_lock; /* Protect cq list */
+ void *dbr_recov_cq_page;
+ struct bnxt_re_cq *dbr_recov_cq;
+};
+
+struct bnxt_re_ah_info {
+ union ib_gid sgid;
+ struct ib_gid_attr sgid_attr;
+ u16 vlan_tag;
+ u8 nw_type;
+};
+
+struct ifnet *bnxt_re_get_netdev(struct ib_device *ibdev,
+ u8 port_num);
+
+int bnxt_re_query_device(struct ib_device *ibdev,
+ struct ib_device_attr *ib_attr,
+ struct ib_udata *udata);
+int bnxt_re_modify_device(struct ib_device *ibdev,
+ int device_modify_mask,
+ struct ib_device_modify *device_modify);
+int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num,
+ struct ib_port_attr *port_attr);
+int bnxt_re_modify_port(struct ib_device *ibdev, u8 port_num,
+ int port_modify_mask,
+ struct ib_port_modify *port_modify);
+int bnxt_re_get_port_immutable(struct ib_device *ibdev, u8 port_num,
+ struct ib_port_immutable *immutable);
+void bnxt_re_compat_qfwstr(void);
+int bnxt_re_query_pkey(struct ib_device *ibdev, u8 port_num,
+ u16 index, u16 *pkey);
+int bnxt_re_del_gid(struct ib_device *ibdev, u8 port_num,
+ unsigned int index, void **context);
+int bnxt_re_add_gid(struct ib_device *ibdev, u8 port_num,
+ unsigned int index, const union ib_gid *gid,
+ const struct ib_gid_attr *attr, void **context);
+int bnxt_re_query_gid(struct ib_device *ibdev, u8 port_num,
+ int index, union ib_gid *gid);
+enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev,
+ u8 port_num);
+int bnxt_re_alloc_pd(struct ib_pd *pd_in, struct ib_udata *udata);
+void bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata);
+
+int bnxt_re_create_ah(struct ib_ah *ah_in, struct ib_ah_attr *attr,
+ u32 flags, struct ib_udata *udata);
+
+int bnxt_re_modify_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr);
+int bnxt_re_query_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr);
+
+void bnxt_re_destroy_ah(struct ib_ah *ib_ah, u32 flags);
+int bnxt_re_create_srq(struct ib_srq *srq_in,
+ struct ib_srq_init_attr *srq_init_attr,
+ struct ib_udata *udata);
+int bnxt_re_modify_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr,
+ enum ib_srq_attr_mask srq_attr_mask,
+ struct ib_udata *udata);
+int bnxt_re_query_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr);
+void bnxt_re_destroy_srq(struct ib_srq *ib_srq,
+ struct ib_udata *udata);
+int bnxt_re_post_srq_recv(struct ib_srq *ib_srq, const struct ib_recv_wr *wr,
+ const struct ib_recv_wr **bad_wr);
+struct ib_qp *bnxt_re_create_qp(struct ib_pd *qp_in,
+ struct ib_qp_init_attr *qp_init_attr,
+ struct ib_udata *udata);
+int bnxt_re_modify_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
+ int qp_attr_mask, struct ib_udata *udata);
+int bnxt_re_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
+ int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr);
+int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata);
+int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr,
+ const struct ib_send_wr **bad_wr);
+int bnxt_re_post_recv(struct ib_qp *ib_qp, const struct ib_recv_wr *wr,
+ const struct ib_recv_wr **bad_wr);
+int bnxt_re_create_cq(struct ib_cq *cq_in,
+ const struct ib_cq_init_attr *attr,
+ struct ib_udata *udata);
+void bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata);
+int bnxt_re_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period);
+int bnxt_re_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata);
+int bnxt_re_poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc);
+int bnxt_re_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags);
+struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *pd, int mr_access_flags);
+int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg,
+ int sg_nents, unsigned int *sg_offset);
+struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type,
+ u32 max_num_sg, struct ib_udata *udata);
+int bnxt_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata);
+struct ib_mw *bnxt_re_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type,
+ struct ib_udata *udata);
+int bnxt_re_dealloc_mw(struct ib_mw *mw);
+struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
+ u64 virt_addr, int mr_access_flags,
+ struct ib_udata *udata);
+int
+bnxt_re_rereg_user_mr(struct ib_mr *mr, int flags, u64 start, u64 length,
+ u64 virt_addr, int mr_access_flags, struct ib_pd *pd,
+ struct ib_udata *udata);
+int bnxt_re_alloc_ucontext(struct ib_ucontext *uctx_in,
+ struct ib_udata *udata);
+void bnxt_re_dealloc_ucontext(struct ib_ucontext *ib_uctx);
+int bnxt_re_mmap(struct ib_ucontext *context, struct vm_area_struct *vma);
+int bnxt_re_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
+ const struct ib_wc *wc, const struct ib_grh *grh,
+ const struct ib_mad_hdr *in_mad, size_t in_mad_size,
+ struct ib_mad_hdr *out_mad, size_t *out_mad_size,
+ u16 *out_mad_pkey_index);
+unsigned long bnxt_re_lock_cqs(struct bnxt_re_qp *qp);
+void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp, unsigned long flags);
+void bnxt_re_disassociate_ucntx(struct ib_ucontext *ibcontext);
+static inline int __bnxt_re_set_vma_data(void *bnxt_re_uctx,
+ struct vm_area_struct *vma);
+void bnxt_re_update_shadow_ah(struct bnxt_re_dev *rdev);
+void bnxt_re_handle_cqn(struct bnxt_qplib_cq *cq);
+static inline int
+bnxt_re_get_cached_gid(struct ib_device *dev, u8 port_num, int index,
+ union ib_gid *sgid, struct ib_gid_attr **sgid_attr,
+ struct ib_global_route *grh, struct ib_ah *ah);
+static inline enum rdma_network_type
+bnxt_re_gid_to_network_type(struct ib_gid_attr *sgid_attr,
+ union ib_gid *sgid);
+static inline
+struct ib_umem *ib_umem_get_compat(struct bnxt_re_dev *rdev,
+ struct ib_ucontext *ucontext,
+ struct ib_udata *udata,
+ unsigned long addr,
+ size_t size, int access, int dmasync);
+static inline
+struct ib_umem *ib_umem_get_flags_compat(struct bnxt_re_dev *rdev,
+ struct ib_ucontext *ucontext,
+ struct ib_udata *udata,
+ unsigned long addr,
+ size_t size, int access, int dmasync);
+static inline size_t ib_umem_num_pages_compat(struct ib_umem *umem);
+static inline void bnxt_re_peer_mem_release(struct ib_umem *umem);
+void bnxt_re_resolve_dmac_task(struct work_struct *work);
+
+static inline enum ib_qp_type __from_hw_to_ib_qp_type(u8 type)
+{
+ switch (type) {
+ case CMDQ_CREATE_QP1_TYPE_GSI:
+ case CMDQ_CREATE_QP_TYPE_GSI:
+ return IB_QPT_GSI;
+ case CMDQ_CREATE_QP_TYPE_RC:
+ return IB_QPT_RC;
+ case CMDQ_CREATE_QP_TYPE_UD:
+ return IB_QPT_UD;
+ case CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE:
+ return IB_QPT_RAW_ETHERTYPE;
+ default:
+ return IB_QPT_MAX;
+ }
+}
+
+static inline u8 __from_ib_qp_state(enum ib_qp_state state)
+{
+ switch (state) {
+ case IB_QPS_RESET:
+ return CMDQ_MODIFY_QP_NEW_STATE_RESET;
+ case IB_QPS_INIT:
+ return CMDQ_MODIFY_QP_NEW_STATE_INIT;
+ case IB_QPS_RTR:
+ return CMDQ_MODIFY_QP_NEW_STATE_RTR;
+ case IB_QPS_RTS:
+ return CMDQ_MODIFY_QP_NEW_STATE_RTS;
+ case IB_QPS_SQD:
+ return CMDQ_MODIFY_QP_NEW_STATE_SQD;
+ case IB_QPS_SQE:
+ return CMDQ_MODIFY_QP_NEW_STATE_SQE;
+ case IB_QPS_ERR:
+ default:
+ return CMDQ_MODIFY_QP_NEW_STATE_ERR;
+ }
+}
+
+static inline u32 __from_ib_mtu(enum ib_mtu mtu)
+{
+ switch (mtu) {
+ case IB_MTU_256:
+ return CMDQ_MODIFY_QP_PATH_MTU_MTU_256;
+ case IB_MTU_512:
+ return CMDQ_MODIFY_QP_PATH_MTU_MTU_512;
+ case IB_MTU_1024:
+ return CMDQ_MODIFY_QP_PATH_MTU_MTU_1024;
+ case IB_MTU_2048:
+ return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
+ case IB_MTU_4096:
+ return CMDQ_MODIFY_QP_PATH_MTU_MTU_4096;
+ default:
+ return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
+ }
+}
+
+static inline enum ib_mtu __to_ib_mtu(u32 mtu)
+{
+ switch (mtu & CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) {
+ case CMDQ_MODIFY_QP_PATH_MTU_MTU_256:
+ return IB_MTU_256;
+ case CMDQ_MODIFY_QP_PATH_MTU_MTU_512:
+ return IB_MTU_512;
+ case CMDQ_MODIFY_QP_PATH_MTU_MTU_1024:
+ return IB_MTU_1024;
+ case CMDQ_MODIFY_QP_PATH_MTU_MTU_2048:
+ return IB_MTU_2048;
+ case CMDQ_MODIFY_QP_PATH_MTU_MTU_4096:
+ return IB_MTU_4096;
+ case CMDQ_MODIFY_QP_PATH_MTU_MTU_8192:
+ return IB_MTU_8192;
+ default:
+ return IB_MTU_2048;
+ }
+}
+
+static inline enum ib_qp_state __to_ib_qp_state(u8 state)
+{
+ switch (state) {
+ case CMDQ_MODIFY_QP_NEW_STATE_RESET:
+ return IB_QPS_RESET;
+ case CMDQ_MODIFY_QP_NEW_STATE_INIT:
+ return IB_QPS_INIT;
+ case CMDQ_MODIFY_QP_NEW_STATE_RTR:
+ return IB_QPS_RTR;
+ case CMDQ_MODIFY_QP_NEW_STATE_RTS:
+ return IB_QPS_RTS;
+ case CMDQ_MODIFY_QP_NEW_STATE_SQD:
+ return IB_QPS_SQD;
+ case CMDQ_MODIFY_QP_NEW_STATE_SQE:
+ return IB_QPS_SQE;
+ case CMDQ_MODIFY_QP_NEW_STATE_ERR:
+ default:
+ return IB_QPS_ERR;
+ }
+}
+
+static inline int bnxt_re_init_pow2_flag(struct bnxt_re_uctx_req *req,
+ struct bnxt_re_uctx_resp *resp)
+{
+ resp->comp_mask |= BNXT_RE_COMP_MASK_UCNTX_POW2_DISABLED;
+ if (!(req->comp_mask & BNXT_RE_COMP_MASK_REQ_UCNTX_POW2_SUPPORT)) {
+ resp->comp_mask &= ~BNXT_RE_COMP_MASK_UCNTX_POW2_DISABLED;
+ return -EINVAL;
+ }
+ return 0;
+}
+
+static inline u32 bnxt_re_init_depth(u32 ent, struct bnxt_re_ucontext *uctx)
+{
+ return uctx ? (uctx->cmask & BNXT_RE_COMP_MASK_UCNTX_POW2_DISABLED) ?
+ ent : roundup_pow_of_two(ent) : ent;
+}
+
+static inline int bnxt_re_init_rsvd_wqe_flag(struct bnxt_re_uctx_req *req,
+ struct bnxt_re_uctx_resp *resp,
+ bool genp5)
+{
+ resp->comp_mask |= BNXT_RE_COMP_MASK_UCNTX_RSVD_WQE_DISABLED;
+ if (!(req->comp_mask & BNXT_RE_COMP_MASK_REQ_UCNTX_RSVD_WQE)) {
+ resp->comp_mask &= ~BNXT_RE_COMP_MASK_UCNTX_RSVD_WQE_DISABLED;
+ return -EINVAL;
+ } else if (!genp5) {
+ resp->comp_mask &= ~BNXT_RE_COMP_MASK_UCNTX_RSVD_WQE_DISABLED;
+ }
+ return 0;
+}
+
+static inline u32 bnxt_re_get_diff(struct bnxt_re_ucontext *uctx,
+ struct bnxt_qplib_chip_ctx *cctx)
+{
+ if (!uctx) {
+ /* return res-wqe only for gen p4 for user resource */
+ return _is_chip_gen_p5_p7(cctx) ? 0 : BNXT_QPLIB_RESERVED_QP_WRS;
+ } else if (uctx->cmask & BNXT_RE_COMP_MASK_UCNTX_RSVD_WQE_DISABLED) {
+ return 0;
+ }
+ /* old lib */
+ return BNXT_QPLIB_RESERVED_QP_WRS;
+}
+
+static inline void bnxt_re_init_qpmtu(struct bnxt_re_qp *qp, int mtu,
+ int mask, struct ib_qp_attr *qp_attr,
+ bool *is_qpmtu_high)
+{
+ int qpmtu, qpmtu_int;
+ int ifmtu, ifmtu_int;
+
+ ifmtu = iboe_get_mtu(mtu);
+ ifmtu_int = ib_mtu_enum_to_int(ifmtu);
+ qpmtu = ifmtu;
+ qpmtu_int = ifmtu_int;
+ if (mask & IB_QP_PATH_MTU) {
+ qpmtu = qp_attr->path_mtu;
+ qpmtu_int = ib_mtu_enum_to_int(qpmtu);
+ if (qpmtu_int > ifmtu_int) {
+ /* Trim the QP path mtu to interface mtu and update
+ * the new mtu to user qp for retransmission psn
+ * calculations.
+ */
+ qpmtu = ifmtu;
+ qpmtu_int = ifmtu_int;
+ *is_qpmtu_high = true;
+ }
+ }
+ qp->qplib_qp.path_mtu = __from_ib_mtu(qpmtu);
+ qp->qplib_qp.mtu = qpmtu_int;
+ qp->qplib_qp.modify_flags |=
+ CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
+}
+
+inline unsigned long compare_ether_header(void *a, void *b)
+{
+ u32 *a32 = (u32 *)((u8 *)a + 2);
+ u32 *b32 = (u32 *)((u8 *)b + 2);
+
+ return (*(u16 *)a ^ *(u16 *)b) | (a32[0] ^ b32[0]) |
+ (a32[1] ^ b32[1]) | (a32[2] ^ b32[2]);
+}
+
+struct vlan_hdr {
+ __be16 h_vlan_TCI;
+ __be16 h_vlan_encapsulated_proto;
+};
+
+inline uint16_t
+crc16(uint16_t crc, const void *buffer, unsigned int len)
+{
+ const unsigned char *cp = buffer;
+ /* CRC table for the CRC-16. The poly is 0x8005 (x16 + x15 + x2 + 1). */
+ static uint16_t const crc16_table[256] = {
+ 0x0000, 0xC0C1, 0xC181, 0x0140, 0xC301, 0x03C0, 0x0280, 0xC241,
+ 0xC601, 0x06C0, 0x0780, 0xC741, 0x0500, 0xC5C1, 0xC481, 0x0440,
+ 0xCC01, 0x0CC0, 0x0D80, 0xCD41, 0x0F00, 0xCFC1, 0xCE81, 0x0E40,
+ 0x0A00, 0xCAC1, 0xCB81, 0x0B40, 0xC901, 0x09C0, 0x0880, 0xC841,
+ 0xD801, 0x18C0, 0x1980, 0xD941, 0x1B00, 0xDBC1, 0xDA81, 0x1A40,
+ 0x1E00, 0xDEC1, 0xDF81, 0x1F40, 0xDD01, 0x1DC0, 0x1C80, 0xDC41,
+ 0x1400, 0xD4C1, 0xD581, 0x1540, 0xD701, 0x17C0, 0x1680, 0xD641,
+ 0xD201, 0x12C0, 0x1380, 0xD341, 0x1100, 0xD1C1, 0xD081, 0x1040,
+ 0xF001, 0x30C0, 0x3180, 0xF141, 0x3300, 0xF3C1, 0xF281, 0x3240,
+ 0x3600, 0xF6C1, 0xF781, 0x3740, 0xF501, 0x35C0, 0x3480, 0xF441,
+ 0x3C00, 0xFCC1, 0xFD81, 0x3D40, 0xFF01, 0x3FC0, 0x3E80, 0xFE41,
+ 0xFA01, 0x3AC0, 0x3B80, 0xFB41, 0x3900, 0xF9C1, 0xF881, 0x3840,
+ 0x2800, 0xE8C1, 0xE981, 0x2940, 0xEB01, 0x2BC0, 0x2A80, 0xEA41,
+ 0xEE01, 0x2EC0, 0x2F80, 0xEF41, 0x2D00, 0xEDC1, 0xEC81, 0x2C40,
+ 0xE401, 0x24C0, 0x2580, 0xE541, 0x2700, 0xE7C1, 0xE681, 0x2640,
+ 0x2200, 0xE2C1, 0xE381, 0x2340, 0xE101, 0x21C0, 0x2080, 0xE041,
+ 0xA001, 0x60C0, 0x6180, 0xA141, 0x6300, 0xA3C1, 0xA281, 0x6240,
+ 0x6600, 0xA6C1, 0xA781, 0x6740, 0xA501, 0x65C0, 0x6480, 0xA441,
+ 0x6C00, 0xACC1, 0xAD81, 0x6D40, 0xAF01, 0x6FC0, 0x6E80, 0xAE41,
+ 0xAA01, 0x6AC0, 0x6B80, 0xAB41, 0x6900, 0xA9C1, 0xA881, 0x6840,
+ 0x7800, 0xB8C1, 0xB981, 0x7940, 0xBB01, 0x7BC0, 0x7A80, 0xBA41,
+ 0xBE01, 0x7EC0, 0x7F80, 0xBF41, 0x7D00, 0xBDC1, 0xBC81, 0x7C40,
+ 0xB401, 0x74C0, 0x7580, 0xB541, 0x7700, 0xB7C1, 0xB681, 0x7640,
+ 0x7200, 0xB2C1, 0xB381, 0x7340, 0xB101, 0x71C0, 0x7080, 0xB041,
+ 0x5000, 0x90C1, 0x9181, 0x5140, 0x9301, 0x53C0, 0x5280, 0x9241,
+ 0x9601, 0x56C0, 0x5780, 0x9741, 0x5500, 0x95C1, 0x9481, 0x5440,
+ 0x9C01, 0x5CC0, 0x5D80, 0x9D41, 0x5F00, 0x9FC1, 0x9E81, 0x5E40,
+ 0x5A00, 0x9AC1, 0x9B81, 0x5B40, 0x9901, 0x59C0, 0x5880, 0x9841,
+ 0x8801, 0x48C0, 0x4980, 0x8941, 0x4B00, 0x8BC1, 0x8A81, 0x4A40,
+ 0x4E00, 0x8EC1, 0x8F81, 0x4F40, 0x8D01, 0x4DC0, 0x4C80, 0x8C41,
+ 0x4400, 0x84C1, 0x8581, 0x4540, 0x8701, 0x47C0, 0x4680, 0x8641,
+ 0x8201, 0x42C0, 0x4380, 0x8341, 0x4100, 0x81C1, 0x8081, 0x4040
+ };
+
+ while (len--)
+ crc = (((crc >> 8) & 0xffU) ^
+ crc16_table[(crc ^ *cp++) & 0xffU]) & 0x0000ffffU;
+ return crc;
+}
+
+static inline int __bnxt_re_set_vma_data(void *bnxt_re_uctx,
+ struct vm_area_struct *vma)
+{
+ return 0;
+}
+
+static inline bool bnxt_re_check_if_vlan_valid(struct bnxt_re_dev *rdev,
+ u16 vlan_id)
+{
+ bool ret = true;
+ /*
+ * Check if the vlan is configured in the host.
+ * If not configured, it can be a transparent
+ * VLAN. So dont report the vlan id.
+ */
+ return ret;
+}
+
+#endif
diff --git a/sys/dev/bnxt/bnxt_re/main.c b/sys/dev/bnxt/bnxt_re/main.c
new file mode 100644
index 000000000000..e6c6f754ea47
--- /dev/null
+++ b/sys/dev/bnxt/bnxt_re/main.c
@@ -0,0 +1,4467 @@
+/*
+ * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term
+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Description: Main component of the bnxt_re driver
+ */
+
+#include <linux/if_ether.h>
+#include <linux/module.h>
+#include <linux/errno.h>
+#include <linux/pci.h>
+#include <linux/dma-mapping.h>
+#include <linux/slab.h>
+#include <linux/sched.h>
+#include <linux/delay.h>
+#include <linux/fs.h>
+#include <rdma/ib_user_verbs.h>
+#include <rdma/ib_addr.h>
+#include <rdma/ib_cache.h>
+#include <dev/mlx5/port.h>
+#include <dev/mlx5/vport.h>
+#include <linux/list.h>
+#include <rdma/ib_smi.h>
+#include <rdma/ib_umem.h>
+#include <linux/in.h>
+#include <linux/etherdevice.h>
+
+#include "bnxt_re.h"
+#include "ib_verbs.h"
+#include "bnxt_re-abi.h"
+#include "bnxt.h"
+
+static char drv_version[] =
+ "Broadcom NetXtreme-C/E RoCE Driver " ROCE_DRV_MODULE_NAME \
+ " v" ROCE_DRV_MODULE_VERSION " (" ROCE_DRV_MODULE_RELDATE ")\n";
+
+#define BNXT_RE_DESC "Broadcom NetXtreme RoCE"
+#define BNXT_ADEV_NAME "if_bnxt"
+
+MODULE_DESCRIPTION("Broadcom NetXtreme-C/E RoCE Driver");
+MODULE_LICENSE("Dual BSD/GPL");
+MODULE_DEPEND(bnxt_re, linuxkpi, 1, 1, 1);
+MODULE_DEPEND(bnxt_re, ibcore, 1, 1, 1);
+MODULE_DEPEND(bnxt_re, if_bnxt, 1, 1, 1);
+MODULE_VERSION(bnxt_re, 1);
+
+
+DEFINE_MUTEX(bnxt_re_mutex); /* mutex lock for driver */
+
+static unsigned int restrict_mrs = 0;
+module_param(restrict_mrs, uint, 0);
+MODULE_PARM_DESC(restrict_mrs, " Restrict the no. of MRs 0 = 256K , 1 = 64K");
+
+unsigned int restrict_stats = 0;
+module_param(restrict_stats, uint, 0);
+MODULE_PARM_DESC(restrict_stats, "Restrict stats query frequency to ethtool coalesce value. Disabled by default");
+
+unsigned int enable_fc = 1;
+module_param(enable_fc, uint, 0);
+MODULE_PARM_DESC(enable_fc, "Enable default PFC, CC,ETS during driver load. 1 - fc enable, 0 - fc disable - Default is 1");
+
+unsigned int min_tx_depth = 1;
+module_param(min_tx_depth, uint, 0);
+MODULE_PARM_DESC(min_tx_depth, "Minimum TX depth - Default is 1");
+
+static uint8_t max_msix_vec[BNXT_RE_MAX_DEVICES] = {0};
+static unsigned int max_msix_vec_argc;
+module_param_array(max_msix_vec, byte, &max_msix_vec_argc, 0444);
+MODULE_PARM_DESC(max_msix_vec, "Max MSI-x vectors per PF (2 - 64) - Default is 64");
+
+unsigned int cmdq_shadow_qd = RCFW_CMD_NON_BLOCKING_SHADOW_QD;
+module_param_named(cmdq_shadow_qd, cmdq_shadow_qd, uint, 0644);
+MODULE_PARM_DESC(cmdq_shadow_qd, "Perf Stat Debug: Shadow QD Range (1-64) - Default is 64");
+
+
+/* globals */
+struct list_head bnxt_re_dev_list = LINUX_LIST_HEAD_INIT(bnxt_re_dev_list);
+static int bnxt_re_probe_count;
+
+DEFINE_MUTEX(bnxt_re_dev_lock);
+static u32 gmod_exit;
+static u32 gadd_dev_inprogress;
+
+static void bnxt_re_task(struct work_struct *work_task);
+static struct workqueue_struct *bnxt_re_wq;
+static int bnxt_re_query_hwrm_intf_version(struct bnxt_re_dev *rdev);
+static int bnxt_re_hwrm_qcfg(struct bnxt_re_dev *rdev, u32 *db_len,
+ u32 *offset);
+static int bnxt_re_ib_init(struct bnxt_re_dev *rdev);
+static void bnxt_re_ib_init_2(struct bnxt_re_dev *rdev);
+void _bnxt_re_remove(struct auxiliary_device *adev);
+void writel_fbsd(struct bnxt_softc *bp, u32, u8, u32);
+u32 readl_fbsd(struct bnxt_softc *bp, u32, u8);
+static int bnxt_re_hwrm_dbr_pacing_qcfg(struct bnxt_re_dev *rdev);
+
+int bnxt_re_register_netdevice_notifier(struct notifier_block *nb)
+{
+ int rc;
+ rc = register_netdevice_notifier(nb);
+ return rc;
+}
+
+int bnxt_re_unregister_netdevice_notifier(struct notifier_block *nb)
+{
+ int rc;
+ rc = unregister_netdevice_notifier(nb);
+ return rc;
+}
+
+void bnxt_re_set_dma_device(struct ib_device *ibdev, struct bnxt_re_dev *rdev)
+{
+ ibdev->dma_device = &rdev->en_dev->pdev->dev;
+}
+
+void bnxt_re_init_resolve_wq(struct bnxt_re_dev *rdev)
+{
+ rdev->resolve_wq = create_singlethread_workqueue("bnxt_re_resolve_wq");
+ INIT_LIST_HEAD(&rdev->mac_wq_list);
+}
+
+void bnxt_re_uninit_resolve_wq(struct bnxt_re_dev *rdev)
+{
+ struct bnxt_re_resolve_dmac_work *tmp_work = NULL, *tmp_st;
+ if (!rdev->resolve_wq)
+ return;
+ flush_workqueue(rdev->resolve_wq);
+ list_for_each_entry_safe(tmp_work, tmp_st, &rdev->mac_wq_list, list) {
+ list_del(&tmp_work->list);
+ kfree(tmp_work);
+ }
+ destroy_workqueue(rdev->resolve_wq);
+ rdev->resolve_wq = NULL;
+}
+
+u32 readl_fbsd(struct bnxt_softc *bp, u32 reg_off, u8 bar_idx)
+{
+
+ if (bar_idx)
+ return bus_space_read_8(bp->doorbell_bar.tag, bp->doorbell_bar.handle, reg_off);
+ else
+ return bus_space_read_8(bp->hwrm_bar.tag, bp->hwrm_bar.handle, reg_off);
+}
+
+void writel_fbsd(struct bnxt_softc *bp, u32 reg_off, u8 bar_idx, u32 val)
+{
+ if (bar_idx)
+ bus_space_write_8(bp->doorbell_bar.tag, bp->doorbell_bar.handle, reg_off, htole32(val));
+ else
+ bus_space_write_8(bp->hwrm_bar.tag, bp->hwrm_bar.handle, reg_off, htole32(val));
+}
+
+static void bnxt_re_update_fifo_occup_slabs(struct bnxt_re_dev *rdev,
+ u32 fifo_occup)
+{
+ if (fifo_occup > rdev->dbg_stats->dbq.fifo_occup_water_mark)
+ rdev->dbg_stats->dbq.fifo_occup_water_mark = fifo_occup;
+
+ if (fifo_occup > 8 * rdev->pacing_algo_th)
+ rdev->dbg_stats->dbq.fifo_occup_slab_4++;
+ else if (fifo_occup > 4 * rdev->pacing_algo_th)
+ rdev->dbg_stats->dbq.fifo_occup_slab_3++;
+ else if (fifo_occup > 2 * rdev->pacing_algo_th)
+ rdev->dbg_stats->dbq.fifo_occup_slab_2++;
+ else if (fifo_occup > rdev->pacing_algo_th)
+ rdev->dbg_stats->dbq.fifo_occup_slab_1++;
+}
+
+static void bnxt_re_update_do_pacing_slabs(struct bnxt_re_dev *rdev)
+{
+ struct bnxt_qplib_db_pacing_data *pacing_data = rdev->qplib_res.pacing_data;
+
+ if (pacing_data->do_pacing > rdev->dbg_stats->dbq.do_pacing_water_mark)
+ rdev->dbg_stats->dbq.do_pacing_water_mark = pacing_data->do_pacing;
+
+ if (pacing_data->do_pacing > 16 * rdev->dbr_def_do_pacing)
+ rdev->dbg_stats->dbq.do_pacing_slab_5++;
+ else if (pacing_data->do_pacing > 8 * rdev->dbr_def_do_pacing)
+ rdev->dbg_stats->dbq.do_pacing_slab_4++;
+ else if (pacing_data->do_pacing > 4 * rdev->dbr_def_do_pacing)
+ rdev->dbg_stats->dbq.do_pacing_slab_3++;
+ else if (pacing_data->do_pacing > 2 * rdev->dbr_def_do_pacing)
+ rdev->dbg_stats->dbq.do_pacing_slab_2++;
+ else if (pacing_data->do_pacing > rdev->dbr_def_do_pacing)
+ rdev->dbg_stats->dbq.do_pacing_slab_1++;
+}
+
+static bool bnxt_re_is_qp1_qp(struct bnxt_re_qp *qp)
+{
+ return qp->ib_qp.qp_type == IB_QPT_GSI;
+}
+
+static struct bnxt_re_qp *bnxt_re_get_qp1_qp(struct bnxt_re_dev *rdev)
+{
+ struct bnxt_re_qp *qp;
+
+ mutex_lock(&rdev->qp_lock);
+ list_for_each_entry(qp, &rdev->qp_list, list) {
+ if (bnxt_re_is_qp1_qp(qp)) {
+ mutex_unlock(&rdev->qp_lock);
+ return qp;
+ }
+ }
+ mutex_unlock(&rdev->qp_lock);
+ return NULL;
+}
+
+/* Set the maximum number of each resource that the driver actually wants
+ * to allocate. This may be up to the maximum number the firmware has
+ * reserved for the function. The driver may choose to allocate fewer
+ * resources than the firmware maximum.
+ */
+static void bnxt_re_limit_pf_res(struct bnxt_re_dev *rdev)
+{
+ struct bnxt_qplib_max_res dev_res = {};
+ struct bnxt_qplib_chip_ctx *cctx;
+ struct bnxt_qplib_dev_attr *attr;
+ struct bnxt_qplib_ctx *hctx;
+ int i;
+
+ attr = rdev->dev_attr;
+ hctx = rdev->qplib_res.hctx;
+ cctx = rdev->chip_ctx;
+
+ bnxt_qplib_max_res_supported(cctx, &rdev->qplib_res, &dev_res, false);
+ if (!_is_chip_gen_p5_p7(cctx)) {
+ hctx->qp_ctx.max = min_t(u32, dev_res.max_qp, attr->max_qp);
+ hctx->mrw_ctx.max = min_t(u32, dev_res.max_mr, attr->max_mr);
+ /* To accommodate 16k MRs and 16k AHs,
+ * driver has to allocate 32k backing store memory
+ */
+ hctx->mrw_ctx.max *= 2;
+ hctx->srq_ctx.max = min_t(u32, dev_res.max_srq, attr->max_srq);
+ hctx->cq_ctx.max = min_t(u32, dev_res.max_cq, attr->max_cq);
+ for (i = 0; i < MAX_TQM_ALLOC_REQ; i++)
+ hctx->tqm_ctx.qcount[i] = attr->tqm_alloc_reqs[i];
+ } else {
+ hctx->qp_ctx.max = attr->max_qp ? attr->max_qp : dev_res.max_qp;
+ hctx->mrw_ctx.max = attr->max_mr ? attr->max_mr : dev_res.max_mr;
+ hctx->srq_ctx.max = attr->max_srq ? attr->max_srq : dev_res.max_srq;
+ hctx->cq_ctx.max = attr->max_cq ? attr->max_cq : dev_res.max_cq;
+ }
+}
+
+static void bnxt_re_limit_vf_res(struct bnxt_re_dev *rdev,
+ struct bnxt_qplib_vf_res *vf_res,
+ u32 num_vf)
+{
+ struct bnxt_qplib_chip_ctx *cctx = rdev->chip_ctx;
+ struct bnxt_qplib_max_res dev_res = {};
+
+ bnxt_qplib_max_res_supported(cctx, &rdev->qplib_res, &dev_res, true);
+ vf_res->max_qp = dev_res.max_qp / num_vf;
+ vf_res->max_srq = dev_res.max_srq / num_vf;
+ vf_res->max_cq = dev_res.max_cq / num_vf;
+ /*
+ * MR and AH shares the same backing store, the value specified
+ * for max_mrw is split into half by the FW for MR and AH
+ */
+ vf_res->max_mrw = dev_res.max_mr * 2 / num_vf;
+ vf_res->max_gid = BNXT_RE_MAX_GID_PER_VF;
+}
+
+static void bnxt_re_set_resource_limits(struct bnxt_re_dev *rdev)
+{
+ struct bnxt_qplib_ctx *hctx;
+
+ hctx = rdev->qplib_res.hctx;
+ memset(&hctx->vf_res, 0, sizeof(struct bnxt_qplib_vf_res));
+ bnxt_re_limit_pf_res(rdev);
+
+ if (rdev->num_vfs)
+ bnxt_re_limit_vf_res(rdev, &hctx->vf_res, rdev->num_vfs);
+}
+
+static void bnxt_re_dettach_irq(struct bnxt_re_dev *rdev)
+{
+ struct bnxt_qplib_rcfw *rcfw = NULL;
+ struct bnxt_qplib_nq *nq;
+ int indx;
+
+ rcfw = &rdev->rcfw;
+ for (indx = 0; indx < rdev->nqr.max_init; indx++) {
+ nq = &rdev->nqr.nq[indx];
+ mutex_lock(&nq->lock);
+ bnxt_qplib_nq_stop_irq(nq, false);
+ mutex_unlock(&nq->lock);
+ }
+
+ bnxt_qplib_rcfw_stop_irq(rcfw, false);
+}
+
+static void bnxt_re_detach_err_device(struct bnxt_re_dev *rdev)
+{
+ /* Free the MSIx vectors only so that L2 can proceed with MSIx disable */
+ bnxt_re_dettach_irq(rdev);
+
+ /* Set the state as detached to prevent sending any more commands */
+ set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags);
+ set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags);
+ wake_up_all(&rdev->rcfw.cmdq.waitq);
+}
+
+#define MAX_DSCP_PRI_TUPLE 64
+
+struct bnxt_re_dcb_work {
+ struct work_struct work;
+ struct bnxt_re_dev *rdev;
+ struct hwrm_async_event_cmpl cmpl;
+};
+
+static void bnxt_re_init_dcb_wq(struct bnxt_re_dev *rdev)
+{
+ rdev->dcb_wq = create_singlethread_workqueue("bnxt_re_dcb_wq");
+}
+
+static void bnxt_re_uninit_dcb_wq(struct bnxt_re_dev *rdev)
+{
+ if (!rdev->dcb_wq)
+ return;
+ flush_workqueue(rdev->dcb_wq);
+ destroy_workqueue(rdev->dcb_wq);
+ rdev->dcb_wq = NULL;
+}
+
+static void bnxt_re_init_aer_wq(struct bnxt_re_dev *rdev)
+{
+ rdev->aer_wq = create_singlethread_workqueue("bnxt_re_aer_wq");
+}
+
+static void bnxt_re_uninit_aer_wq(struct bnxt_re_dev *rdev)
+{
+ if (!rdev->aer_wq)
+ return;
+ flush_workqueue(rdev->aer_wq);
+ destroy_workqueue(rdev->aer_wq);
+ rdev->aer_wq = NULL;
+}
+
+static int bnxt_re_update_qp1_tos_dscp(struct bnxt_re_dev *rdev)
+{
+ struct bnxt_re_qp *qp;
+
+ if (!_is_chip_gen_p5_p7(rdev->chip_ctx))
+ return 0;
+
+ qp = bnxt_re_get_qp1_qp(rdev);
+ if (!qp)
+ return 0;
+
+ qp->qplib_qp.modify_flags = CMDQ_MODIFY_QP_MODIFY_MASK_TOS_DSCP;
+ qp->qplib_qp.tos_dscp = rdev->cc_param.qp1_tos_dscp;
+
+ return bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp);
+}
+
+static void bnxt_re_reconfigure_dscp(struct bnxt_re_dev *rdev)
+{
+ struct bnxt_qplib_cc_param *cc_param;
+ struct bnxt_re_tc_rec *tc_rec;
+ bool update_cc = false;
+ u8 dscp_user;
+ int rc;
+
+ cc_param = &rdev->cc_param;
+ tc_rec = &rdev->tc_rec[0];
+
+ if (!(cc_param->roce_dscp_user || cc_param->cnp_dscp_user))
+ return;
+
+ if (cc_param->cnp_dscp_user) {
+ dscp_user = (cc_param->cnp_dscp_user & 0x3f);
+ if ((tc_rec->cnp_dscp_bv & (1ul << dscp_user)) &&
+ (cc_param->alt_tos_dscp != dscp_user)) {
+ cc_param->alt_tos_dscp = dscp_user;
+ cc_param->mask |= CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_TOS_DSCP;
+ update_cc = true;
+ }
+ }
+
+ if (cc_param->roce_dscp_user) {
+ dscp_user = (cc_param->roce_dscp_user & 0x3f);
+ if ((tc_rec->roce_dscp_bv & (1ul << dscp_user)) &&
+ (cc_param->tos_dscp != dscp_user)) {
+ cc_param->tos_dscp = dscp_user;
+ cc_param->mask |= CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP;
+ update_cc = true;
+ }
+ }
+
+ if (update_cc) {
+ rc = bnxt_qplib_modify_cc(&rdev->qplib_res, cc_param);
+ if (rc)
+ dev_err(rdev_to_dev(rdev), "Failed to apply cc settings\n");
+ }
+}
+
+static void bnxt_re_dcb_wq_task(struct work_struct *work)
+{
+ struct bnxt_qplib_cc_param *cc_param;
+ struct bnxt_re_tc_rec *tc_rec;
+ struct bnxt_re_dev *rdev;
+ struct bnxt_re_dcb_work *dcb_work =
+ container_of(work, struct bnxt_re_dcb_work, work);
+ int rc;
+
+ rdev = dcb_work->rdev;
+ if (!rdev)
+ goto exit;
+
+ mutex_lock(&rdev->cc_lock);
+
+ cc_param = &rdev->cc_param;
+ rc = bnxt_qplib_query_cc_param(&rdev->qplib_res, cc_param);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "Failed to query ccparam rc:%d", rc);
+ goto fail;
+ }
+ tc_rec = &rdev->tc_rec[0];
+ /*
+ * Upon the receival of DCB Async event:
+ * If roce_dscp or cnp_dscp or both (which user configured using configfs)
+ * is in the list, re-program the value using modify_roce_cc command
+ */
+ bnxt_re_reconfigure_dscp(rdev);
+
+ cc_param->roce_pri = tc_rec->roce_prio;
+ if (cc_param->qp1_tos_dscp != cc_param->tos_dscp) {
+ cc_param->qp1_tos_dscp = cc_param->tos_dscp;
+ rc = bnxt_re_update_qp1_tos_dscp(rdev);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "%s:Failed to modify QP1 rc:%d",
+ __func__, rc);
+ goto fail;
+ }
+ }
+
+fail:
+ mutex_unlock(&rdev->cc_lock);
+exit:
+ kfree(dcb_work);
+}
+
+static int bnxt_re_hwrm_dbr_pacing_broadcast_event(struct bnxt_re_dev *rdev)
+{
+ struct hwrm_func_dbr_pacing_broadcast_event_output resp = {0};
+ struct hwrm_func_dbr_pacing_broadcast_event_input req = {0};
+ struct bnxt_en_dev *en_dev = rdev->en_dev;
+ struct bnxt_fw_msg fw_msg;
+ int rc;
+
+ memset(&fw_msg, 0, sizeof(fw_msg));
+ bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
+ HWRM_FUNC_DBR_PACING_BROADCAST_EVENT, -1, -1);
+ bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
+ sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev));
+ rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
+ if (rc) {
+ dev_dbg(rdev_to_dev(rdev),
+ "Failed to send dbr pacing broadcast event rc:%d", rc);
+ return rc;
+ }
+ return 0;
+}
+
+static int bnxt_re_hwrm_dbr_pacing_nqlist_query(struct bnxt_re_dev *rdev)
+{
+ struct hwrm_func_dbr_pacing_nqlist_query_output resp = {0};
+ struct hwrm_func_dbr_pacing_nqlist_query_input req = {0};
+ struct bnxt_dbq_nq_list *nq_list = &rdev->nq_list;
+ struct bnxt_en_dev *en_dev = rdev->en_dev;
+ bool primary_found = false;
+ struct bnxt_fw_msg fw_msg;
+ struct bnxt_qplib_nq *nq;
+ int rc, i, j = 1;
+ u16 *nql_ptr;
+
+ nq = &rdev->nqr.nq[0];
+
+ memset(&fw_msg, 0, sizeof(fw_msg));
+ bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
+ HWRM_FUNC_DBR_PACING_NQLIST_QUERY, -1, -1);
+ bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
+ sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev));
+ rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "Failed to send dbr pacing nq list query rc:%d", rc);
+ return rc;
+ }
+ nq_list->num_nql_entries = le32_to_cpu(resp.num_nqs);
+ nql_ptr = &resp.nq_ring_id0;
+ /* populate the nq_list of the primary function with list received
+ * from FW. Fill the NQ IDs of secondary functions from index 1 to
+ * num_nql_entries - 1. Fill the nq_list->nq_id[0] with the
+ * nq_id of the primary pf
+ */
+ for (i = 0; i < nq_list->num_nql_entries; i++) {
+ u16 nq_id = *nql_ptr;
+
+ dev_dbg(rdev_to_dev(rdev),
+ "nq_list->nq_id[%d] = %d\n", i, nq_id);
+ if (nq_id != nq->ring_id) {
+ nq_list->nq_id[j] = nq_id;
+ j++;
+ } else {
+ primary_found = true;
+ nq_list->nq_id[0] = nq->ring_id;
+ }
+ nql_ptr++;
+ }
+ if (primary_found)
+ bnxt_qplib_dbr_pacing_set_primary_pf(rdev->chip_ctx, 1);
+
+ return 0;
+}
+
+static void __wait_for_fifo_occupancy_below_th(struct bnxt_re_dev *rdev)
+{
+ struct bnxt_qplib_db_pacing_data *pacing_data = rdev->qplib_res.pacing_data;
+ u32 read_val, fifo_occup;
+ bool first_read = true;
+
+ /* loop shouldn't run infintely as the occupancy usually goes
+ * below pacing algo threshold as soon as pacing kicks in.
+ */
+ while (1) {
+ read_val = readl_fbsd(rdev->en_dev->softc, rdev->dbr_db_fifo_reg_off, 0);
+ fifo_occup = pacing_data->fifo_max_depth -
+ ((read_val & pacing_data->fifo_room_mask) >>
+ pacing_data->fifo_room_shift);
+ /* Fifo occupancy cannot be greater the MAX FIFO depth */
+ if (fifo_occup > pacing_data->fifo_max_depth)
+ break;
+
+ if (first_read) {
+ bnxt_re_update_fifo_occup_slabs(rdev, fifo_occup);
+ first_read = false;
+ }
+ if (fifo_occup < pacing_data->pacing_th)
+ break;
+ }
+}
+
+static void bnxt_re_set_default_pacing_data(struct bnxt_re_dev *rdev)
+{
+ struct bnxt_qplib_db_pacing_data *pacing_data = rdev->qplib_res.pacing_data;
+
+ pacing_data->do_pacing = rdev->dbr_def_do_pacing;
+ pacing_data->pacing_th = rdev->pacing_algo_th;
+ pacing_data->alarm_th =
+ pacing_data->pacing_th * BNXT_RE_PACING_ALARM_TH_MULTIPLE(rdev->chip_ctx);
+}
+
+#define CAG_RING_MASK 0x7FF
+#define CAG_RING_SHIFT 17
+#define WATERMARK_MASK 0xFFF
+#define WATERMARK_SHIFT 0
+
+static bool bnxt_re_check_if_dbq_intr_triggered(struct bnxt_re_dev *rdev)
+{
+ u32 read_val;
+ int j;
+
+ for (j = 0; j < 10; j++) {
+ read_val = readl_fbsd(rdev->en_dev->softc, rdev->dbr_aeq_arm_reg_off, 0);
+ dev_dbg(rdev_to_dev(rdev), "AEQ ARM status = 0x%x\n",
+ read_val);
+ if (!read_val)
+ return true;
+ }
+ return false;
+}
+
+int bnxt_re_set_dbq_throttling_reg(struct bnxt_re_dev *rdev, u16 nq_id, u32 throttle)
+{
+ u32 cag_ring_water_mark = 0, read_val;
+ u32 throttle_val;
+
+ /* Convert throttle percentage to value */
+ throttle_val = (rdev->qplib_res.pacing_data->fifo_max_depth * throttle) / 100;
+
+ if (bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) {
+ cag_ring_water_mark = (nq_id & CAG_RING_MASK) << CAG_RING_SHIFT |
+ (throttle_val & WATERMARK_MASK);
+ writel_fbsd(rdev->en_dev->softc, rdev->dbr_throttling_reg_off, 0, cag_ring_water_mark);
+ read_val = readl_fbsd(rdev->en_dev->softc , rdev->dbr_throttling_reg_off, 0);
+ dev_dbg(rdev_to_dev(rdev),
+ "%s: dbr_throttling_reg_off read_val = 0x%x\n",
+ __func__, read_val);
+ if (read_val != cag_ring_water_mark) {
+ dev_dbg(rdev_to_dev(rdev),
+ "nq_id = %d write_val=0x%x read_val=0x%x\n",
+ nq_id, cag_ring_water_mark, read_val);
+ return 1;
+ }
+ }
+ writel_fbsd(rdev->en_dev->softc, rdev->dbr_aeq_arm_reg_off, 0, 1);
+ return 0;
+}
+
+static void bnxt_re_set_dbq_throttling_for_non_primary(struct bnxt_re_dev *rdev)
+{
+ struct bnxt_dbq_nq_list *nq_list;
+ struct bnxt_qplib_nq *nq;
+ int i;
+
+ nq_list = &rdev->nq_list;
+ /* Run a loop for other Active functions if this is primary function */
+ if (bnxt_qplib_dbr_pacing_is_primary_pf(rdev->chip_ctx)) {
+ dev_dbg(rdev_to_dev(rdev), "%s: nq_list->num_nql_entries= %d\n",
+ __func__, nq_list->num_nql_entries);
+ nq = &rdev->nqr.nq[0];
+ for (i = nq_list->num_nql_entries - 1; i > 0; i--) {
+ u16 nq_id = nq_list->nq_id[i];
+ if (nq)
+ dev_dbg(rdev_to_dev(rdev),
+ "%s: nq_id = %d cur_fn_ring_id = %d\n",
+ __func__, nq_id, nq->ring_id);
+ if (bnxt_re_set_dbq_throttling_reg
+ (rdev, nq_id, 0))
+ break;
+ bnxt_re_check_if_dbq_intr_triggered(rdev);
+ }
+ }
+}
+
+static void bnxt_re_handle_dbr_nq_pacing_notification(struct bnxt_re_dev *rdev)
+{
+ struct bnxt_qplib_nq *nq;
+ int rc = 0;
+
+ nq = &rdev->nqr.nq[0];
+
+ /* Query the NQ list*/
+ rc = bnxt_re_hwrm_dbr_pacing_nqlist_query(rdev);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev),
+ "Failed to Query NQ list rc= %d", rc);
+ return;
+ }
+ /*Configure GRC access for Throttling and aeq_arm register */
+ writel_fbsd(rdev->en_dev->softc, BNXT_GRCPF_REG_WINDOW_BASE_OUT + 28, 0,
+ rdev->chip_ctx->dbr_aeq_arm_reg & BNXT_GRC_BASE_MASK);
+
+ rdev->dbr_throttling_reg_off =
+ (rdev->chip_ctx->dbr_throttling_reg &
+ BNXT_GRC_OFFSET_MASK) + 0x8000;
+ rdev->dbr_aeq_arm_reg_off =
+ (rdev->chip_ctx->dbr_aeq_arm_reg &
+ BNXT_GRC_OFFSET_MASK) + 0x8000;
+
+ bnxt_re_set_dbq_throttling_reg(rdev, nq->ring_id, rdev->dbq_watermark);
+}
+
+static void bnxt_re_dbq_wq_task(struct work_struct *work)
+{
+ struct bnxt_re_dbq_work *dbq_work =
+ container_of(work, struct bnxt_re_dbq_work, work);
+ struct bnxt_re_dev *rdev;
+
+ rdev = dbq_work->rdev;
+
+ if (!rdev)
+ goto exit;
+ switch (dbq_work->event) {
+ case BNXT_RE_DBQ_EVENT_SCHED:
+ dev_dbg(rdev_to_dev(rdev), "%s: Handle DBQ Pacing event\n",
+ __func__);
+ if (!bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx))
+ bnxt_re_hwrm_dbr_pacing_broadcast_event(rdev);
+ else
+ bnxt_re_pacing_alert(rdev);
+ break;
+ case BNXT_RE_DBR_PACING_EVENT:
+ dev_dbg(rdev_to_dev(rdev), "%s: Sched interrupt/pacing worker\n",
+ __func__);
+ if (_is_chip_p7(rdev->chip_ctx))
+ bnxt_re_pacing_alert(rdev);
+ else if (!rdev->chip_ctx->modes.dbr_pacing_v0)
+ bnxt_re_hwrm_dbr_pacing_qcfg(rdev);
+ break;
+ case BNXT_RE_DBR_NQ_PACING_NOTIFICATION:
+ bnxt_re_handle_dbr_nq_pacing_notification(rdev);
+ /* Issue a broadcast event to notify other functions
+ * that primary changed
+ */
+ bnxt_re_hwrm_dbr_pacing_broadcast_event(rdev);
+ break;
+ }
+exit:
+ kfree(dbq_work);
+}
+
+static void bnxt_re_async_notifier(void *handle, struct hwrm_async_event_cmpl *cmpl)
+{
+ struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle);
+ struct bnxt_re_dcb_work *dcb_work;
+ struct bnxt_re_dbq_work *dbq_work;
+ struct bnxt_re_dev *rdev;
+ u16 event_id;
+ u32 data1;
+ u32 data2 = 0;
+
+ if (!cmpl) {
+ pr_err("Async event, bad completion\n");
+ return;
+ }
+
+ if (!en_info || !en_info->en_dev) {
+ pr_err("Async event, bad en_info or en_dev\n");
+ return;
+ }
+ rdev = en_info->rdev;
+
+ event_id = le16_to_cpu(cmpl->event_id);
+ data1 = le32_to_cpu(cmpl->event_data1);
+ data2 = le32_to_cpu(cmpl->event_data2);
+
+ if (!rdev || !rdev_to_dev(rdev)) {
+ dev_dbg(NULL, "Async event, bad rdev or netdev\n");
+ return;
+ }
+
+ if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags) ||
+ !test_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags)) {
+ dev_dbg(NULL, "Async event, device already detached\n");
+ return;
+ }
+ if (data2 >= 0)
+ dev_dbg(rdev_to_dev(rdev), "Async event_id = %d data1 = %d data2 = %d",
+ event_id, data1, data2);
+
+ switch (event_id) {
+ case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DCB_CONFIG_CHANGE:
+ /* Not handling the event in older FWs */
+ if (!is_qport_service_type_supported(rdev))
+ break;
+ if (!rdev->dcb_wq)
+ break;
+ dcb_work = kzalloc(sizeof(*dcb_work), GFP_ATOMIC);
+ if (!dcb_work)
+ break;
+
+ dcb_work->rdev = rdev;
+ memcpy(&dcb_work->cmpl, cmpl, sizeof(*cmpl));
+ INIT_WORK(&dcb_work->work, bnxt_re_dcb_wq_task);
+ queue_work(rdev->dcb_wq, &dcb_work->work);
+ break;
+ case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY:
+ if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) {
+ /* Set rcfw flag to control commands send to Bono */
+ set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags);
+ /* Set bnxt_re flag to control commands send via L2 driver */
+ set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags);
+ wake_up_all(&rdev->rcfw.cmdq.waitq);
+ }
+ break;
+ case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_THRESHOLD:
+ if (!rdev->dbr_pacing)
+ break;
+ dbq_work = kzalloc(sizeof(*dbq_work), GFP_ATOMIC);
+ if (!dbq_work)
+ goto unlock;
+ dbq_work->rdev = rdev;
+ dbq_work->event = BNXT_RE_DBR_PACING_EVENT;
+ INIT_WORK(&dbq_work->work, bnxt_re_dbq_wq_task);
+ queue_work(rdev->dbq_wq, &dbq_work->work);
+ rdev->dbr_sw_stats->dbq_int_recv++;
+ break;
+ case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_NQ_UPDATE:
+ if (!rdev->dbr_pacing)
+ break;
+
+ dbq_work = kzalloc(sizeof(*dbq_work), GFP_ATOMIC);
+ if (!dbq_work)
+ goto unlock;
+ dbq_work->rdev = rdev;
+ dbq_work->event = BNXT_RE_DBR_NQ_PACING_NOTIFICATION;
+ INIT_WORK(&dbq_work->work, bnxt_re_dbq_wq_task);
+ queue_work(rdev->dbq_wq, &dbq_work->work);
+ break;
+
+ default:
+ break;
+ }
+unlock:
+ return;
+}
+
+static void bnxt_re_db_fifo_check(struct work_struct *work)
+{
+ struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev,
+ dbq_fifo_check_work);
+ struct bnxt_qplib_db_pacing_data *pacing_data;
+ u32 pacing_save;
+
+ if (!mutex_trylock(&rdev->dbq_lock))
+ return;
+ pacing_data = rdev->qplib_res.pacing_data;
+ pacing_save = rdev->do_pacing_save;
+ __wait_for_fifo_occupancy_below_th(rdev);
+ cancel_delayed_work_sync(&rdev->dbq_pacing_work);
+ if (rdev->dbr_recovery_on)
+ goto recovery_on;
+ if (pacing_save > rdev->dbr_def_do_pacing) {
+ /* Double the do_pacing value during the congestion */
+ pacing_save = pacing_save << 1;
+ } else {
+ /*
+ * when a new congestion is detected increase the do_pacing
+ * by 8 times. And also increase the pacing_th by 4 times. The
+ * reason to increase pacing_th is to give more space for the
+ * queue to oscillate down without getting empty, but also more
+ * room for the queue to increase without causing another alarm.
+ */
+ pacing_save = pacing_save << 3;
+ pacing_data->pacing_th = rdev->pacing_algo_th * 4;
+ }
+
+ if (pacing_save > BNXT_RE_MAX_DBR_DO_PACING)
+ pacing_save = BNXT_RE_MAX_DBR_DO_PACING;
+
+ pacing_data->do_pacing = pacing_save;
+ rdev->do_pacing_save = pacing_data->do_pacing;
+ pacing_data->alarm_th =
+ pacing_data->pacing_th * BNXT_RE_PACING_ALARM_TH_MULTIPLE(rdev->chip_ctx);
+recovery_on:
+ schedule_delayed_work(&rdev->dbq_pacing_work,
+ msecs_to_jiffies(rdev->dbq_pacing_time));
+ rdev->dbr_sw_stats->dbq_pacing_alerts++;
+ mutex_unlock(&rdev->dbq_lock);
+}
+
+static void bnxt_re_pacing_timer_exp(struct work_struct *work)
+{
+ struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev,
+ dbq_pacing_work.work);
+ struct bnxt_qplib_db_pacing_data *pacing_data;
+ u32 read_val, fifo_occup;
+ struct bnxt_qplib_nq *nq;
+
+ if (!mutex_trylock(&rdev->dbq_lock))
+ return;
+
+ pacing_data = rdev->qplib_res.pacing_data;
+ read_val = readl_fbsd(rdev->en_dev->softc , rdev->dbr_db_fifo_reg_off, 0);
+ fifo_occup = pacing_data->fifo_max_depth -
+ ((read_val & pacing_data->fifo_room_mask) >>
+ pacing_data->fifo_room_shift);
+
+ if (fifo_occup > pacing_data->pacing_th)
+ goto restart_timer;
+
+ /*
+ * Instead of immediately going back to the default do_pacing
+ * reduce it by 1/8 times and restart the timer.
+ */
+ pacing_data->do_pacing = pacing_data->do_pacing - (pacing_data->do_pacing >> 3);
+ pacing_data->do_pacing = max_t(u32, rdev->dbr_def_do_pacing, pacing_data->do_pacing);
+ /*
+ * If the fifo_occup is less than the interrupt enable threshold
+ * enable the interrupt on the primary PF.
+ */
+ if (rdev->dbq_int_disable && fifo_occup < rdev->pacing_en_int_th) {
+ if (bnxt_qplib_dbr_pacing_is_primary_pf(rdev->chip_ctx)) {
+ if (!rdev->chip_ctx->modes.dbr_pacing_v0) {
+ nq = &rdev->nqr.nq[0];
+ bnxt_re_set_dbq_throttling_reg(rdev, nq->ring_id,
+ rdev->dbq_watermark);
+ rdev->dbr_sw_stats->dbq_int_en++;
+ rdev->dbq_int_disable = false;
+ }
+ }
+ }
+ if (pacing_data->do_pacing <= rdev->dbr_def_do_pacing) {
+ bnxt_re_set_default_pacing_data(rdev);
+ rdev->dbr_sw_stats->dbq_pacing_complete++;
+ goto dbq_unlock;
+ }
+restart_timer:
+ schedule_delayed_work(&rdev->dbq_pacing_work,
+ msecs_to_jiffies(rdev->dbq_pacing_time));
+ bnxt_re_update_do_pacing_slabs(rdev);
+ rdev->dbr_sw_stats->dbq_pacing_resched++;
+dbq_unlock:
+ rdev->do_pacing_save = pacing_data->do_pacing;
+ mutex_unlock(&rdev->dbq_lock);
+}
+
+void bnxt_re_pacing_alert(struct bnxt_re_dev *rdev)
+{
+ struct bnxt_qplib_db_pacing_data *pacing_data;
+
+ if (!rdev->dbr_pacing)
+ return;
+ mutex_lock(&rdev->dbq_lock);
+ pacing_data = rdev->qplib_res.pacing_data;
+
+ /*
+ * Increase the alarm_th to max so that other user lib instances do not
+ * keep alerting the driver.
+ */
+ pacing_data->alarm_th = pacing_data->fifo_max_depth;
+ pacing_data->do_pacing = BNXT_RE_MAX_DBR_DO_PACING;
+ cancel_work_sync(&rdev->dbq_fifo_check_work);
+ schedule_work(&rdev->dbq_fifo_check_work);
+ mutex_unlock(&rdev->dbq_lock);
+}
+
+void bnxt_re_schedule_dbq_event(struct bnxt_qplib_res *res)
+{
+ struct bnxt_re_dbq_work *dbq_work;
+ struct bnxt_re_dev *rdev;
+
+ rdev = container_of(res, struct bnxt_re_dev, qplib_res);
+
+ atomic_set(&rdev->dbq_intr_running, 1);
+
+ if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags))
+ goto exit;
+ /* Run the loop to send dbq event to other functions
+ * for newer FW
+ */
+ if (bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx) &&
+ !rdev->chip_ctx->modes.dbr_pacing_v0)
+ bnxt_re_set_dbq_throttling_for_non_primary(rdev);
+
+ dbq_work = kzalloc(sizeof(*dbq_work), GFP_ATOMIC);
+ if (!dbq_work)
+ goto exit;
+ dbq_work->rdev = rdev;
+ dbq_work->event = BNXT_RE_DBQ_EVENT_SCHED;
+ INIT_WORK(&dbq_work->work, bnxt_re_dbq_wq_task);
+ queue_work(rdev->dbq_wq, &dbq_work->work);
+ rdev->dbr_sw_stats->dbq_int_recv++;
+ rdev->dbq_int_disable = true;
+exit:
+ atomic_set(&rdev->dbq_intr_running, 0);
+}
+
+static void bnxt_re_free_msix(struct bnxt_re_dev *rdev)
+{
+ struct bnxt_en_dev *en_dev = rdev->en_dev;
+ int rc;
+
+ rc = en_dev->en_ops->bnxt_free_msix(rdev->en_dev, BNXT_ROCE_ULP);
+ if (rc)
+ dev_err(rdev_to_dev(rdev), "netdev %p free_msix failed! rc = 0x%x",
+ rdev->netdev, rc);
+}
+
+static int bnxt_re_request_msix(struct bnxt_re_dev *rdev)
+{
+ struct bnxt_en_dev *en_dev = rdev->en_dev;
+ int rc = 0, num_msix_want, num_msix_got;
+ struct bnxt_msix_entry *entry;
+
+ /*
+ * Request MSIx based on the function type. This is
+ * a temporory solution to enable max VFs when NPAR is
+ * enabled.
+ * TODO - change the scheme with an adapter specific check
+ * as the latest adapters can support more NQs. For now
+ * this change satisfy all adapter versions.
+ */
+
+ if (rdev->is_virtfn)
+ num_msix_want = BNXT_RE_MAX_MSIX_VF;
+ else if (BNXT_EN_NPAR(en_dev))
+ num_msix_want = BNXT_RE_MAX_MSIX_NPAR_PF;
+ else if (_is_chip_gen_p5_p7(rdev->chip_ctx))
+ num_msix_want = rdev->num_msix_requested ?: BNXT_RE_MAX_MSIX_GEN_P5_PF;
+ else
+ num_msix_want = BNXT_RE_MAX_MSIX_PF;
+
+ /*
+ * Since MSIX vectors are used for both NQs and CREQ, we should try to
+ * allocate num_online_cpus + 1 by taking into account the CREQ. This
+ * leaves the number of MSIX vectors for NQs match the number of CPUs
+ * and allows the system to be fully utilized
+ */
+ num_msix_want = min_t(u32, num_msix_want, num_online_cpus() + 1);
+ num_msix_want = min_t(u32, num_msix_want, BNXT_RE_MAX_MSIX);
+ num_msix_want = max_t(u32, num_msix_want, BNXT_RE_MIN_MSIX);
+
+ entry = rdev->nqr.msix_entries;
+
+ num_msix_got = en_dev->en_ops->bnxt_request_msix(en_dev, BNXT_ROCE_ULP,
+ entry, num_msix_want);
+ if (num_msix_got < BNXT_RE_MIN_MSIX) {
+ rc = -EINVAL;
+ goto done;
+ }
+ if (num_msix_got != num_msix_want)
+ dev_warn(rdev_to_dev(rdev),
+ "bnxt_request_msix: wanted %d vectors, got %d\n",
+ num_msix_want, num_msix_got);
+
+ rdev->nqr.num_msix = num_msix_got;
+ return 0;
+done:
+ if (num_msix_got)
+ bnxt_re_free_msix(rdev);
+ return rc;
+}
+
+static int __wait_for_ib_unregister(struct bnxt_re_dev *rdev,
+ struct bnxt_re_en_dev_info *en_info)
+{
+ u64 timeout = 0;
+ u32 cur_prod = 0, cur_cons = 0;
+ int retry = 0, rc = 0, ret = 0;
+
+ cur_prod = rdev->rcfw.cmdq.hwq.prod;
+ cur_cons = rdev->rcfw.cmdq.hwq.cons;
+ timeout = msecs_to_jiffies(BNXT_RE_RECOVERY_IB_UNINIT_WAIT_TIME_MS);
+ retry = BNXT_RE_RECOVERY_IB_UNINIT_WAIT_RETRY;
+ /* During module exit, increase timeout ten-fold to 100 mins to wait
+ * as long as possible for ib_unregister() to complete
+ */
+ if (rdev->mod_exit)
+ retry *= 10;
+ do {
+ /*
+ * Since the caller of this function invokes with bnxt_re_mutex held,
+ * release it to avoid holding a lock while in wait / sleep mode.
+ */
+ mutex_unlock(&bnxt_re_mutex);
+ rc = wait_event_timeout(en_info->waitq,
+ en_info->ib_uninit_done,
+ timeout);
+ mutex_lock(&bnxt_re_mutex);
+
+ if (!bnxt_re_is_rdev_valid(rdev))
+ break;
+
+ if (rc)
+ break;
+
+ if (!RCFW_NO_FW_ACCESS(&rdev->rcfw)) {
+ /* No need to check for cmdq stall during module exit,
+ * wait for ib unregister to complete
+ */
+ if (!rdev->mod_exit)
+ ret = __check_cmdq_stall(&rdev->rcfw, &cur_prod, &cur_cons);
+ if (ret || en_info->ib_uninit_done)
+ break;
+ }
+ } while (retry--);
+
+ return rc;
+}
+
+static int bnxt_re_handle_start(struct auxiliary_device *adev)
+{
+ struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(adev);
+ struct bnxt_re_dev *rdev = NULL;
+ struct ifnet *real_dev;
+ struct bnxt_en_dev *en_dev;
+ struct ifnet *netdev;
+ int rc = 0;
+
+ if (!en_info || !en_info->en_dev) {
+ pr_err("Start, bad en_info or en_dev\n");
+ return -EINVAL;
+ }
+ netdev = en_info->en_dev->net;
+ if (en_info->rdev) {
+ dev_info(rdev_to_dev(en_info->rdev),
+ "%s: Device is already added adev %p rdev: %p\n",
+ __func__, adev, en_info->rdev);
+ return 0;
+ }
+
+ en_dev = en_info->en_dev;
+ real_dev = rdma_vlan_dev_real_dev(netdev);
+ if (!real_dev)
+ real_dev = netdev;
+ rc = bnxt_re_add_device(&rdev, real_dev,
+ en_info->gsi_mode,
+ BNXT_RE_POST_RECOVERY_INIT,
+ en_info->wqe_mode,
+ en_info->num_msix_requested, adev);
+ if (rc) {
+ /* Add device failed. Unregister the device.
+ * This has to be done explicitly as
+ * bnxt_re_stop would not have unregistered
+ */
+ rtnl_lock();
+ en_dev->en_ops->bnxt_unregister_device(en_dev, BNXT_ROCE_ULP);
+ rtnl_unlock();
+ mutex_lock(&bnxt_re_dev_lock);
+ gadd_dev_inprogress--;
+ mutex_unlock(&bnxt_re_dev_lock);
+ return rc;
+ }
+ rdev->adev = adev;
+ rtnl_lock();
+ bnxt_re_get_link_speed(rdev);
+ rtnl_unlock();
+ rc = bnxt_re_ib_init(rdev);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "Failed ib_init\n");
+ return rc;
+ }
+ bnxt_re_ib_init_2(rdev);
+
+ return rc;
+}
+
+static void bnxt_re_stop(void *handle)
+{
+ struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle);
+ struct ifnet *netdev;
+ struct bnxt_re_dev *rdev;
+ struct bnxt_en_dev *en_dev;
+ int rc = 0;
+
+ rtnl_unlock();
+ mutex_lock(&bnxt_re_mutex);
+ if (!en_info || !en_info->en_dev) {
+ pr_err("Stop, bad en_info or en_dev\n");
+ goto exit;
+ }
+ netdev = en_info->en_dev->net;
+ rdev = en_info->rdev;
+ if (!rdev)
+ goto exit;
+
+ if (!bnxt_re_is_rdev_valid(rdev))
+ goto exit;
+
+ /*
+ * Check if fw has undergone reset or is in a fatal condition.
+ * If so, set flags so that no further commands are sent down to FW
+ */
+ en_dev = rdev->en_dev;
+ if (en_dev->en_state & BNXT_STATE_FW_FATAL_COND ||
+ en_dev->en_state & BNXT_STATE_FW_RESET_DET) {
+ /* Set rcfw flag to control commands send to Bono */
+ set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags);
+ /* Set bnxt_re flag to control commands send via L2 driver */
+ set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags);
+ wake_up_all(&rdev->rcfw.cmdq.waitq);
+ }
+
+ if (test_bit(BNXT_RE_FLAG_STOP_IN_PROGRESS, &rdev->flags))
+ goto exit;
+ set_bit(BNXT_RE_FLAG_STOP_IN_PROGRESS, &rdev->flags);
+
+ en_info->wqe_mode = rdev->chip_ctx->modes.wqe_mode;
+ en_info->gsi_mode = rdev->gsi_ctx.gsi_qp_mode;
+ en_info->num_msix_requested = rdev->num_msix_requested;
+ en_info->ib_uninit_done = false;
+
+ if (rdev->dbr_pacing)
+ bnxt_re_set_pacing_dev_state(rdev);
+
+ dev_info(rdev_to_dev(rdev), "%s: L2 driver notified to stop."
+ "Attempting to stop and Dispatching event "
+ "to inform the stack\n", __func__);
+ init_waitqueue_head(&en_info->waitq);
+ /* Schedule a work item to handle IB UNINIT for recovery */
+ bnxt_re_schedule_work(rdev, NETDEV_UNREGISTER,
+ NULL, netdev, rdev->adev);
+ rc = __wait_for_ib_unregister(rdev, en_info);
+ if (!bnxt_re_is_rdev_valid(rdev))
+ goto exit;
+ if (!rc) {
+ dev_info(rdev_to_dev(rdev), "%s: Attempt to stop failed\n",
+ __func__);
+ bnxt_re_detach_err_device(rdev);
+ goto exit;
+ }
+ bnxt_re_remove_device(rdev, BNXT_RE_PRE_RECOVERY_REMOVE, rdev->adev);
+exit:
+ mutex_unlock(&bnxt_re_mutex);
+ /* Take rtnl_lock before return, bnxt_re_stop is called with rtnl_lock */
+ rtnl_lock();
+
+ return;
+}
+
+static void bnxt_re_start(void *handle)
+{
+ rtnl_unlock();
+ mutex_lock(&bnxt_re_mutex);
+ if (bnxt_re_handle_start((struct auxiliary_device *)handle))
+ pr_err("Failed to start RoCE device");
+ mutex_unlock(&bnxt_re_mutex);
+ /* Take rtnl_lock before return, bnxt_re_start is called with rtnl_lock */
+ rtnl_lock();
+ return;
+}
+
+static void bnxt_re_shutdown(void *p)
+{
+ struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(p);
+ struct bnxt_re_dev *rdev;
+
+ if (!en_info) {
+ pr_err("Shutdown, bad en_info\n");
+ return;
+ }
+ rtnl_unlock();
+ mutex_lock(&bnxt_re_mutex);
+ rdev = en_info->rdev;
+ if (!rdev || !bnxt_re_is_rdev_valid(rdev))
+ goto exit;
+
+ /* rtnl_lock held by L2 before coming here */
+ bnxt_re_stopqps_and_ib_uninit(rdev);
+ bnxt_re_remove_device(rdev, BNXT_RE_COMPLETE_REMOVE, rdev->adev);
+exit:
+ mutex_unlock(&bnxt_re_mutex);
+ rtnl_lock();
+ return;
+}
+
+static void bnxt_re_stop_irq(void *handle)
+{
+ struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle);
+ struct bnxt_qplib_rcfw *rcfw = NULL;
+ struct bnxt_re_dev *rdev;
+ struct bnxt_qplib_nq *nq;
+ int indx;
+
+ if (!en_info) {
+ pr_err("Stop irq, bad en_info\n");
+ return;
+ }
+ rdev = en_info->rdev;
+
+ if (!rdev)
+ return;
+
+ rcfw = &rdev->rcfw;
+ for (indx = 0; indx < rdev->nqr.max_init; indx++) {
+ nq = &rdev->nqr.nq[indx];
+ mutex_lock(&nq->lock);
+ bnxt_qplib_nq_stop_irq(nq, false);
+ mutex_unlock(&nq->lock);
+ }
+
+ if (test_bit(BNXT_RE_FLAG_ALLOC_RCFW, &rdev->flags))
+ bnxt_qplib_rcfw_stop_irq(rcfw, false);
+}
+
+static void bnxt_re_start_irq(void *handle, struct bnxt_msix_entry *ent)
+{
+ struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle);
+ struct bnxt_msix_entry *msix_ent = NULL;
+ struct bnxt_qplib_rcfw *rcfw = NULL;
+ struct bnxt_re_dev *rdev;
+ struct bnxt_qplib_nq *nq;
+ int indx, rc, vec;
+
+ if (!en_info) {
+ pr_err("Start irq, bad en_info\n");
+ return;
+ }
+ rdev = en_info->rdev;
+ if (!rdev)
+ return;
+ if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags))
+ return;
+ msix_ent = rdev->nqr.msix_entries;
+ rcfw = &rdev->rcfw;
+
+ if (!ent) {
+ /* Not setting the f/w timeout bit in rcfw.
+ * During the driver unload the first command
+ * to f/w will timeout and that will set the
+ * timeout bit.
+ */
+ dev_err(rdev_to_dev(rdev), "Failed to re-start IRQs\n");
+ return;
+ }
+
+ /* Vectors may change after restart, so update with new vectors
+ * in device structure.
+ */
+ for (indx = 0; indx < rdev->nqr.num_msix; indx++)
+ rdev->nqr.msix_entries[indx].vector = ent[indx].vector;
+
+ if (test_bit(BNXT_RE_FLAG_ALLOC_RCFW, &rdev->flags)) {
+ rc = bnxt_qplib_rcfw_start_irq(rcfw, msix_ent[BNXT_RE_AEQ_IDX].vector,
+ false);
+ if (rc) {
+ dev_warn(rdev_to_dev(rdev),
+ "Failed to reinit CREQ\n");
+ return;
+ }
+ }
+ for (indx = 0 ; indx < rdev->nqr.max_init; indx++) {
+ nq = &rdev->nqr.nq[indx];
+ vec = indx + 1;
+ rc = bnxt_qplib_nq_start_irq(nq, indx, msix_ent[vec].vector,
+ false);
+ if (rc) {
+ dev_warn(rdev_to_dev(rdev),
+ "Failed to reinit NQ index %d\n", indx);
+ return;
+ }
+ }
+}
+
+/*
+ * Except for ulp_async_notifier, the remaining ulp_ops
+ * below are called with rtnl_lock held
+ */
+static struct bnxt_ulp_ops bnxt_re_ulp_ops = {
+ .ulp_async_notifier = bnxt_re_async_notifier,
+ .ulp_stop = bnxt_re_stop,
+ .ulp_start = bnxt_re_start,
+ .ulp_shutdown = bnxt_re_shutdown,
+ .ulp_irq_stop = bnxt_re_stop_irq,
+ .ulp_irq_restart = bnxt_re_start_irq,
+};
+
+static inline const char *bnxt_re_netevent(unsigned long event)
+{
+ BNXT_RE_NETDEV_EVENT(event, NETDEV_UP);
+ BNXT_RE_NETDEV_EVENT(event, NETDEV_DOWN);
+ BNXT_RE_NETDEV_EVENT(event, NETDEV_CHANGE);
+ BNXT_RE_NETDEV_EVENT(event, NETDEV_REGISTER);
+ BNXT_RE_NETDEV_EVENT(event, NETDEV_UNREGISTER);
+ BNXT_RE_NETDEV_EVENT(event, NETDEV_CHANGEADDR);
+ return "Unknown";
+}
+
+/* RoCE -> Net driver */
+
+/* Driver registration routines used to let the networking driver (bnxt_en)
+ * to know that the RoCE driver is now installed */
+static void bnxt_re_unregister_netdev(struct bnxt_re_dev *rdev)
+{
+ struct bnxt_en_dev *en_dev = rdev->en_dev;
+ int rc;
+
+ rtnl_lock();
+ rc = en_dev->en_ops->bnxt_unregister_device(rdev->en_dev,
+ BNXT_ROCE_ULP);
+ rtnl_unlock();
+ if (rc)
+ dev_err(rdev_to_dev(rdev), "netdev %p unregister failed! rc = 0x%x",
+ rdev->en_dev->net, rc);
+
+ clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags);
+}
+
+static int bnxt_re_register_netdev(struct bnxt_re_dev *rdev)
+{
+ struct bnxt_en_dev *en_dev = rdev->en_dev;
+ int rc = 0;
+
+ rtnl_lock();
+ rc = en_dev->en_ops->bnxt_register_device(en_dev,
+ BNXT_ROCE_ULP,
+ &bnxt_re_ulp_ops,
+ rdev->adev);
+ rtnl_unlock();
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "netdev %p register failed! rc = 0x%x",
+ rdev->netdev, rc);
+ return rc;
+ }
+
+ return rc;
+}
+
+static void bnxt_re_set_db_offset(struct bnxt_re_dev *rdev)
+{
+ struct bnxt_qplib_chip_ctx *cctx;
+ struct bnxt_en_dev *en_dev;
+ struct bnxt_qplib_res *res;
+ u32 l2db_len = 0;
+ u32 offset = 0;
+ u32 barlen;
+ int rc;
+
+ res = &rdev->qplib_res;
+ en_dev = rdev->en_dev;
+ cctx = rdev->chip_ctx;
+
+ /* Issue qcfg */
+ rc = bnxt_re_hwrm_qcfg(rdev, &l2db_len, &offset);
+ if (rc)
+ dev_info(rdev_to_dev(rdev),
+ "Couldn't get DB bar size, Low latency framework is disabled\n");
+ /* set register offsets for both UC and WC */
+ if (_is_chip_p7(cctx))
+ res->dpi_tbl.ucreg.offset = offset;
+ else
+ res->dpi_tbl.ucreg.offset = res->is_vf ? BNXT_QPLIB_DBR_VF_DB_OFFSET :
+ BNXT_QPLIB_DBR_PF_DB_OFFSET;
+ res->dpi_tbl.wcreg.offset = res->dpi_tbl.ucreg.offset;
+
+ /* If WC mapping is disabled by L2 driver then en_dev->l2_db_size
+ * is equal to the DB-Bar actual size. This indicates that L2
+ * is mapping entire bar as UC-. RoCE driver can't enable WC mapping
+ * in such cases and DB-push will be disabled.
+ */
+ barlen = pci_resource_len(res->pdev, RCFW_DBR_PCI_BAR_REGION);
+ if (cctx->modes.db_push && l2db_len && en_dev->l2_db_size != barlen) {
+ res->dpi_tbl.wcreg.offset = en_dev->l2_db_size;
+ dev_info(rdev_to_dev(rdev),
+ "Low latency framework is enabled\n");
+ }
+
+ return;
+}
+
+static void bnxt_re_set_drv_mode(struct bnxt_re_dev *rdev, u8 mode)
+{
+ struct bnxt_qplib_chip_ctx *cctx;
+ struct bnxt_en_dev *en_dev;
+
+ en_dev = rdev->en_dev;
+ cctx = rdev->chip_ctx;
+ cctx->modes.wqe_mode = _is_chip_gen_p5_p7(rdev->chip_ctx) ?
+ mode : BNXT_QPLIB_WQE_MODE_STATIC;
+ cctx->modes.te_bypass = false;
+ if (bnxt_re_hwrm_qcaps(rdev))
+ dev_err(rdev_to_dev(rdev),
+ "Failed to query hwrm qcaps\n");
+ /*
+ * TODO: Need a better mechanism for spreading of the
+ * 512 extended PPP pages in the presence of VF and
+ * NPAR, until then not enabling push
+ */
+ if (_is_chip_p7(rdev->chip_ctx) && cctx->modes.db_push) {
+ if (rdev->is_virtfn || BNXT_EN_NPAR(en_dev))
+ cctx->modes.db_push = false;
+ }
+
+ rdev->roce_mode = en_dev->flags & BNXT_EN_FLAG_ROCE_CAP;
+ dev_dbg(rdev_to_dev(rdev),
+ "RoCE is supported on the device - caps:0x%x",
+ rdev->roce_mode);
+ if (!_is_chip_gen_p5_p7(rdev->chip_ctx))
+ rdev->roce_mode = BNXT_RE_FLAG_ROCEV2_CAP;
+ cctx->hw_stats_size = en_dev->hw_ring_stats_size;
+}
+
+static void bnxt_re_destroy_chip_ctx(struct bnxt_re_dev *rdev)
+{
+ struct bnxt_qplib_chip_ctx *chip_ctx;
+ struct bnxt_qplib_res *res;
+
+ if (!rdev->chip_ctx)
+ return;
+
+ res = &rdev->qplib_res;
+ bnxt_qplib_unmap_db_bar(res);
+
+ kfree(res->hctx);
+ res->rcfw = NULL;
+ kfree(rdev->dev_attr);
+ rdev->dev_attr = NULL;
+
+ chip_ctx = rdev->chip_ctx;
+ rdev->chip_ctx = NULL;
+ res->cctx = NULL;
+ res->hctx = NULL;
+ res->pdev = NULL;
+ res->netdev = NULL;
+ kfree(chip_ctx);
+}
+
+static int bnxt_re_setup_chip_ctx(struct bnxt_re_dev *rdev, u8 wqe_mode)
+{
+ struct bnxt_qplib_chip_ctx *chip_ctx;
+ struct bnxt_en_dev *en_dev;
+ int rc;
+
+ en_dev = rdev->en_dev;
+ /* Supply pci device to qplib */
+ rdev->qplib_res.pdev = en_dev->pdev;
+ rdev->qplib_res.netdev = rdev->netdev;
+ rdev->qplib_res.en_dev = en_dev;
+
+ chip_ctx = kzalloc(sizeof(*chip_ctx), GFP_KERNEL);
+ if (!chip_ctx)
+ return -ENOMEM;
+ rdev->chip_ctx = chip_ctx;
+ rdev->qplib_res.cctx = chip_ctx;
+ rc = bnxt_re_query_hwrm_intf_version(rdev);
+ if (rc)
+ goto fail;
+ rdev->dev_attr = kzalloc(sizeof(*rdev->dev_attr), GFP_KERNEL);
+ if (!rdev->dev_attr) {
+ rc = -ENOMEM;
+ goto fail;
+ }
+ rdev->qplib_res.dattr = rdev->dev_attr;
+ rdev->qplib_res.rcfw = &rdev->rcfw;
+ rdev->qplib_res.is_vf = rdev->is_virtfn;
+
+ rdev->qplib_res.hctx = kzalloc(sizeof(*rdev->qplib_res.hctx),
+ GFP_KERNEL);
+ if (!rdev->qplib_res.hctx) {
+ rc = -ENOMEM;
+ goto fail;
+ }
+ bnxt_re_set_drv_mode(rdev, wqe_mode);
+
+ bnxt_re_set_db_offset(rdev);
+ rc = bnxt_qplib_map_db_bar(&rdev->qplib_res);
+ if (rc)
+ goto fail;
+
+ rc = bnxt_qplib_enable_atomic_ops_to_root(en_dev->pdev);
+ if (rc)
+ dev_dbg(rdev_to_dev(rdev),
+ "platform doesn't support global atomics");
+
+ return 0;
+fail:
+ kfree(rdev->chip_ctx);
+ rdev->chip_ctx = NULL;
+
+ kfree(rdev->dev_attr);
+ rdev->dev_attr = NULL;
+
+ kfree(rdev->qplib_res.hctx);
+ rdev->qplib_res.hctx = NULL;
+ return rc;
+}
+
+static u16 bnxt_re_get_rtype(struct bnxt_re_dev *rdev) {
+ return _is_chip_gen_p5_p7(rdev->chip_ctx) ?
+ HWRM_RING_ALLOC_INPUT_RING_TYPE_NQ :
+ HWRM_RING_ALLOC_INPUT_RING_TYPE_ROCE_CMPL;
+}
+
+static int bnxt_re_net_ring_free(struct bnxt_re_dev *rdev, u16 fw_ring_id)
+{
+ int rc = -EINVAL;
+ struct hwrm_ring_free_input req = {0};
+ struct hwrm_ring_free_output resp;
+ struct bnxt_en_dev *en_dev = rdev->en_dev;
+ struct bnxt_fw_msg fw_msg;
+
+ if (!en_dev)
+ return rc;
+
+ /* To avoid unnecessary error messages during recovery.
+ * HW is anyway in error state. So dont send down the command */
+ if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags))
+ return 0;
+
+ /* allocation had failed, no need to issue hwrm */
+ if (fw_ring_id == 0xffff)
+ return 0;
+
+ memset(&fw_msg, 0, sizeof(fw_msg));
+
+ bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_RING_FREE, -1, -1);
+ req.ring_type = bnxt_re_get_rtype(rdev);
+ req.ring_id = cpu_to_le16(fw_ring_id);
+ bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
+ sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
+ rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev),
+ "Failed to free HW ring with rc = 0x%x", rc);
+ return rc;
+ }
+ dev_dbg(rdev_to_dev(rdev), "HW ring freed with id = 0x%x\n",
+ fw_ring_id);
+
+ return rc;
+}
+
+static int bnxt_re_net_ring_alloc(struct bnxt_re_dev *rdev,
+ struct bnxt_re_ring_attr *ring_attr,
+ u16 *fw_ring_id)
+{
+ int rc = -EINVAL;
+ struct hwrm_ring_alloc_input req = {0};
+ struct hwrm_ring_alloc_output resp;
+ struct bnxt_en_dev *en_dev = rdev->en_dev;
+ struct bnxt_fw_msg fw_msg;
+
+ if (!en_dev)
+ return rc;
+
+ memset(&fw_msg, 0, sizeof(fw_msg));
+ bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_RING_ALLOC, -1, -1);
+ req.flags = cpu_to_le16(ring_attr->flags);
+ req.enables = 0;
+ req.page_tbl_addr = cpu_to_le64(ring_attr->dma_arr[0]);
+ if (ring_attr->pages > 1) {
+ /* Page size is in log2 units */
+ req.page_size = BNXT_PAGE_SHIFT;
+ req.page_tbl_depth = 1;
+ } else {
+ req.page_size = 4;
+ req.page_tbl_depth = 0;
+ }
+
+ req.fbo = 0;
+ /* Association of ring index with doorbell index and MSIX number */
+ req.logical_id = cpu_to_le16(ring_attr->lrid);
+ req.length = cpu_to_le32(ring_attr->depth + 1);
+ req.ring_type = ring_attr->type;
+ req.int_mode = ring_attr->mode;
+ bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
+ sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
+ rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev),
+ "Failed to allocate HW ring with rc = 0x%x", rc);
+ return rc;
+ }
+ *fw_ring_id = le16_to_cpu(resp.ring_id);
+ dev_dbg(rdev_to_dev(rdev),
+ "HW ring allocated with id = 0x%x at slot 0x%x",
+ resp.ring_id, ring_attr->lrid);
+
+ return rc;
+}
+
+static int bnxt_re_net_stats_ctx_free(struct bnxt_re_dev *rdev,
+ u32 fw_stats_ctx_id, u16 tid)
+{
+ struct bnxt_en_dev *en_dev = rdev->en_dev;
+ struct hwrm_stat_ctx_free_input req = {0};
+ struct hwrm_stat_ctx_free_output resp;
+ struct bnxt_fw_msg fw_msg;
+ int rc = -EINVAL;
+
+ if (!en_dev)
+ return rc;
+
+ /* To avoid unnecessary error messages during recovery.
+ * HW is anyway in error state. So dont send down the command */
+ if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags))
+ return 0;
+ memset(&fw_msg, 0, sizeof(fw_msg));
+ bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_STAT_CTX_FREE, -1, tid);
+ req.stat_ctx_id = cpu_to_le32(fw_stats_ctx_id);
+ bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
+ sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
+ rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev),
+ "Failed to free HW stats ctx with rc = 0x%x", rc);
+ return rc;
+ }
+ dev_dbg(rdev_to_dev(rdev),
+ "HW stats ctx freed with id = 0x%x", fw_stats_ctx_id);
+
+ return rc;
+}
+
+static int bnxt_re_net_stats_ctx_alloc(struct bnxt_re_dev *rdev, u16 tid)
+{
+ struct hwrm_stat_ctx_alloc_output resp = {};
+ struct hwrm_stat_ctx_alloc_input req = {};
+ struct bnxt_en_dev *en_dev = rdev->en_dev;
+ struct bnxt_qplib_stats *stat;
+ struct bnxt_qplib_ctx *hctx;
+ struct bnxt_fw_msg fw_msg;
+ int rc = 0;
+
+ hctx = rdev->qplib_res.hctx;
+ stat = (tid == 0xffff) ? &hctx->stats : &hctx->stats2;
+ stat->fw_id = INVALID_STATS_CTX_ID;
+
+ if (!en_dev)
+ return -EINVAL;
+
+ memset(&fw_msg, 0, sizeof(fw_msg));
+ bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
+ HWRM_STAT_CTX_ALLOC, -1, tid);
+ req.update_period_ms = cpu_to_le32(1000);
+ req.stats_dma_length = rdev->chip_ctx->hw_stats_size;
+ req.stats_dma_addr = cpu_to_le64(stat->dma_map);
+ req.stat_ctx_flags = HWRM_STAT_CTX_ALLOC_INPUT_STAT_CTX_FLAGS_ROCE;
+ bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
+ sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
+ rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev),
+ "Failed to allocate HW stats ctx, rc = 0x%x", rc);
+ return rc;
+ }
+ stat->fw_id = le32_to_cpu(resp.stat_ctx_id);
+ dev_dbg(rdev_to_dev(rdev), "HW stats ctx allocated with id = 0x%x",
+ stat->fw_id);
+
+ return rc;
+}
+
+static void bnxt_re_net_unregister_async_event(struct bnxt_re_dev *rdev)
+{
+ const struct bnxt_en_ops *en_ops;
+
+ if (rdev->is_virtfn ||
+ test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags))
+ return;
+
+ memset(rdev->event_bitmap, 0, sizeof(rdev->event_bitmap));
+ en_ops = rdev->en_dev->en_ops;
+ if (en_ops->bnxt_register_fw_async_events
+ (rdev->en_dev, BNXT_ROCE_ULP,
+ (unsigned long *)rdev->event_bitmap,
+ HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_NQ_UPDATE))
+ dev_err(rdev_to_dev(rdev),
+ "Failed to unregister async event");
+}
+
+static void bnxt_re_net_register_async_event(struct bnxt_re_dev *rdev)
+{
+ const struct bnxt_en_ops *en_ops;
+
+ if (rdev->is_virtfn)
+ return;
+
+ rdev->event_bitmap[0] |=
+ BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DCB_CONFIG_CHANGE) |
+ BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY);
+
+ rdev->event_bitmap[2] |=
+ BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT - 64);
+ rdev->event_bitmap[2] |=
+ BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_THRESHOLD - 64) |
+ BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_NQ_UPDATE - 64);
+ en_ops = rdev->en_dev->en_ops;
+ if (en_ops->bnxt_register_fw_async_events
+ (rdev->en_dev, BNXT_ROCE_ULP,
+ (unsigned long *)rdev->event_bitmap,
+ HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_NQ_UPDATE))
+ dev_err(rdev_to_dev(rdev),
+ "Failed to reg Async event");
+}
+
+static int bnxt_re_query_hwrm_intf_version(struct bnxt_re_dev *rdev)
+{
+ struct bnxt_en_dev *en_dev = rdev->en_dev;
+ struct hwrm_ver_get_output resp = {0};
+ struct hwrm_ver_get_input req = {0};
+ struct bnxt_qplib_chip_ctx *cctx;
+ struct bnxt_fw_msg fw_msg;
+ int rc = 0;
+
+ memset(&fw_msg, 0, sizeof(fw_msg));
+ bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
+ HWRM_VER_GET, -1, -1);
+ req.hwrm_intf_maj = HWRM_VERSION_MAJOR;
+ req.hwrm_intf_min = HWRM_VERSION_MINOR;
+ req.hwrm_intf_upd = HWRM_VERSION_UPDATE;
+ bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
+ sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
+ rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev),
+ "Failed to query HW version, rc = 0x%x", rc);
+ return rc;
+ }
+ cctx = rdev->chip_ctx;
+ cctx->hwrm_intf_ver = (u64) le16_to_cpu(resp.hwrm_intf_major) << 48 |
+ (u64) le16_to_cpu(resp.hwrm_intf_minor) << 32 |
+ (u64) le16_to_cpu(resp.hwrm_intf_build) << 16 |
+ le16_to_cpu(resp.hwrm_intf_patch);
+
+ cctx->hwrm_cmd_max_timeout = le16_to_cpu(resp.max_req_timeout);
+
+ if (!cctx->hwrm_cmd_max_timeout)
+ cctx->hwrm_cmd_max_timeout = RCFW_FW_STALL_MAX_TIMEOUT;
+
+ cctx->chip_num = le16_to_cpu(resp.chip_num);
+ cctx->chip_rev = resp.chip_rev;
+ cctx->chip_metal = resp.chip_metal;
+ return 0;
+}
+
+/* Query device config using common hwrm */
+static int bnxt_re_hwrm_qcfg(struct bnxt_re_dev *rdev, u32 *db_len,
+ u32 *offset)
+{
+ struct bnxt_en_dev *en_dev = rdev->en_dev;
+ struct hwrm_func_qcfg_output resp = {0};
+ struct hwrm_func_qcfg_input req = {0};
+ struct bnxt_fw_msg fw_msg;
+ int rc;
+
+ memset(&fw_msg, 0, sizeof(fw_msg));
+ bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
+ HWRM_FUNC_QCFG, -1, -1);
+ req.fid = cpu_to_le16(0xffff);
+ bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
+ sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
+ rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev),
+ "Failed to query config, rc = %#x", rc);
+ return rc;
+ }
+
+ *db_len = PAGE_ALIGN(le16_to_cpu(resp.l2_doorbell_bar_size_kb) * 1024);
+ *offset = PAGE_ALIGN(le16_to_cpu(resp.legacy_l2_db_size_kb) * 1024);
+ return 0;
+}
+
+/* Query function capabilities using common hwrm */
+int bnxt_re_hwrm_qcaps(struct bnxt_re_dev *rdev)
+{
+ struct bnxt_en_dev *en_dev = rdev->en_dev;
+ struct hwrm_func_qcaps_output resp = {0};
+ struct hwrm_func_qcaps_input req = {0};
+ struct bnxt_qplib_chip_ctx *cctx;
+ struct bnxt_fw_msg fw_msg;
+ u8 push_enable = false;
+ int rc;
+
+ cctx = rdev->chip_ctx;
+ memset(&fw_msg, 0, sizeof(fw_msg));
+ bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
+ HWRM_FUNC_QCAPS, -1, -1);
+ req.fid = cpu_to_le16(0xffff);
+ bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
+ sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
+ rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev),
+ "Failed to query capabilities, rc = %#x", rc);
+ return rc;
+ }
+ if (_is_chip_p7(rdev->chip_ctx))
+ push_enable =
+ (resp.flags_ext &
+ HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT_PPP_PUSH_MODE_SUPPORTED) ?
+ true : false;
+ else
+ push_enable =
+ (resp.flags & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_WCB_PUSH_MODE) ?
+ true : false;
+ cctx->modes.db_push = push_enable;
+
+ cctx->modes.dbr_pacing =
+ resp.flags_ext & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT_DBR_PACING_SUPPORTED ?
+ true : false;
+ cctx->modes.dbr_pacing_ext =
+ resp.flags_ext2 &
+ HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT2_DBR_PACING_EXT_SUPPORTED ?
+ true : false;
+ cctx->modes.dbr_drop_recov =
+ (resp.flags_ext2 &
+ HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT2_SW_DBR_DROP_RECOVERY_SUPPORTED) ?
+ true : false;
+ cctx->modes.dbr_pacing_v0 =
+ (resp.flags_ext2 &
+ HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT2_DBR_PACING_V0_SUPPORTED) ?
+ true : false;
+ dev_dbg(rdev_to_dev(rdev),
+ "%s: cctx->modes.dbr_pacing = %d cctx->modes.dbr_pacing_ext = %d, dbr_drop_recov %d\n",
+ __func__, cctx->modes.dbr_pacing, cctx->modes.dbr_pacing_ext, cctx->modes.dbr_drop_recov);
+
+ return 0;
+}
+
+static int bnxt_re_hwrm_dbr_pacing_qcfg(struct bnxt_re_dev *rdev)
+{
+ struct bnxt_qplib_db_pacing_data *pacing_data = rdev->qplib_res.pacing_data;
+ struct hwrm_func_dbr_pacing_qcfg_output resp = {0};
+ struct hwrm_func_dbr_pacing_qcfg_input req = {0};
+ struct bnxt_en_dev *en_dev = rdev->en_dev;
+ struct bnxt_qplib_chip_ctx *cctx;
+ struct bnxt_fw_msg fw_msg;
+ u32 primary_nq_id;
+ int rc;
+
+ cctx = rdev->chip_ctx;
+ memset(&fw_msg, 0, sizeof(fw_msg));
+ bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
+ HWRM_FUNC_DBR_PACING_QCFG, -1, -1);
+ bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
+ sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev));
+ rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
+ if (rc) {
+ dev_dbg(rdev_to_dev(rdev),
+ "Failed to query dbr pacing config, rc = %#x", rc);
+ return rc;
+ }
+
+ primary_nq_id = le32_to_cpu(resp.primary_nq_id);
+ if (primary_nq_id == 0xffffffff &&
+ !bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) {
+ dev_err(rdev_to_dev(rdev), "%s:%d Invoke bnxt_qplib_dbr_pacing_set_primary_pf with 1\n",
+ __func__, __LINE__);
+ bnxt_qplib_dbr_pacing_set_primary_pf(rdev->chip_ctx, 1);
+ }
+
+ if (bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) {
+ struct bnxt_qplib_nq *nq;
+
+ nq = &rdev->nqr.nq[0];
+ /* Reset the primary capability */
+ if (nq->ring_id != primary_nq_id)
+ bnxt_qplib_dbr_pacing_set_primary_pf(rdev->chip_ctx, 0);
+ }
+
+ if ((resp.dbr_stat_db_fifo_reg &
+ HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_MASK) ==
+ HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_GRC)
+ cctx->dbr_stat_db_fifo =
+ resp.dbr_stat_db_fifo_reg &
+ ~HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_MASK;
+
+ if ((resp.dbr_throttling_aeq_arm_reg &
+ HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_THROTTLING_AEQ_ARM_REG_ADDR_SPACE_MASK)
+ == HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_THROTTLING_AEQ_ARM_REG_ADDR_SPACE_GRC) {
+ cctx->dbr_aeq_arm_reg = resp.dbr_throttling_aeq_arm_reg &
+ ~HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_MASK;
+ cctx->dbr_throttling_reg = cctx->dbr_aeq_arm_reg - 4;
+ }
+ pacing_data->fifo_max_depth = le32_to_cpu(resp.dbr_stat_db_max_fifo_depth);
+ if (!pacing_data->fifo_max_depth)
+ pacing_data->fifo_max_depth = BNXT_RE_MAX_FIFO_DEPTH(cctx);
+ pacing_data->fifo_room_mask = le32_to_cpu(resp.dbr_stat_db_fifo_reg_fifo_room_mask);
+ pacing_data->fifo_room_shift = resp.dbr_stat_db_fifo_reg_fifo_room_shift;
+ dev_dbg(rdev_to_dev(rdev),
+ "%s: nq:0x%x primary_pf:%d db_fifo:0x%x aeq_arm:0x%x i"
+ "fifo_max_depth 0x%x , resp.dbr_stat_db_max_fifo_depth 0x%x);\n",
+ __func__, resp.primary_nq_id, cctx->modes.dbr_primary_pf,
+ cctx->dbr_stat_db_fifo, cctx->dbr_aeq_arm_reg,
+ pacing_data->fifo_max_depth,
+ le32_to_cpu(resp.dbr_stat_db_max_fifo_depth));
+ return 0;
+}
+
+static int bnxt_re_hwrm_dbr_pacing_cfg(struct bnxt_re_dev *rdev, bool enable)
+{
+ struct hwrm_func_dbr_pacing_cfg_output resp = {0};
+ struct hwrm_func_dbr_pacing_cfg_input req = {0};
+ struct bnxt_en_dev *en_dev = rdev->en_dev;
+ struct bnxt_fw_msg fw_msg;
+ int rc;
+
+ if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags))
+ return 0;
+
+ memset(&fw_msg, 0, sizeof(fw_msg));
+ bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
+ HWRM_FUNC_DBR_PACING_CFG, -1, -1);
+ if (enable) {
+ req.flags = HWRM_FUNC_DBR_PACING_CFG_INPUT_FLAGS_DBR_NQ_EVENT_ENABLE;
+ req.enables =
+ cpu_to_le32(HWRM_FUNC_DBR_PACING_CFG_INPUT_ENABLES_PRIMARY_NQ_ID_VALID |
+ HWRM_FUNC_DBR_PACING_CFG_INPUT_ENABLES_PACING_THRESHOLD_VALID);
+ } else {
+ req.flags = HWRM_FUNC_DBR_PACING_CFG_INPUT_FLAGS_DBR_NQ_EVENT_DISABLE;
+ }
+ req.primary_nq_id = cpu_to_le32(rdev->dbq_nq_id);
+ req.pacing_threshold = cpu_to_le32(rdev->dbq_watermark);
+ dev_dbg(rdev_to_dev(rdev), "%s: nq_id = 0x%x pacing_threshold = 0x%x",
+ __func__, req.primary_nq_id, req.pacing_threshold);
+ bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
+ sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev));
+ rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
+ if (rc) {
+ dev_dbg(rdev_to_dev(rdev),
+ "Failed to set dbr pacing config, rc = %#x", rc);
+ return rc;
+ }
+ return 0;
+}
+
+/* Net -> RoCE driver */
+
+/* Device */
+struct bnxt_re_dev *bnxt_re_from_netdev(struct ifnet *netdev)
+{
+ struct bnxt_re_dev *rdev;
+
+ rcu_read_lock();
+ list_for_each_entry_rcu(rdev, &bnxt_re_dev_list, list) {
+ if (rdev->netdev == netdev) {
+ rcu_read_unlock();
+ dev_dbg(rdev_to_dev(rdev),
+ "netdev (%p) found, ref_count = 0x%x",
+ netdev, atomic_read(&rdev->ref_count));
+ return rdev;
+ }
+ }
+ rcu_read_unlock();
+ return NULL;
+}
+
+static ssize_t show_rev(struct device *device, struct device_attribute *attr,
+ char *buf)
+{
+ struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev);
+
+ return scnprintf(buf, PAGE_SIZE, "0x%x\n", rdev->en_dev->pdev->vendor);
+}
+
+
+static ssize_t show_hca(struct device *device, struct device_attribute *attr,
+ char *buf)
+{
+ struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev);
+
+ return scnprintf(buf, PAGE_SIZE, "%s\n", rdev->ibdev.node_desc);
+}
+
+static DEVICE_ATTR(hw_rev, 0444, show_rev, NULL);
+static DEVICE_ATTR(hca_type, 0444, show_hca, NULL);
+static struct device_attribute *bnxt_re_attributes[] = {
+ &dev_attr_hw_rev,
+ &dev_attr_hca_type
+};
+
+int ib_register_device_compat(struct bnxt_re_dev *rdev)
+{
+ struct ib_device *ibdev = &rdev->ibdev;
+ char name[IB_DEVICE_NAME_MAX];
+
+ memset(name, 0, IB_DEVICE_NAME_MAX);
+ strlcpy(name, "bnxt_re%d", IB_DEVICE_NAME_MAX);
+
+ strlcpy(ibdev->name, name, IB_DEVICE_NAME_MAX);
+
+ return ib_register_device(ibdev, NULL);
+}
+
+static int bnxt_re_register_ib(struct bnxt_re_dev *rdev)
+{
+ struct ib_device *ibdev = &rdev->ibdev;
+ int ret = 0;
+
+ /* ib device init */
+ ibdev->owner = THIS_MODULE;
+ ibdev->uverbs_abi_ver = BNXT_RE_ABI_VERSION;
+ ibdev->node_type = RDMA_NODE_IB_CA;
+ strlcpy(ibdev->node_desc, BNXT_RE_DESC " HCA",
+ strlen(BNXT_RE_DESC) + 5);
+ ibdev->phys_port_cnt = 1;
+
+ bnxt_qplib_get_guid(rdev->dev_addr, (u8 *)&ibdev->node_guid);
+
+ /* Data path irqs is one less than the max msix vectors */
+ ibdev->num_comp_vectors = rdev->nqr.num_msix - 1;
+ bnxt_re_set_dma_device(ibdev, rdev);
+ ibdev->local_dma_lkey = BNXT_QPLIB_RSVD_LKEY;
+
+ /* User space */
+ ibdev->uverbs_cmd_mask =
+ (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
+ (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
+ (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
+ (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
+ (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
+ (1ull << IB_USER_VERBS_CMD_REG_MR) |
+ (1ull << IB_USER_VERBS_CMD_DEREG_MR) |
+ (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
+ (1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
+ (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
+ (1ull << IB_USER_VERBS_CMD_CREATE_QP) |
+ (1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
+ (1ull << IB_USER_VERBS_CMD_QUERY_QP) |
+ (1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
+ (1ull << IB_USER_VERBS_CMD_REREG_MR) |
+ (1ull << IB_USER_VERBS_CMD_RESIZE_CQ) |
+ (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) |
+ (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) |
+ (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) |
+ (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ) |
+ (1ull << IB_USER_VERBS_CMD_ALLOC_MW) |
+ (1ull << IB_USER_VERBS_CMD_DEALLOC_MW) |
+ (1ull << IB_USER_VERBS_CMD_CREATE_AH) |
+ (1ull << IB_USER_VERBS_CMD_MODIFY_AH) |
+ (1ull << IB_USER_VERBS_CMD_QUERY_AH) |
+ (1ull << IB_USER_VERBS_CMD_DESTROY_AH);
+
+ ibdev->uverbs_ex_cmd_mask = (1ull << IB_USER_VERBS_EX_CMD_MODIFY_QP);
+ ibdev->uverbs_cmd_mask |= (1ull << IB_USER_VERBS_CMD_POLL_CQ);
+
+#define bnxt_re_ib_ah bnxt_re_ah
+#define bnxt_re_ib_cq bnxt_re_cq
+#define bnxt_re_ib_pd bnxt_re_pd
+#define bnxt_re_ib_srq bnxt_re_srq
+#define bnxt_re_ib_ucontext bnxt_re_ucontext
+ INIT_IB_DEVICE_OPS(&ibdev->ops, bnxt_re, BNXT_RE);
+
+ ibdev->query_device = bnxt_re_query_device;
+ ibdev->modify_device = bnxt_re_modify_device;
+ ibdev->query_port = bnxt_re_query_port;
+ ibdev->modify_port = bnxt_re_modify_port;
+ ibdev->get_port_immutable = bnxt_re_get_port_immutable;
+ ibdev->query_pkey = bnxt_re_query_pkey;
+ ibdev->query_gid = bnxt_re_query_gid;
+ ibdev->get_netdev = bnxt_re_get_netdev;
+ ibdev->add_gid = bnxt_re_add_gid;
+ ibdev->del_gid = bnxt_re_del_gid;
+ ibdev->get_link_layer = bnxt_re_get_link_layer;
+ ibdev->alloc_pd = bnxt_re_alloc_pd;
+ ibdev->dealloc_pd = bnxt_re_dealloc_pd;
+ ibdev->create_ah = bnxt_re_create_ah;
+ ibdev->modify_ah = bnxt_re_modify_ah;
+ ibdev->query_ah = bnxt_re_query_ah;
+ ibdev->destroy_ah = bnxt_re_destroy_ah;
+ ibdev->create_srq = bnxt_re_create_srq;
+ ibdev->modify_srq = bnxt_re_modify_srq;
+ ibdev->query_srq = bnxt_re_query_srq;
+ ibdev->destroy_srq = bnxt_re_destroy_srq;
+ ibdev->post_srq_recv = bnxt_re_post_srq_recv;
+ ibdev->create_qp = bnxt_re_create_qp;
+ ibdev->modify_qp = bnxt_re_modify_qp;
+ ibdev->query_qp = bnxt_re_query_qp;
+ ibdev->destroy_qp = bnxt_re_destroy_qp;
+ ibdev->post_send = bnxt_re_post_send;
+ ibdev->post_recv = bnxt_re_post_recv;
+ ibdev->create_cq = bnxt_re_create_cq;
+ ibdev->modify_cq = bnxt_re_modify_cq;
+ ibdev->destroy_cq = bnxt_re_destroy_cq;
+ ibdev->resize_cq = bnxt_re_resize_cq;
+ ibdev->poll_cq = bnxt_re_poll_cq;
+ ibdev->req_notify_cq = bnxt_re_req_notify_cq;
+ ibdev->get_dma_mr = bnxt_re_get_dma_mr;
+ ibdev->get_hw_stats = bnxt_re_get_hw_stats;
+ ibdev->alloc_hw_stats = bnxt_re_alloc_hw_port_stats;
+ ibdev->dereg_mr = bnxt_re_dereg_mr;
+ ibdev->alloc_mr = bnxt_re_alloc_mr;
+ ibdev->map_mr_sg = bnxt_re_map_mr_sg;
+ ibdev->alloc_mw = bnxt_re_alloc_mw;
+ ibdev->dealloc_mw = bnxt_re_dealloc_mw;
+ ibdev->reg_user_mr = bnxt_re_reg_user_mr;
+ ibdev->rereg_user_mr = bnxt_re_rereg_user_mr;
+ ibdev->disassociate_ucontext = bnxt_re_disassociate_ucntx;
+ ibdev->alloc_ucontext = bnxt_re_alloc_ucontext;
+ ibdev->dealloc_ucontext = bnxt_re_dealloc_ucontext;
+ ibdev->mmap = bnxt_re_mmap;
+ ibdev->process_mad = bnxt_re_process_mad;
+
+ ret = ib_register_device_compat(rdev);
+ return ret;
+}
+
+static void bnxt_re_dev_dealloc(struct bnxt_re_dev *rdev)
+{
+ int i = BNXT_RE_REF_WAIT_COUNT;
+
+ dev_dbg(rdev_to_dev(rdev), "%s:Remove the device %p\n", __func__, rdev);
+ /* Wait for rdev refcount to come down */
+ while ((atomic_read(&rdev->ref_count) > 1) && i--)
+ msleep(100);
+
+ if (atomic_read(&rdev->ref_count) > 1)
+ dev_err(rdev_to_dev(rdev),
+ "Failed waiting for ref count to deplete %d",
+ atomic_read(&rdev->ref_count));
+
+ atomic_set(&rdev->ref_count, 0);
+ if_rele(rdev->netdev);
+ rdev->netdev = NULL;
+ synchronize_rcu();
+
+ kfree(rdev->gid_map);
+ kfree(rdev->dbg_stats);
+ ib_dealloc_device(&rdev->ibdev);
+}
+
+static struct bnxt_re_dev *bnxt_re_dev_alloc(struct ifnet *netdev,
+ struct bnxt_en_dev *en_dev)
+{
+ struct bnxt_re_dev *rdev;
+ u32 count;
+
+ /* Allocate bnxt_re_dev instance here */
+ rdev = (struct bnxt_re_dev *)compat_ib_alloc_device(sizeof(*rdev));
+ if (!rdev) {
+ pr_err("%s: bnxt_re_dev allocation failure!",
+ ROCE_DRV_MODULE_NAME);
+ return NULL;
+ }
+ /* Default values */
+ atomic_set(&rdev->ref_count, 0);
+ rdev->netdev = netdev;
+ dev_hold(rdev->netdev);
+ rdev->en_dev = en_dev;
+ rdev->id = rdev->en_dev->pdev->devfn;
+ INIT_LIST_HEAD(&rdev->qp_list);
+ mutex_init(&rdev->qp_lock);
+ mutex_init(&rdev->cc_lock);
+ mutex_init(&rdev->dbq_lock);
+ bnxt_re_clear_rsors_stat(&rdev->stats.rsors);
+ rdev->cosq[0] = rdev->cosq[1] = 0xFFFF;
+ rdev->min_tx_depth = 1;
+ rdev->stats.stats_query_sec = 1;
+ /* Disable priority vlan as the default mode is DSCP based PFC */
+ rdev->cc_param.disable_prio_vlan_tx = 1;
+
+ /* Initialize worker for DBR Pacing */
+ INIT_WORK(&rdev->dbq_fifo_check_work, bnxt_re_db_fifo_check);
+ INIT_DELAYED_WORK(&rdev->dbq_pacing_work, bnxt_re_pacing_timer_exp);
+ rdev->gid_map = kzalloc(sizeof(*(rdev->gid_map)) *
+ BNXT_RE_MAX_SGID_ENTRIES,
+ GFP_KERNEL);
+ if (!rdev->gid_map) {
+ ib_dealloc_device(&rdev->ibdev);
+ return NULL;
+ }
+ for(count = 0; count < BNXT_RE_MAX_SGID_ENTRIES; count++)
+ rdev->gid_map[count] = -1;
+
+ rdev->dbg_stats = kzalloc(sizeof(*rdev->dbg_stats), GFP_KERNEL);
+ if (!rdev->dbg_stats) {
+ ib_dealloc_device(&rdev->ibdev);
+ return NULL;
+ }
+
+ return rdev;
+}
+
+static int bnxt_re_handle_unaffi_async_event(
+ struct creq_func_event *unaffi_async)
+{
+ switch (unaffi_async->event) {
+ case CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR:
+ case CREQ_FUNC_EVENT_EVENT_TX_DATA_ERROR:
+ case CREQ_FUNC_EVENT_EVENT_RX_WQE_ERROR:
+ case CREQ_FUNC_EVENT_EVENT_RX_DATA_ERROR:
+ case CREQ_FUNC_EVENT_EVENT_CQ_ERROR:
+ case CREQ_FUNC_EVENT_EVENT_TQM_ERROR:
+ case CREQ_FUNC_EVENT_EVENT_CFCQ_ERROR:
+ case CREQ_FUNC_EVENT_EVENT_CFCS_ERROR:
+ case CREQ_FUNC_EVENT_EVENT_CFCC_ERROR:
+ case CREQ_FUNC_EVENT_EVENT_CFCM_ERROR:
+ case CREQ_FUNC_EVENT_EVENT_TIM_ERROR:
+ break;
+ default:
+ return -EINVAL;
+ }
+ return 0;
+}
+
+static int bnxt_re_handle_qp_async_event(void *qp_event, struct bnxt_re_qp *qp)
+{
+ struct creq_qp_error_notification *err_event;
+ struct ib_event event;
+ unsigned int flags;
+
+ if (qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR &&
+ !qp->qplib_qp.is_user) {
+ flags = bnxt_re_lock_cqs(qp);
+ bnxt_qplib_add_flush_qp(&qp->qplib_qp);
+ bnxt_re_unlock_cqs(qp, flags);
+ }
+ memset(&event, 0, sizeof(event));
+ event.device = &qp->rdev->ibdev;
+ event.element.qp = &qp->ib_qp;
+ event.event = IB_EVENT_QP_FATAL;
+
+ err_event = qp_event;
+ switch(err_event->res_err_state_reason) {
+ case CFCQ_RES_ERR_STATE_REASON_RES_EXCEED_MAX:
+ case CFCQ_RES_ERR_STATE_REASON_RES_PAYLOAD_LENGTH_MISMATCH:
+ case CFCQ_RES_ERR_STATE_REASON_RES_OPCODE_ERROR:
+ case CFCQ_RES_ERR_STATE_REASON_RES_PSN_SEQ_ERROR_RETRY_LIMIT:
+ case CFCQ_RES_ERR_STATE_REASON_RES_RX_INVALID_R_KEY:
+ case CFCQ_RES_ERR_STATE_REASON_RES_RX_DOMAIN_ERROR:
+ case CFCQ_RES_ERR_STATE_REASON_RES_RX_NO_PERMISSION:
+ case CFCQ_RES_ERR_STATE_REASON_RES_RX_RANGE_ERROR:
+ case CFCQ_RES_ERR_STATE_REASON_RES_TX_INVALID_R_KEY:
+ case CFCQ_RES_ERR_STATE_REASON_RES_TX_DOMAIN_ERROR:
+ case CFCQ_RES_ERR_STATE_REASON_RES_TX_NO_PERMISSION:
+ case CFCQ_RES_ERR_STATE_REASON_RES_TX_RANGE_ERROR:
+ case CFCQ_RES_ERR_STATE_REASON_RES_IVALID_DUP_RKEY:
+ case CFCQ_RES_ERR_STATE_REASON_RES_UNALIGN_ATOMIC:
+ event.event = IB_EVENT_QP_ACCESS_ERR;
+ break;
+ case CFCQ_RES_ERR_STATE_REASON_RES_EXCEEDS_WQE:
+ case CFCQ_RES_ERR_STATE_REASON_RES_WQE_FORMAT_ERROR:
+ case CFCQ_RES_ERR_STATE_REASON_RES_SRQ_LOAD_ERROR:
+ case CFCQ_RES_ERR_STATE_REASON_RES_UNSUPPORTED_OPCODE:
+ case CFCQ_RES_ERR_STATE_REASON_RES_REM_INVALIDATE:
+ event.event = IB_EVENT_QP_REQ_ERR;
+ break;
+ case CFCQ_RES_ERR_STATE_REASON_RES_IRRQ_OFLOW:
+ case CFCQ_RES_ERR_STATE_REASON_RES_CMP_ERROR:
+ case CFCQ_RES_ERR_STATE_REASON_RES_CQ_LOAD_ERROR:
+ case CFCQ_RES_ERR_STATE_REASON_RES_TX_PCI_ERROR:
+ case CFCQ_RES_ERR_STATE_REASON_RES_RX_PCI_ERROR:
+ case CFCQ_RES_ERR_STATE_REASON_RES_MEMORY_ERROR:
+ case CFCQ_RES_ERR_STATE_REASON_RES_SRQ_ERROR:
+ event.event = IB_EVENT_QP_FATAL;
+ break;
+ default:
+ if (qp->qplib_qp.srq)
+ event.event = IB_EVENT_QP_LAST_WQE_REACHED;
+ break;
+ }
+
+ if (err_event->res_err_state_reason)
+ dev_err(rdev_to_dev(qp->rdev),
+ "%s %s qp_id: %d cons (%d %d) req (%d %d) res (%d %d)\n",
+ __func__, qp->qplib_qp.is_user ? "user" : "kernel",
+ qp->qplib_qp.id,
+ err_event->sq_cons_idx,
+ err_event->rq_cons_idx,
+ err_event->req_slow_path_state,
+ err_event->req_err_state_reason,
+ err_event->res_slow_path_state,
+ err_event->res_err_state_reason);
+
+ if (event.device && qp->ib_qp.event_handler)
+ qp->ib_qp.event_handler(&event, qp->ib_qp.qp_context);
+
+ return 0;
+}
+
+static int bnxt_re_handle_cq_async_error(void *event, struct bnxt_re_cq *cq)
+{
+ struct creq_cq_error_notification *cqerr;
+ bool send = false;
+
+ cqerr = event;
+ switch (cqerr->cq_err_reason) {
+ case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_INVALID_ERROR:
+ case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_OVERFLOW_ERROR:
+ case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_LOAD_ERROR:
+ case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_INVALID_ERROR:
+ case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_OVERFLOW_ERROR:
+ case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_LOAD_ERROR:
+ send = true;
+ default:
+ break;
+ }
+
+ if (send && cq->ibcq.event_handler) {
+ struct ib_event ibevent = {};
+
+ ibevent.event = IB_EVENT_CQ_ERR;
+ ibevent.element.cq = &cq->ibcq;
+ ibevent.device = &cq->rdev->ibdev;
+
+ dev_err(rdev_to_dev(cq->rdev),
+ "%s err reason %d\n", __func__, cqerr->cq_err_reason);
+ cq->ibcq.event_handler(&ibevent, cq->ibcq.cq_context);
+ }
+
+ cq->qplib_cq.is_cq_err_event = true;
+
+ return 0;
+}
+
+static int bnxt_re_handle_affi_async_event(struct creq_qp_event *affi_async,
+ void *obj)
+{
+ struct bnxt_qplib_qp *qplqp;
+ struct bnxt_qplib_cq *qplcq;
+ struct bnxt_re_qp *qp;
+ struct bnxt_re_cq *cq;
+ int rc = 0;
+ u8 event;
+
+ if (!obj)
+ return rc; /* QP was already dead, still return success */
+
+ event = affi_async->event;
+ switch (event) {
+ case CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION:
+ qplqp = obj;
+ qp = container_of(qplqp, struct bnxt_re_qp, qplib_qp);
+ rc = bnxt_re_handle_qp_async_event(affi_async, qp);
+ break;
+ case CREQ_QP_EVENT_EVENT_CQ_ERROR_NOTIFICATION:
+ qplcq = obj;
+ cq = container_of(qplcq, struct bnxt_re_cq, qplib_cq);
+ rc = bnxt_re_handle_cq_async_error(affi_async, cq);
+ break;
+ default:
+ rc = -EINVAL;
+ }
+
+ return rc;
+}
+
+static int bnxt_re_aeq_handler(struct bnxt_qplib_rcfw *rcfw,
+ void *aeqe, void *obj)
+{
+ struct creq_func_event *unaffi_async;
+ struct creq_qp_event *affi_async;
+ u8 type;
+ int rc;
+
+ type = ((struct creq_base *)aeqe)->type;
+ if (type == CREQ_BASE_TYPE_FUNC_EVENT) {
+ unaffi_async = aeqe;
+ rc = bnxt_re_handle_unaffi_async_event(unaffi_async);
+ } else {
+ affi_async = aeqe;
+ rc = bnxt_re_handle_affi_async_event(affi_async, obj);
+ }
+
+ return rc;
+}
+
+static int bnxt_re_srqn_handler(struct bnxt_qplib_nq *nq,
+ struct bnxt_qplib_srq *handle, u8 event)
+{
+ struct bnxt_re_srq *srq = to_bnxt_re(handle, struct bnxt_re_srq,
+ qplib_srq);
+ struct ib_event ib_event;
+
+ if (srq == NULL) {
+ pr_err("%s: SRQ is NULL, SRQN not handled",
+ ROCE_DRV_MODULE_NAME);
+ return -EINVAL;
+ }
+ ib_event.device = &srq->rdev->ibdev;
+ ib_event.element.srq = &srq->ibsrq;
+ if (event == NQ_SRQ_EVENT_EVENT_SRQ_THRESHOLD_EVENT)
+ ib_event.event = IB_EVENT_SRQ_LIMIT_REACHED;
+ else
+ ib_event.event = IB_EVENT_SRQ_ERR;
+
+ if (srq->ibsrq.event_handler) {
+ /* Lock event_handler? */
+ (*srq->ibsrq.event_handler)(&ib_event,
+ srq->ibsrq.srq_context);
+ }
+ return 0;
+}
+
+static int bnxt_re_cqn_handler(struct bnxt_qplib_nq *nq,
+ struct bnxt_qplib_cq *handle)
+{
+ struct bnxt_re_cq *cq = to_bnxt_re(handle, struct bnxt_re_cq,
+ qplib_cq);
+ u32 *cq_ptr;
+
+ if (cq == NULL) {
+ pr_err("%s: CQ is NULL, CQN not handled",
+ ROCE_DRV_MODULE_NAME);
+ return -EINVAL;
+ }
+ /* CQ already in destroy path. Do not handle any more events */
+ if (handle->destroyed || !atomic_read(&cq->ibcq.usecnt)) {
+ if (!handle->destroyed)
+ dev_dbg(NULL, "%s: CQ being destroyed, CQN not handled",
+ ROCE_DRV_MODULE_NAME);
+ return 0;
+ }
+
+ if (cq->ibcq.comp_handler) {
+ if (cq->uctx_cq_page) {
+ cq_ptr = (u32 *)cq->uctx_cq_page;
+ *cq_ptr = cq->qplib_cq.toggle;
+ }
+ /* Lock comp_handler? */
+ (*cq->ibcq.comp_handler)(&cq->ibcq, cq->ibcq.cq_context);
+ }
+
+ return 0;
+}
+
+struct bnxt_qplib_nq *bnxt_re_get_nq(struct bnxt_re_dev *rdev)
+{
+ int min, indx;
+
+ mutex_lock(&rdev->nqr.load_lock);
+ for (indx = 0, min = 0; indx < (rdev->nqr.num_msix - 1); indx++) {
+ if (rdev->nqr.nq[min].load > rdev->nqr.nq[indx].load)
+ min = indx;
+ }
+ rdev->nqr.nq[min].load++;
+ mutex_unlock(&rdev->nqr.load_lock);
+
+ return &rdev->nqr.nq[min];
+}
+
+void bnxt_re_put_nq(struct bnxt_re_dev *rdev, struct bnxt_qplib_nq *nq)
+{
+ mutex_lock(&rdev->nqr.load_lock);
+ nq->load--;
+ mutex_unlock(&rdev->nqr.load_lock);
+}
+
+static bool bnxt_re_check_min_attr(struct bnxt_re_dev *rdev)
+{
+ struct bnxt_qplib_dev_attr *attr;
+ bool rc = true;
+
+ attr = rdev->dev_attr;
+
+ if (!attr->max_cq || !attr->max_qp ||
+ !attr->max_sgid || !attr->max_mr) {
+ dev_err(rdev_to_dev(rdev),"Insufficient RoCE resources");
+ dev_dbg(rdev_to_dev(rdev),
+ "max_cq = %d, max_qp = %d, max_dpi = %d, max_sgid = %d, max_mr = %d",
+ attr->max_cq, attr->max_qp, attr->max_dpi,
+ attr->max_sgid, attr->max_mr);
+ rc = false;
+ }
+ return rc;
+}
+
+static void bnxt_re_dispatch_event(struct ib_device *ibdev, struct ib_qp *qp,
+ u8 port_num, enum ib_event_type event)
+{
+ struct ib_event ib_event;
+
+ ib_event.device = ibdev;
+ if (qp) {
+ ib_event.element.qp = qp;
+ ib_event.event = event;
+ if (qp->event_handler)
+ qp->event_handler(&ib_event, qp->qp_context);
+ } else {
+ ib_event.element.port_num = port_num;
+ ib_event.event = event;
+ ib_dispatch_event(&ib_event);
+ }
+
+ dev_dbg(rdev_to_dev(to_bnxt_re_dev(ibdev, ibdev)),
+ "ibdev %p Event 0x%x port_num 0x%x", ibdev, event, port_num);
+}
+
+static bool bnxt_re_is_qp1_or_shadow_qp(struct bnxt_re_dev *rdev,
+ struct bnxt_re_qp *qp)
+{
+ if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL)
+ return (qp->ib_qp.qp_type == IB_QPT_GSI) ||
+ (qp == rdev->gsi_ctx.gsi_sqp);
+ else
+ return (qp->ib_qp.qp_type == IB_QPT_GSI);
+}
+
+static void bnxt_re_stop_all_nonqp1_nonshadow_qps(struct bnxt_re_dev *rdev)
+{
+ struct bnxt_qplib_qp *qpl_qp;
+ bool dev_detached = false;
+ struct ib_qp_attr qp_attr;
+ int num_qps_stopped = 0;
+ int mask = IB_QP_STATE;
+ struct bnxt_re_qp *qp;
+ unsigned long flags;
+
+ if (!rdev)
+ return;
+
+restart:
+ if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags))
+ dev_detached = true;
+
+ qp_attr.qp_state = IB_QPS_ERR;
+ mutex_lock(&rdev->qp_lock);
+ list_for_each_entry(qp, &rdev->qp_list, list) {
+ qpl_qp = &qp->qplib_qp;
+ if (dev_detached || !bnxt_re_is_qp1_or_shadow_qp(rdev, qp)) {
+ if (qpl_qp->state !=
+ CMDQ_MODIFY_QP_NEW_STATE_RESET &&
+ qpl_qp->state !=
+ CMDQ_MODIFY_QP_NEW_STATE_ERR) {
+ if (dev_detached) {
+ /*
+ * Cant actually send the command down,
+ * marking the state for bookkeeping
+ */
+ qpl_qp->state =
+ CMDQ_MODIFY_QP_NEW_STATE_ERR;
+ qpl_qp->cur_qp_state = qpl_qp->state;
+ if (!qpl_qp->is_user) {
+ /* Add to flush list */
+ flags = bnxt_re_lock_cqs(qp);
+ bnxt_qplib_add_flush_qp(qpl_qp);
+ bnxt_re_unlock_cqs(qp, flags);
+ }
+ } else {
+ num_qps_stopped++;
+ bnxt_re_modify_qp(&qp->ib_qp,
+ &qp_attr, mask,
+ NULL);
+ }
+
+ bnxt_re_dispatch_event(&rdev->ibdev, &qp->ib_qp,
+ 1, IB_EVENT_QP_FATAL);
+ /*
+ * 1. Release qp_lock after a budget to unblock other verb
+ * requests (like qp_destroy) from stack.
+ * 2. Traverse through the qp_list freshly as addition / deletion
+ * might have happened since qp_lock is getting released here.
+ */
+ if (num_qps_stopped % BNXT_RE_STOP_QPS_BUDGET == 0) {
+ mutex_unlock(&rdev->qp_lock);
+ goto restart;
+ }
+ }
+ }
+ }
+
+ mutex_unlock(&rdev->qp_lock);
+}
+
+static int bnxt_re_update_gid(struct bnxt_re_dev *rdev)
+{
+ struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl;
+ struct bnxt_qplib_gid gid;
+ u16 gid_idx, index;
+ int rc = 0;
+
+ if (!test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags))
+ return 0;
+
+ if (sgid_tbl == NULL) {
+ dev_err(rdev_to_dev(rdev), "QPLIB: SGID table not allocated");
+ return -EINVAL;
+ }
+
+ for (index = 0; index < sgid_tbl->active; index++) {
+ gid_idx = sgid_tbl->hw_id[index];
+
+ if (!memcmp(&sgid_tbl->tbl[index], &bnxt_qplib_gid_zero,
+ sizeof(bnxt_qplib_gid_zero)))
+ continue;
+ /* Need to modify the VLAN enable setting of non VLAN GID only
+ * as setting is done for VLAN GID while adding GID
+ *
+ * If disable_prio_vlan_tx is enable, then we'll need to remove the
+ * vlan entry from the sgid_tbl.
+ */
+ if (sgid_tbl->vlan[index] == true)
+ continue;
+
+ memcpy(&gid, &sgid_tbl->tbl[index], sizeof(gid));
+
+ rc = bnxt_qplib_update_sgid(sgid_tbl, &gid, gid_idx,
+ rdev->dev_addr);
+ }
+
+ return rc;
+}
+
+static void bnxt_re_clear_cc(struct bnxt_re_dev *rdev)
+{
+ struct bnxt_qplib_cc_param *cc_param = &rdev->cc_param;
+
+ if (_is_chip_p7(rdev->chip_ctx)) {
+ cc_param->mask = CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP;
+ } else {
+ cc_param->mask = (CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_CC_MODE |
+ CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ENABLE_CC |
+ CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_ECN);
+
+ if (!is_qport_service_type_supported(rdev))
+ cc_param->mask |=
+ (CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_VLAN_PCP |
+ CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_TOS_DSCP |
+ CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP);
+ }
+
+ cc_param->cur_mask = cc_param->mask;
+
+ if (bnxt_qplib_modify_cc(&rdev->qplib_res, cc_param))
+ dev_err(rdev_to_dev(rdev), "Failed to modify cc\n");
+}
+
+static int bnxt_re_setup_cc(struct bnxt_re_dev *rdev)
+{
+ struct bnxt_qplib_cc_param *cc_param = &rdev->cc_param;
+ int rc;
+
+ if (_is_chip_p7(rdev->chip_ctx)) {
+ cc_param->enable = 0x0;
+ cc_param->mask = CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP;
+ } else {
+ cc_param->enable = 0x1;
+ cc_param->mask = (CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_CC_MODE |
+ CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ENABLE_CC |
+ CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_ECN);
+
+ if (!is_qport_service_type_supported(rdev))
+ cc_param->mask |=
+ (CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_VLAN_PCP |
+ CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_TOS_DSCP |
+ CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP);
+ }
+
+ cc_param->cur_mask = cc_param->mask;
+
+ rc = bnxt_qplib_modify_cc(&rdev->qplib_res, cc_param);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "Failed to modify cc\n");
+ return rc;
+ }
+ /* Reset the programming mask */
+ cc_param->mask = 0;
+ if (cc_param->qp1_tos_dscp != cc_param->tos_dscp) {
+ cc_param->qp1_tos_dscp = cc_param->tos_dscp;
+ rc = bnxt_re_update_qp1_tos_dscp(rdev);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "%s:Failed to modify QP1:%d",
+ __func__, rc);
+ goto clear;
+ }
+ }
+ return 0;
+
+clear:
+ bnxt_re_clear_cc(rdev);
+ return rc;
+}
+
+int bnxt_re_query_hwrm_dscp2pri(struct bnxt_re_dev *rdev,
+ struct bnxt_re_dscp2pri *d2p, u16 *count,
+ u16 target_id)
+{
+ struct bnxt_en_dev *en_dev = rdev->en_dev;
+ struct hwrm_queue_dscp2pri_qcfg_input req;
+ struct hwrm_queue_dscp2pri_qcfg_output resp;
+ struct bnxt_re_dscp2pri *dscp2pri;
+ struct bnxt_fw_msg fw_msg;
+ u16 in_count = *count;
+ dma_addr_t dma_handle;
+ int rc = 0, i;
+ u16 data_len;
+ u8 *kmem;
+
+ data_len = *count * sizeof(*dscp2pri);
+ memset(&fw_msg, 0, sizeof(fw_msg));
+ memset(&req, 0, sizeof(req));
+ bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
+ HWRM_QUEUE_DSCP2PRI_QCFG, -1, target_id);
+ req.port_id = (target_id == 0xFFFF) ? en_dev->pf_port_id : 1;
+
+ kmem = dma_zalloc_coherent(&en_dev->pdev->dev, data_len, &dma_handle,
+ GFP_KERNEL);
+ if (!kmem) {
+ dev_err(rdev_to_dev(rdev),
+ "dma_zalloc_coherent failure, length = %u\n",
+ (unsigned)data_len);
+ return -ENOMEM;
+ }
+ req.dest_data_addr = cpu_to_le64(dma_handle);
+ req.dest_data_buffer_size = cpu_to_le16(data_len);
+ bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
+ sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
+ rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
+ if (rc)
+ goto out;
+
+ /* Upload the DSCP-MASK-PRI tuple(s) */
+ dscp2pri = (struct bnxt_re_dscp2pri *)kmem;
+ for (i = 0; i < le16_to_cpu(resp.entry_cnt) && i < in_count; i++) {
+ d2p[i].dscp = dscp2pri->dscp;
+ d2p[i].mask = dscp2pri->mask;
+ d2p[i].pri = dscp2pri->pri;
+ dscp2pri++;
+ }
+ *count = le16_to_cpu(resp.entry_cnt);
+out:
+ dma_free_coherent(&en_dev->pdev->dev, data_len, kmem, dma_handle);
+ return rc;
+}
+
+int bnxt_re_prio_vlan_tx_update(struct bnxt_re_dev *rdev)
+{
+ /* Remove the VLAN from the GID entry */
+ if (rdev->cc_param.disable_prio_vlan_tx)
+ rdev->qplib_res.prio = false;
+ else
+ rdev->qplib_res.prio = true;
+
+ return bnxt_re_update_gid(rdev);
+}
+
+int bnxt_re_set_hwrm_dscp2pri(struct bnxt_re_dev *rdev,
+ struct bnxt_re_dscp2pri *d2p, u16 count,
+ u16 target_id)
+{
+ struct bnxt_en_dev *en_dev = rdev->en_dev;
+ struct hwrm_queue_dscp2pri_cfg_input req;
+ struct hwrm_queue_dscp2pri_cfg_output resp;
+ struct bnxt_fw_msg fw_msg;
+ struct bnxt_re_dscp2pri *dscp2pri;
+ int i, rc, data_len = 3 * 256;
+ dma_addr_t dma_handle;
+ u8 *kmem;
+
+ memset(&req, 0, sizeof(req));
+ memset(&fw_msg, 0, sizeof(fw_msg));
+ bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
+ HWRM_QUEUE_DSCP2PRI_CFG, -1, target_id);
+ req.port_id = (target_id == 0xFFFF) ? en_dev->pf_port_id : 1;
+
+ kmem = dma_alloc_coherent(&en_dev->pdev->dev, data_len, &dma_handle,
+ GFP_KERNEL);
+ if (!kmem) {
+ dev_err(rdev_to_dev(rdev),
+ "dma_alloc_coherent failure, length = %u\n",
+ (unsigned)data_len);
+ return -ENOMEM;
+ }
+ req.src_data_addr = cpu_to_le64(dma_handle);
+
+ /* Download the DSCP-MASK-PRI tuple(s) */
+ dscp2pri = (struct bnxt_re_dscp2pri *)kmem;
+ for (i = 0; i < count; i++) {
+ dscp2pri->dscp = d2p[i].dscp;
+ dscp2pri->mask = d2p[i].mask;
+ dscp2pri->pri = d2p[i].pri;
+ dscp2pri++;
+ }
+
+ req.entry_cnt = cpu_to_le16(count);
+ bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
+ sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
+ rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
+ dma_free_coherent(&en_dev->pdev->dev, data_len, kmem, dma_handle);
+ return rc;
+}
+
+int bnxt_re_query_hwrm_qportcfg(struct bnxt_re_dev *rdev,
+ struct bnxt_re_tc_rec *tc_rec, u16 tid)
+{
+ u8 max_tc, tc, *qptr, *type_ptr0, *type_ptr1;
+ struct hwrm_queue_qportcfg_output resp = {0};
+ struct hwrm_queue_qportcfg_input req = {0};
+ struct bnxt_en_dev *en_dev = rdev->en_dev;
+ struct bnxt_fw_msg fw_msg;
+ bool def_init = false;
+ u8 *tmp_type;
+ u8 cos_id;
+ int rc;
+
+ memset(&fw_msg, 0, sizeof(fw_msg));
+ bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_QUEUE_QPORTCFG,
+ -1, tid);
+ req.port_id = (tid == 0xFFFF) ? en_dev->pf_port_id : 1;
+ if (BNXT_EN_ASYM_Q(en_dev))
+ req.flags = htole32(HWRM_QUEUE_QPORTCFG_INPUT_FLAGS_PATH_RX);
+
+ bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
+ sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
+ rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
+ if (rc)
+ return rc;
+
+ if (!resp.max_configurable_queues)
+ return -EINVAL;
+
+ max_tc = resp.max_configurable_queues;
+ tc_rec->max_tc = max_tc;
+
+ if (resp.queue_cfg_info & HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_CFG_INFO_USE_PROFILE_TYPE)
+ tc_rec->serv_type_enabled = true;
+
+ qptr = &resp.queue_id0;
+ type_ptr0 = &resp.queue_id0_service_profile_type;
+ type_ptr1 = &resp.queue_id1_service_profile_type;
+ for (tc = 0; tc < max_tc; tc++) {
+ tmp_type = tc ? type_ptr1 + (tc - 1) : type_ptr0;
+
+ cos_id = *qptr++;
+ /* RoCE CoS queue is the first cos queue.
+ * For MP12 and MP17 order is 405 and 141015.
+ */
+ if (is_bnxt_roce_queue(rdev, *qptr, *tmp_type)) {
+ tc_rec->cos_id_roce = cos_id;
+ tc_rec->tc_roce = tc;
+ } else if (is_bnxt_cnp_queue(rdev, *qptr, *tmp_type)) {
+ tc_rec->cos_id_cnp = cos_id;
+ tc_rec->tc_cnp = tc;
+ } else if (!def_init) {
+ def_init = true;
+ tc_rec->tc_def = tc;
+ tc_rec->cos_id_def = cos_id;
+ }
+ qptr++;
+ }
+
+ return rc;
+}
+
+int bnxt_re_hwrm_cos2bw_qcfg(struct bnxt_re_dev *rdev, u16 target_id,
+ struct bnxt_re_cos2bw_cfg *cfg)
+{
+ struct bnxt_en_dev *en_dev = rdev->en_dev;
+ struct hwrm_queue_cos2bw_qcfg_output resp;
+ struct hwrm_queue_cos2bw_qcfg_input req = {0};
+ struct bnxt_fw_msg fw_msg;
+ int rc, indx;
+ void *data;
+
+ memset(&fw_msg, 0, sizeof(fw_msg));
+ bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
+ HWRM_QUEUE_COS2BW_QCFG, -1, target_id);
+ req.port_id = (target_id == 0xFFFF) ? en_dev->pf_port_id : 1;
+
+ bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
+ sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
+ rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
+ if (rc)
+ return rc;
+ data = &resp.queue_id0 + offsetof(struct bnxt_re_cos2bw_cfg,
+ queue_id);
+ for (indx = 0; indx < 8; indx++, data += (sizeof(cfg->cfg))) {
+ memcpy(&cfg->cfg, data, sizeof(cfg->cfg));
+ if (indx == 0)
+ cfg->queue_id = resp.queue_id0;
+ cfg++;
+ }
+
+ return rc;
+}
+
+int bnxt_re_hwrm_cos2bw_cfg(struct bnxt_re_dev *rdev, u16 target_id,
+ struct bnxt_re_cos2bw_cfg *cfg)
+{
+ struct bnxt_en_dev *en_dev = rdev->en_dev;
+ struct hwrm_queue_cos2bw_cfg_input req = {0};
+ struct hwrm_queue_cos2bw_cfg_output resp = {0};
+ struct bnxt_fw_msg fw_msg;
+ void *data;
+ int indx;
+ int rc;
+
+ memset(&fw_msg, 0, sizeof(fw_msg));
+ bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
+ HWRM_QUEUE_COS2BW_CFG, -1, target_id);
+ req.port_id = (target_id == 0xFFFF) ? en_dev->pf_port_id : 1;
+
+ /* Chimp wants enable bit to retain previous
+ * config done by L2 driver
+ */
+ for (indx = 0; indx < 8; indx++) {
+ if (cfg[indx].queue_id < 40) {
+ req.enables |= cpu_to_le32(
+ HWRM_QUEUE_COS2BW_CFG_INPUT_ENABLES_COS_QUEUE_ID0_VALID <<
+ indx);
+ }
+
+ data = (char *)&req.unused_0 + indx * (sizeof(*cfg) - 4);
+ memcpy(data, &cfg[indx].queue_id, sizeof(*cfg) - 4);
+ if (indx == 0) {
+ req.queue_id0 = cfg[0].queue_id;
+ req.unused_0 = 0;
+ }
+ }
+
+ memset(&resp, 0, sizeof(resp));
+ bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
+ sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
+ rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
+ return rc;
+}
+
+int bnxt_re_host_pf_id_query(struct bnxt_re_dev *rdev,
+ struct bnxt_qplib_query_fn_info *fn_info,
+ u32 *pf_mask, u32 *first_pf)
+{
+ struct hwrm_func_host_pf_ids_query_output resp = {0};
+ struct hwrm_func_host_pf_ids_query_input req;
+ struct bnxt_en_dev *en_dev = rdev->en_dev;
+ struct bnxt_fw_msg fw_msg;
+ int rc;
+
+ memset(&fw_msg, 0, sizeof(fw_msg));
+ memset(&req, 0, sizeof(req));
+ bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
+ HWRM_FUNC_HOST_PF_IDS_QUERY, -1, -1);
+ /* To query the info from the host EPs */
+ switch (fn_info->host) {
+ case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_SOC:
+ case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_0:
+ case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_1:
+ case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_2:
+ case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_3:
+ req.host = fn_info->host;
+ break;
+ default:
+ req.host = HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_0;
+ break;
+ }
+
+ req.filter = fn_info->filter;
+ if (req.filter > HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_FILTER_ROCE)
+ req.filter = HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_FILTER_ALL;
+
+ bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
+ sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
+ rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
+
+
+ *first_pf = le16_to_cpu(resp.first_pf_id);
+ *pf_mask = le16_to_cpu(resp.pf_ordinal_mask);
+
+ return rc;
+}
+
+static void bnxt_re_put_stats_ctx(struct bnxt_re_dev *rdev)
+{
+ struct bnxt_qplib_ctx *hctx;
+ struct bnxt_qplib_res *res;
+ u16 tid = 0xffff;
+
+ res = &rdev->qplib_res;
+ hctx = res->hctx;
+
+ if (test_and_clear_bit(BNXT_RE_FLAG_STATS_CTX_ALLOC, &rdev->flags)) {
+ bnxt_re_net_stats_ctx_free(rdev, hctx->stats.fw_id, tid);
+ bnxt_qplib_free_stat_mem(res, &hctx->stats);
+ }
+}
+
+static void bnxt_re_put_stats2_ctx(struct bnxt_re_dev *rdev)
+{
+ test_and_clear_bit(BNXT_RE_FLAG_STATS_CTX2_ALLOC, &rdev->flags);
+}
+
+static int bnxt_re_get_stats_ctx(struct bnxt_re_dev *rdev)
+{
+ struct bnxt_qplib_ctx *hctx;
+ struct bnxt_qplib_res *res;
+ u16 tid = 0xffff;
+ int rc;
+
+ res = &rdev->qplib_res;
+ hctx = res->hctx;
+
+ rc = bnxt_qplib_alloc_stat_mem(res->pdev, rdev->chip_ctx, &hctx->stats);
+ if (rc)
+ return -ENOMEM;
+ rc = bnxt_re_net_stats_ctx_alloc(rdev, tid);
+ if (rc)
+ goto free_stat_mem;
+ set_bit(BNXT_RE_FLAG_STATS_CTX_ALLOC, &rdev->flags);
+
+ return 0;
+
+free_stat_mem:
+ bnxt_qplib_free_stat_mem(res, &hctx->stats);
+
+ return rc;
+}
+
+static int bnxt_re_update_dev_attr(struct bnxt_re_dev *rdev)
+{
+ int rc;
+
+ rc = bnxt_qplib_get_dev_attr(&rdev->rcfw);
+ if (rc)
+ return rc;
+ if (!bnxt_re_check_min_attr(rdev))
+ return -EINVAL;
+ return 0;
+}
+
+static void bnxt_re_free_tbls(struct bnxt_re_dev *rdev)
+{
+ bnxt_qplib_clear_tbls(&rdev->qplib_res);
+ bnxt_qplib_free_tbls(&rdev->qplib_res);
+}
+
+static int bnxt_re_alloc_init_tbls(struct bnxt_re_dev *rdev)
+{
+ struct bnxt_qplib_chip_ctx *chip_ctx = rdev->chip_ctx;
+ u8 pppp_factor = 0;
+ int rc;
+
+ /*
+ * TODO: Need a better mechanism for spreading of the
+ * 512 extended PPP pages. For now, spreading it
+ * based on port_count
+ */
+ if (_is_chip_p7(chip_ctx) && chip_ctx->modes.db_push)
+ pppp_factor = rdev->en_dev->port_count;
+ rc = bnxt_qplib_alloc_tbls(&rdev->qplib_res, pppp_factor);
+ if (rc)
+ return rc;
+ bnxt_qplib_init_tbls(&rdev->qplib_res);
+ set_bit(BNXT_RE_FLAG_TBLS_ALLOCINIT, &rdev->flags);
+
+ return 0;
+}
+
+static void bnxt_re_clean_nqs(struct bnxt_re_dev *rdev)
+{
+ struct bnxt_qplib_nq *nq;
+ int i;
+
+ if (!rdev->nqr.max_init)
+ return;
+
+ for (i = (rdev->nqr.max_init - 1) ; i >= 0; i--) {
+ nq = &rdev->nqr.nq[i];
+ bnxt_qplib_disable_nq(nq);
+ bnxt_re_net_ring_free(rdev, nq->ring_id);
+ bnxt_qplib_free_nq_mem(nq);
+ }
+ rdev->nqr.max_init = 0;
+}
+
+static int bnxt_re_setup_nqs(struct bnxt_re_dev *rdev)
+{
+ struct bnxt_re_ring_attr rattr = {};
+ struct bnxt_qplib_nq *nq;
+ int rc, i;
+ int depth;
+ u32 offt;
+ u16 vec;
+
+ mutex_init(&rdev->nqr.load_lock);
+ /*
+ * TODO: Optimize the depth based on the
+ * number of NQs.
+ */
+ depth = BNXT_QPLIB_NQE_MAX_CNT;
+ for (i = 0; i < rdev->nqr.num_msix - 1; i++) {
+ nq = &rdev->nqr.nq[i];
+ vec = rdev->nqr.msix_entries[i + 1].vector;
+ offt = rdev->nqr.msix_entries[i + 1].db_offset;
+ nq->hwq.max_elements = depth;
+ rc = bnxt_qplib_alloc_nq_mem(&rdev->qplib_res, nq);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev),
+ "Failed to get mem for NQ %d, rc = 0x%x",
+ i, rc);
+ goto fail_mem;
+ }
+
+ rattr.dma_arr = nq->hwq.pbl[PBL_LVL_0].pg_map_arr;
+ rattr.pages = nq->hwq.pbl[rdev->nqr.nq[i].hwq.level].pg_count;
+ rattr.type = bnxt_re_get_rtype(rdev);
+ rattr.mode = HWRM_RING_ALLOC_INPUT_INT_MODE_MSIX;
+ rattr.depth = nq->hwq.max_elements - 1;
+ rattr.lrid = rdev->nqr.msix_entries[i + 1].ring_idx;
+
+ /* Set DBR pacing capability on the first NQ ring only */
+ if (!i && bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx))
+ rattr.flags = HWRM_RING_ALLOC_INPUT_FLAGS_NQ_DBR_PACING;
+ else
+ rattr.flags = 0;
+
+ rc = bnxt_re_net_ring_alloc(rdev, &rattr, &nq->ring_id);
+ if (rc) {
+ nq->ring_id = 0xffff; /* Invalid ring-id */
+ dev_err(rdev_to_dev(rdev),
+ "Failed to get fw id for NQ %d, rc = 0x%x",
+ i, rc);
+ goto fail_ring;
+ }
+
+ rc = bnxt_qplib_enable_nq(nq, i, vec, offt,
+ &bnxt_re_cqn_handler,
+ &bnxt_re_srqn_handler);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev),
+ "Failed to enable NQ %d, rc = 0x%x", i, rc);
+ goto fail_en;
+ }
+ }
+
+ rdev->nqr.max_init = i;
+ return 0;
+fail_en:
+ /* *nq was i'th nq */
+ bnxt_re_net_ring_free(rdev, nq->ring_id);
+fail_ring:
+ bnxt_qplib_free_nq_mem(nq);
+fail_mem:
+ rdev->nqr.max_init = i;
+ return rc;
+}
+
+static void bnxt_re_sysfs_destroy_file(struct bnxt_re_dev *rdev)
+{
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(bnxt_re_attributes); i++)
+ device_remove_file(&rdev->ibdev.dev, bnxt_re_attributes[i]);
+}
+
+static int bnxt_re_sysfs_create_file(struct bnxt_re_dev *rdev)
+{
+ int i, j, rc = 0;
+
+ for (i = 0; i < ARRAY_SIZE(bnxt_re_attributes); i++) {
+ rc = device_create_file(&rdev->ibdev.dev,
+ bnxt_re_attributes[i]);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev),
+ "Failed to create IB sysfs with rc = 0x%x", rc);
+ /* Must clean up all created device files */
+ for (j = 0; j < i; j++)
+ device_remove_file(&rdev->ibdev.dev,
+ bnxt_re_attributes[j]);
+ clear_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags);
+ ib_unregister_device(&rdev->ibdev);
+ return 1;
+ }
+ }
+ return 0;
+}
+
+/* worker thread for polling periodic events. Now used for QoS programming*/
+static void bnxt_re_worker(struct work_struct *work)
+{
+ struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev,
+ worker.work);
+ int rc;
+
+ /* QoS is in 30s cadence for PFs*/
+ if (!rdev->is_virtfn && !rdev->worker_30s--)
+ rdev->worker_30s = 30;
+ /* Use trylock for bnxt_re_dev_lock as this can be
+ * held for long time by debugfs show path while issuing
+ * HWRMS. If the debugfs name update is not done in this
+ * iteration, the driver will check for the same in the
+ * next schedule of the worker i.e after 1 sec.
+ */
+ if (mutex_trylock(&bnxt_re_dev_lock))
+ mutex_unlock(&bnxt_re_dev_lock);
+
+ if (!rdev->stats.stats_query_sec)
+ goto resched;
+
+ if (test_bit(BNXT_RE_FLAG_ISSUE_CFA_FLOW_STATS, &rdev->flags) &&
+ (rdev->is_virtfn ||
+ !_is_ext_stats_supported(rdev->dev_attr->dev_cap_flags))) {
+ if (!(rdev->stats.stats_query_counter++ %
+ rdev->stats.stats_query_sec)) {
+ rc = bnxt_re_get_qos_stats(rdev);
+ if (rc && rc != -ENOMEM)
+ clear_bit(BNXT_RE_FLAG_ISSUE_CFA_FLOW_STATS,
+ &rdev->flags);
+ }
+ }
+
+resched:
+ schedule_delayed_work(&rdev->worker, msecs_to_jiffies(1000));
+}
+
+static int bnxt_re_alloc_dbr_sw_stats_mem(struct bnxt_re_dev *rdev)
+{
+ if (!(rdev->dbr_drop_recov || rdev->dbr_pacing))
+ return 0;
+
+ rdev->dbr_sw_stats = kzalloc(sizeof(*rdev->dbr_sw_stats), GFP_KERNEL);
+ if (!rdev->dbr_sw_stats)
+ return -ENOMEM;
+
+ return 0;
+}
+
+static void bnxt_re_free_dbr_sw_stats_mem(struct bnxt_re_dev *rdev)
+{
+ kfree(rdev->dbr_sw_stats);
+ rdev->dbr_sw_stats = NULL;
+}
+
+static int bnxt_re_initialize_dbr_drop_recov(struct bnxt_re_dev *rdev)
+{
+ rdev->dbr_drop_recov_wq =
+ create_singlethread_workqueue("bnxt_re_dbr_drop_recov");
+ if (!rdev->dbr_drop_recov_wq) {
+ dev_err(rdev_to_dev(rdev), "DBR Drop Revov wq alloc failed!");
+ return -EINVAL;
+ }
+ rdev->dbr_drop_recov = true;
+
+ /* Enable configfs setting dbr_drop_recov by default*/
+ rdev->user_dbr_drop_recov = true;
+
+ rdev->user_dbr_drop_recov_timeout = BNXT_RE_DBR_RECOV_USERLAND_TIMEOUT;
+ return 0;
+}
+
+static void bnxt_re_deinitialize_dbr_drop_recov(struct bnxt_re_dev *rdev)
+{
+ if (rdev->dbr_drop_recov_wq) {
+ flush_workqueue(rdev->dbr_drop_recov_wq);
+ destroy_workqueue(rdev->dbr_drop_recov_wq);
+ rdev->dbr_drop_recov_wq = NULL;
+ }
+ rdev->dbr_drop_recov = false;
+}
+
+static int bnxt_re_initialize_dbr_pacing(struct bnxt_re_dev *rdev)
+{
+ int rc;
+
+ /* Allocate a page for app use */
+ rdev->dbr_page = (void *)__get_free_page(GFP_KERNEL);
+ if (!rdev->dbr_page) {
+ dev_err(rdev_to_dev(rdev), "DBR page allocation failed!");
+ return -ENOMEM;
+ }
+ memset((u8 *)rdev->dbr_page, 0, PAGE_SIZE);
+ rdev->qplib_res.pacing_data = (struct bnxt_qplib_db_pacing_data *)rdev->dbr_page;
+ rc = bnxt_re_hwrm_dbr_pacing_qcfg(rdev);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev),
+ "Failed to query dbr pacing config %d\n", rc);
+ goto fail;
+ }
+ /* Create a work queue for scheduling dbq event */
+ rdev->dbq_wq = create_singlethread_workqueue("bnxt_re_dbq");
+ if (!rdev->dbq_wq) {
+ dev_err(rdev_to_dev(rdev), "DBQ wq alloc failed!");
+ rc = -ENOMEM;
+ goto fail;
+ }
+ /* MAP grc window 2 for reading db fifo depth */
+ writel_fbsd(rdev->en_dev->softc, BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4, 0,
+ rdev->chip_ctx->dbr_stat_db_fifo & BNXT_GRC_BASE_MASK);
+ rdev->dbr_db_fifo_reg_off =
+ (rdev->chip_ctx->dbr_stat_db_fifo & BNXT_GRC_OFFSET_MASK) +
+ 0x2000;
+ rdev->qplib_res.pacing_data->grc_reg_offset = rdev->dbr_db_fifo_reg_off;
+
+ rdev->dbr_bar_addr =
+ pci_resource_start(rdev->qplib_res.pdev, 0) +
+ rdev->dbr_db_fifo_reg_off;
+
+ /* Percentage of DB FIFO */
+ rdev->dbq_watermark = BNXT_RE_PACING_DBQ_THRESHOLD;
+ rdev->pacing_en_int_th = BNXT_RE_PACING_EN_INT_THRESHOLD;
+ rdev->pacing_algo_th = BNXT_RE_PACING_ALGO_THRESHOLD;
+ rdev->dbq_pacing_time = BNXT_RE_DBR_INT_TIME;
+ rdev->dbr_def_do_pacing = BNXT_RE_DBR_DO_PACING_NO_CONGESTION;
+ rdev->do_pacing_save = rdev->dbr_def_do_pacing;
+ bnxt_re_set_default_pacing_data(rdev);
+ dev_dbg(rdev_to_dev(rdev), "Initialized db pacing\n");
+
+ return 0;
+fail:
+ free_page((u64)rdev->dbr_page);
+ rdev->dbr_page = NULL;
+ return rc;
+}
+
+static void bnxt_re_deinitialize_dbr_pacing(struct bnxt_re_dev *rdev)
+{
+ if (rdev->dbq_wq)
+ flush_workqueue(rdev->dbq_wq);
+
+ cancel_work_sync(&rdev->dbq_fifo_check_work);
+ cancel_delayed_work_sync(&rdev->dbq_pacing_work);
+
+ if (rdev->dbq_wq) {
+ destroy_workqueue(rdev->dbq_wq);
+ rdev->dbq_wq = NULL;
+ }
+
+ if (rdev->dbr_page)
+ free_page((u64)rdev->dbr_page);
+ rdev->dbr_page = NULL;
+ rdev->dbr_pacing = false;
+}
+
+/* enable_dbr_pacing needs to be done only for older FWs
+ * where host selects primary function. ie. pacing_ext
+ * flags is not set.
+ */
+int bnxt_re_enable_dbr_pacing(struct bnxt_re_dev *rdev)
+{
+ struct bnxt_qplib_nq *nq;
+
+ nq = &rdev->nqr.nq[0];
+ rdev->dbq_nq_id = nq->ring_id;
+
+ if (!bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx) &&
+ bnxt_qplib_dbr_pacing_is_primary_pf(rdev->chip_ctx)) {
+ if (bnxt_re_hwrm_dbr_pacing_cfg(rdev, true)) {
+ dev_err(rdev_to_dev(rdev),
+ "Failed to set dbr pacing config\n");
+ return -EIO;
+ }
+ /* MAP grc window 8 for ARMing the NQ DBQ */
+ writel_fbsd(rdev->en_dev->softc, BNXT_GRCPF_REG_WINDOW_BASE_OUT + 28 , 0,
+ rdev->chip_ctx->dbr_aeq_arm_reg & BNXT_GRC_BASE_MASK);
+ rdev->dbr_aeq_arm_reg_off =
+ (rdev->chip_ctx->dbr_aeq_arm_reg &
+ BNXT_GRC_OFFSET_MASK) + 0x8000;
+ writel_fbsd(rdev->en_dev->softc, rdev->dbr_aeq_arm_reg_off , 0, 1);
+ }
+
+ return 0;
+}
+
+/* disable_dbr_pacing needs to be done only for older FWs
+ * where host selects primary function. ie. pacing_ext
+ * flags is not set.
+ */
+
+int bnxt_re_disable_dbr_pacing(struct bnxt_re_dev *rdev)
+{
+ int rc = 0;
+
+ if (!bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx) &&
+ bnxt_qplib_dbr_pacing_is_primary_pf(rdev->chip_ctx))
+ rc = bnxt_re_hwrm_dbr_pacing_cfg(rdev, false);
+
+ return rc;
+}
+
+static void bnxt_re_ib_uninit(struct bnxt_re_dev *rdev)
+{
+ if (test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) {
+ bnxt_re_sysfs_destroy_file(rdev);
+ /* Cleanup ib dev */
+ ib_unregister_device(&rdev->ibdev);
+ clear_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags);
+ return;
+ }
+}
+
+static void bnxt_re_dev_uninit(struct bnxt_re_dev *rdev, u8 op_type)
+{
+ struct bnxt_qplib_dpi *kdpi;
+ int rc, wait_count = BNXT_RE_RES_FREE_WAIT_COUNT;
+
+ bnxt_re_net_unregister_async_event(rdev);
+
+ bnxt_re_put_stats2_ctx(rdev);
+ if (test_and_clear_bit(BNXT_RE_FLAG_DEV_LIST_INITIALIZED,
+ &rdev->flags)) {
+ /* did the caller hold the lock? */
+ mutex_lock(&bnxt_re_dev_lock);
+ list_del_rcu(&rdev->list);
+ mutex_unlock(&bnxt_re_dev_lock);
+ }
+
+ bnxt_re_uninit_resolve_wq(rdev);
+ bnxt_re_uninit_dcb_wq(rdev);
+ bnxt_re_uninit_aer_wq(rdev);
+
+ bnxt_re_deinitialize_dbr_drop_recov(rdev);
+
+ if (bnxt_qplib_dbr_pacing_en(rdev->chip_ctx))
+ (void)bnxt_re_disable_dbr_pacing(rdev);
+
+ if (test_and_clear_bit(BNXT_RE_FLAG_WORKER_REG, &rdev->flags)) {
+ cancel_delayed_work_sync(&rdev->worker);
+ }
+
+ /* Wait for ULPs to release references */
+ while (atomic_read(&rdev->stats.rsors.cq_count) && --wait_count)
+ usleep_range(500, 1000);
+ if (!wait_count)
+ dev_err(rdev_to_dev(rdev),
+ "CQ resources not freed by stack, count = 0x%x",
+ atomic_read(&rdev->stats.rsors.cq_count));
+
+ kdpi = &rdev->dpi_privileged;
+ if (kdpi->umdbr) { /* kernel DPI was allocated with success */
+ (void)bnxt_qplib_dealloc_dpi(&rdev->qplib_res, kdpi);
+ /*
+ * Driver just need to know no command had failed
+ * during driver load sequence and below command is
+ * required indeed. Piggybacking dpi allocation status.
+ */
+ }
+
+ /* Protect the device uninitialization and start_irq/stop_irq L2
+ * callbacks with rtnl lock to avoid race condition between these calls
+ */
+ rtnl_lock();
+ if (test_and_clear_bit(BNXT_RE_FLAG_SETUP_NQ, &rdev->flags))
+ bnxt_re_clean_nqs(rdev);
+ rtnl_unlock();
+
+ if (test_and_clear_bit(BNXT_RE_FLAG_TBLS_ALLOCINIT, &rdev->flags))
+ bnxt_re_free_tbls(rdev);
+ if (test_and_clear_bit(BNXT_RE_FLAG_RCFW_CHANNEL_INIT, &rdev->flags)) {
+ rc = bnxt_qplib_deinit_rcfw(&rdev->rcfw);
+ if (rc)
+ dev_warn(rdev_to_dev(rdev),
+ "Failed to deinitialize fw, rc = 0x%x", rc);
+ }
+
+ bnxt_re_put_stats_ctx(rdev);
+
+ if (test_and_clear_bit(BNXT_RE_FLAG_ALLOC_CTX, &rdev->flags))
+ bnxt_qplib_free_hwctx(&rdev->qplib_res);
+
+ rtnl_lock();
+ if (test_and_clear_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags))
+ bnxt_qplib_disable_rcfw_channel(&rdev->rcfw);
+
+ if (rdev->dbr_pacing)
+ bnxt_re_deinitialize_dbr_pacing(rdev);
+
+ bnxt_re_free_dbr_sw_stats_mem(rdev);
+
+ if (test_and_clear_bit(BNXT_RE_FLAG_NET_RING_ALLOC, &rdev->flags))
+ bnxt_re_net_ring_free(rdev, rdev->rcfw.creq.ring_id);
+
+ if (test_and_clear_bit(BNXT_RE_FLAG_ALLOC_RCFW, &rdev->flags))
+ bnxt_qplib_free_rcfw_channel(&rdev->qplib_res);
+
+ if (test_and_clear_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags))
+ bnxt_re_free_msix(rdev);
+ rtnl_unlock();
+
+ bnxt_re_destroy_chip_ctx(rdev);
+
+ if (op_type != BNXT_RE_PRE_RECOVERY_REMOVE) {
+ if (test_and_clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED,
+ &rdev->flags))
+ bnxt_re_unregister_netdev(rdev);
+ }
+}
+
+static int bnxt_re_dev_init(struct bnxt_re_dev *rdev, u8 op_type, u8 wqe_mode)
+{
+ struct bnxt_re_ring_attr rattr = {};
+ struct bnxt_qplib_creq_ctx *creq;
+ int vec, offset;
+ int rc = 0;
+
+ if (op_type != BNXT_RE_POST_RECOVERY_INIT) {
+ /* Registered a new RoCE device instance to netdev */
+ rc = bnxt_re_register_netdev(rdev);
+ if (rc)
+ return -EINVAL;
+ }
+ set_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags);
+
+ rc = bnxt_re_setup_chip_ctx(rdev, wqe_mode);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "Failed to get chip context rc 0x%x", rc);
+ bnxt_re_unregister_netdev(rdev);
+ clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags);
+ rc = -EINVAL;
+ return rc;
+ }
+
+ /* Protect the device initialization and start_irq/stop_irq L2 callbacks
+ * with rtnl lock to avoid race condition between these calls
+ */
+ rtnl_lock();
+ rc = bnxt_re_request_msix(rdev);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev),
+ "Requesting MSI-X vectors failed with rc = 0x%x", rc);
+ rc = -EINVAL;
+ goto release_rtnl;
+ }
+ set_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags);
+
+ /* Establish RCFW Communication Channel to initialize the context
+ memory for the function and all child VFs */
+ rc = bnxt_qplib_alloc_rcfw_channel(&rdev->qplib_res);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev),
+ "Failed to alloc mem for rcfw, rc = %#x\n", rc);
+ goto release_rtnl;
+ }
+ set_bit(BNXT_RE_FLAG_ALLOC_RCFW, &rdev->flags);
+
+ creq = &rdev->rcfw.creq;
+ rattr.dma_arr = creq->hwq.pbl[PBL_LVL_0].pg_map_arr;
+ rattr.pages = creq->hwq.pbl[creq->hwq.level].pg_count;
+ rattr.type = bnxt_re_get_rtype(rdev);
+ rattr.mode = HWRM_RING_ALLOC_INPUT_INT_MODE_MSIX;
+ rattr.depth = BNXT_QPLIB_CREQE_MAX_CNT - 1;
+ rattr.lrid = rdev->nqr.msix_entries[BNXT_RE_AEQ_IDX].ring_idx;
+ rc = bnxt_re_net_ring_alloc(rdev, &rattr, &creq->ring_id);
+ if (rc) {
+ creq->ring_id = 0xffff;
+ dev_err(rdev_to_dev(rdev),
+ "Failed to allocate CREQ fw id with rc = 0x%x", rc);
+ goto release_rtnl;
+ }
+
+ if (!rdev->chip_ctx)
+ goto release_rtnl;
+ /* Program the NQ ID for DBQ notification */
+ if (rdev->chip_ctx->modes.dbr_pacing_v0 ||
+ bnxt_qplib_dbr_pacing_en(rdev->chip_ctx) ||
+ bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) {
+ rc = bnxt_re_initialize_dbr_pacing(rdev);
+ if (!rc)
+ rdev->dbr_pacing = true;
+ else
+ rdev->dbr_pacing = false;
+ dev_dbg(rdev_to_dev(rdev), "%s: initialize db pacing ret %d\n",
+ __func__, rc);
+ }
+
+ vec = rdev->nqr.msix_entries[BNXT_RE_AEQ_IDX].vector;
+ offset = rdev->nqr.msix_entries[BNXT_RE_AEQ_IDX].db_offset;
+ rc = bnxt_qplib_enable_rcfw_channel(&rdev->rcfw, vec, offset,
+ &bnxt_re_aeq_handler);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev),
+ "Failed to enable RCFW channel with rc = 0x%x", rc);
+ goto release_rtnl;
+ }
+ set_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags);
+
+ rc = bnxt_re_update_dev_attr(rdev);
+ if (rc)
+ goto release_rtnl;
+ bnxt_re_set_resource_limits(rdev);
+ if (!rdev->is_virtfn && !_is_chip_gen_p5_p7(rdev->chip_ctx)) {
+ rc = bnxt_qplib_alloc_hwctx(&rdev->qplib_res);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev),
+ "Failed to alloc hw contexts, rc = 0x%x", rc);
+ goto release_rtnl;
+ }
+ set_bit(BNXT_RE_FLAG_ALLOC_CTX, &rdev->flags);
+ }
+
+ rc = bnxt_re_get_stats_ctx(rdev);
+ if (rc)
+ goto release_rtnl;
+
+ rc = bnxt_qplib_init_rcfw(&rdev->rcfw, rdev->is_virtfn);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev),
+ "Failed to initialize fw with rc = 0x%x", rc);
+ goto release_rtnl;
+ }
+ set_bit(BNXT_RE_FLAG_RCFW_CHANNEL_INIT, &rdev->flags);
+
+ /* Based resource count on the 'new' device caps */
+ rc = bnxt_re_update_dev_attr(rdev);
+ if (rc)
+ goto release_rtnl;
+ rc = bnxt_re_alloc_init_tbls(rdev);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "tbls alloc-init failed rc = %#x",
+ rc);
+ goto release_rtnl;
+ }
+ rc = bnxt_re_setup_nqs(rdev);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev), "NQs alloc-init failed rc = %#x\n",
+ rc);
+ if (rdev->nqr.max_init == 0)
+ goto release_rtnl;
+
+ dev_warn(rdev_to_dev(rdev),
+ "expected nqs %d available nqs %d\n",
+ rdev->nqr.num_msix, rdev->nqr.max_init);
+ }
+ set_bit(BNXT_RE_FLAG_SETUP_NQ, &rdev->flags);
+ rtnl_unlock();
+
+ rc = bnxt_qplib_alloc_dpi(&rdev->qplib_res, &rdev->dpi_privileged,
+ rdev, BNXT_QPLIB_DPI_TYPE_KERNEL);
+ if (rc)
+ goto fail;
+
+ if (rdev->dbr_pacing)
+ bnxt_re_enable_dbr_pacing(rdev);
+
+ if (rdev->chip_ctx->modes.dbr_drop_recov)
+ bnxt_re_initialize_dbr_drop_recov(rdev);
+
+ rc = bnxt_re_alloc_dbr_sw_stats_mem(rdev);
+ if (rc)
+ goto fail;
+
+ /* This block of code is needed for error recovery support */
+ if (!rdev->is_virtfn) {
+ struct bnxt_re_tc_rec *tc_rec;
+
+ tc_rec = &rdev->tc_rec[0];
+ rc = bnxt_re_query_hwrm_qportcfg(rdev, tc_rec, 0xFFFF);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev),
+ "Failed to query port config rc:%d", rc);
+ return rc;
+ }
+
+ /* Query f/w defaults of CC params */
+ rc = bnxt_qplib_query_cc_param(&rdev->qplib_res, &rdev->cc_param);
+ if (rc)
+ dev_warn(rdev_to_dev(rdev),
+ "Failed to query CC defaults\n");
+ if (1) {
+ rdev->num_vfs = pci_num_vf(rdev->en_dev->pdev);
+ if (rdev->num_vfs) {
+ bnxt_re_set_resource_limits(rdev);
+ bnxt_qplib_set_func_resources(&rdev->qplib_res);
+ }
+ }
+ }
+ INIT_DELAYED_WORK(&rdev->worker, bnxt_re_worker);
+ set_bit(BNXT_RE_FLAG_WORKER_REG, &rdev->flags);
+ schedule_delayed_work(&rdev->worker, msecs_to_jiffies(1000));
+
+ bnxt_re_init_dcb_wq(rdev);
+ bnxt_re_init_aer_wq(rdev);
+ bnxt_re_init_resolve_wq(rdev);
+ mutex_lock(&bnxt_re_dev_lock);
+ list_add_tail_rcu(&rdev->list, &bnxt_re_dev_list);
+ /* Added to the list, not in progress anymore */
+ gadd_dev_inprogress--;
+ set_bit(BNXT_RE_FLAG_DEV_LIST_INITIALIZED, &rdev->flags);
+ mutex_unlock(&bnxt_re_dev_lock);
+
+
+ return rc;
+release_rtnl:
+ rtnl_unlock();
+fail:
+ bnxt_re_dev_uninit(rdev, BNXT_RE_COMPLETE_REMOVE);
+
+ return rc;
+}
+
+static int bnxt_re_ib_init(struct bnxt_re_dev *rdev)
+{
+ int rc = 0;
+
+ rc = bnxt_re_register_ib(rdev);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev),
+ "Register IB failed with rc = 0x%x", rc);
+ goto fail;
+ }
+ if (bnxt_re_sysfs_create_file(rdev)) {
+ bnxt_re_stopqps_and_ib_uninit(rdev);
+ goto fail;
+ }
+
+ set_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags);
+ set_bit(BNXT_RE_FLAG_ISSUE_ROCE_STATS, &rdev->flags);
+ set_bit(BNXT_RE_FLAG_ISSUE_CFA_FLOW_STATS, &rdev->flags);
+ bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_PORT_ACTIVE);
+ bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_GID_CHANGE);
+
+ return rc;
+fail:
+ bnxt_re_dev_uninit(rdev, BNXT_RE_COMPLETE_REMOVE);
+ return rc;
+}
+
+/* wrapper for ib_init funcs */
+int _bnxt_re_ib_init(struct bnxt_re_dev *rdev)
+{
+ return bnxt_re_ib_init(rdev);
+}
+
+/* wrapper for aux init funcs */
+int _bnxt_re_ib_init2(struct bnxt_re_dev *rdev)
+{
+ bnxt_re_ib_init_2(rdev);
+ return 0; /* add return for future proof */
+}
+
+static void bnxt_re_dev_unreg(struct bnxt_re_dev *rdev)
+{
+ bnxt_re_dev_dealloc(rdev);
+}
+
+
+static int bnxt_re_dev_reg(struct bnxt_re_dev **rdev, struct ifnet *netdev,
+ struct bnxt_en_dev *en_dev)
+{
+ struct ifnet *realdev = NULL;
+
+ realdev = netdev;
+ if (realdev)
+ dev_dbg(NULL, "%s: realdev = %p netdev = %p\n", __func__,
+ realdev, netdev);
+ /*
+ * Note:
+ * The first argument to bnxt_re_dev_alloc() is 'netdev' and
+ * not 'realdev', since in the case of bonding we want to
+ * register the bonded virtual netdev (master) to the ib stack.
+ * And 'en_dev' (for L2/PCI communication) is the first slave
+ * device (PF0 on the card).
+ * In the case of a regular netdev, both netdev and the en_dev
+ * correspond to the same device.
+ */
+ *rdev = bnxt_re_dev_alloc(netdev, en_dev);
+ if (!*rdev) {
+ pr_err("%s: netdev %p not handled",
+ ROCE_DRV_MODULE_NAME, netdev);
+ return -ENOMEM;
+ }
+ bnxt_re_hold(*rdev);
+
+ return 0;
+}
+
+void bnxt_re_get_link_speed(struct bnxt_re_dev *rdev)
+{
+ rdev->espeed = rdev->en_dev->espeed;
+ return;
+}
+
+void bnxt_re_stopqps_and_ib_uninit(struct bnxt_re_dev *rdev)
+{
+ dev_dbg(rdev_to_dev(rdev), "%s: Stopping QPs, IB uninit on rdev: %p\n",
+ __func__, rdev);
+ bnxt_re_stop_all_nonqp1_nonshadow_qps(rdev);
+ bnxt_re_ib_uninit(rdev);
+}
+
+void bnxt_re_remove_device(struct bnxt_re_dev *rdev, u8 op_type,
+ struct auxiliary_device *aux_dev)
+{
+ struct bnxt_re_en_dev_info *en_info;
+ struct bnxt_qplib_cmdq_ctx *cmdq;
+ struct bnxt_qplib_rcfw *rcfw;
+
+ rcfw = &rdev->rcfw;
+ cmdq = &rcfw->cmdq;
+ if (test_bit(FIRMWARE_STALL_DETECTED, &cmdq->flags))
+ set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags);
+
+ dev_dbg(rdev_to_dev(rdev), "%s: Removing rdev: %p\n", __func__, rdev);
+ bnxt_re_dev_uninit(rdev, op_type);
+ en_info = auxiliary_get_drvdata(aux_dev);
+ if (en_info) {
+ rtnl_lock();
+ en_info->rdev = NULL;
+ rtnl_unlock();
+ if (op_type != BNXT_RE_PRE_RECOVERY_REMOVE) {
+ clear_bit(BNXT_RE_FLAG_EN_DEV_PRIMARY_DEV, &en_info->flags);
+ clear_bit(BNXT_RE_FLAG_EN_DEV_SECONDARY_DEV, &en_info->flags);
+ clear_bit(BNXT_RE_FLAG_EN_DEV_NETDEV_REG, &en_info->flags);
+ }
+ }
+ bnxt_re_dev_unreg(rdev);
+}
+
+int bnxt_re_add_device(struct bnxt_re_dev **rdev,
+ struct ifnet *netdev,
+ u8 qp_mode, u8 op_type, u8 wqe_mode,
+ u32 num_msix_requested,
+ struct auxiliary_device *aux_dev)
+{
+ struct bnxt_re_en_dev_info *en_info;
+ struct bnxt_en_dev *en_dev;
+ int rc = 0;
+
+ en_info = auxiliary_get_drvdata(aux_dev);
+ en_dev = en_info->en_dev;
+
+ mutex_lock(&bnxt_re_dev_lock);
+ /* Check if driver already in mod exit and aux_dev is valid */
+ if (gmod_exit || !aux_dev) {
+ mutex_unlock(&bnxt_re_dev_lock);
+ return -ENODEV;
+ }
+ /* Add device in progress */
+ gadd_dev_inprogress++;
+ mutex_unlock(&bnxt_re_dev_lock);
+
+ rc = bnxt_re_dev_reg(rdev, netdev, en_dev);
+ if (rc) {
+ dev_dbg(NULL, "Failed to create add device for netdev %p\n",
+ netdev);
+ /*
+ * For BNXT_RE_POST_RECOVERY_INIT special case
+ * called from bnxt_re_start, the work is
+ * complete only after, bnxt_re_start completes
+ * bnxt_unregister_device in case of failure.
+ * So bnxt_re_start will decrement gadd_dev_inprogress
+ * in case of failure.
+ */
+ if (op_type != BNXT_RE_POST_RECOVERY_INIT) {
+ mutex_lock(&bnxt_re_dev_lock);
+ gadd_dev_inprogress--;
+ mutex_unlock(&bnxt_re_dev_lock);
+ }
+ return rc;
+ }
+
+ if (rc != 0)
+ goto ref_error;
+
+ /*
+ * num_msix_requested = BNXT_RE_MSIX_FROM_MOD_PARAM indicates fresh driver load.
+ * Otherwaise, this invocation can be the result of lag create / destroy,
+ * err revovery, hot fw upgrade, etc..
+ */
+ if (num_msix_requested == BNXT_RE_MSIX_FROM_MOD_PARAM) {
+ if (bnxt_re_probe_count < BNXT_RE_MAX_DEVICES)
+ num_msix_requested = max_msix_vec[bnxt_re_probe_count++];
+ else
+ /* Consider as default when probe_count exceeds its limit */
+ num_msix_requested = 0;
+
+ /* if user specifies only one value, use the same for all PFs */
+ if (max_msix_vec_argc == 1)
+ num_msix_requested = max_msix_vec[0];
+ }
+
+ (*rdev)->num_msix_requested = num_msix_requested;
+ (*rdev)->gsi_ctx.gsi_qp_mode = qp_mode;
+ (*rdev)->adev = aux_dev;
+ (*rdev)->dev_addr = en_dev->softc->func.mac_addr;
+ /* Before updating the rdev pointer in bnxt_re_en_dev_info structure,
+ * take the rtnl lock to avoid accessing invalid rdev pointer from
+ * L2 ULP callbacks. This is applicable in all the places where rdev
+ * pointer is updated in bnxt_re_en_dev_info.
+ */
+ rtnl_lock();
+ en_info->rdev = *rdev;
+ rtnl_unlock();
+ rc = bnxt_re_dev_init(*rdev, op_type, wqe_mode);
+ if (rc) {
+ref_error:
+ bnxt_re_dev_unreg(*rdev);
+ *rdev = NULL;
+ /*
+ * For BNXT_RE_POST_RECOVERY_INIT special case
+ * called from bnxt_re_start, the work is
+ * complete only after, bnxt_re_start completes
+ * bnxt_unregister_device in case of failure.
+ * So bnxt_re_start will decrement gadd_dev_inprogress
+ * in case of failure.
+ */
+ if (op_type != BNXT_RE_POST_RECOVERY_INIT) {
+ mutex_lock(&bnxt_re_dev_lock);
+ gadd_dev_inprogress--;
+ mutex_unlock(&bnxt_re_dev_lock);
+ }
+ }
+ dev_dbg(rdev_to_dev(*rdev), "%s: Adding rdev: %p\n", __func__, *rdev);
+ if (!rc) {
+ set_bit(BNXT_RE_FLAG_EN_DEV_NETDEV_REG, &en_info->flags);
+ }
+ return rc;
+}
+
+struct bnxt_re_dev *bnxt_re_get_peer_pf(struct bnxt_re_dev *rdev)
+{
+ struct pci_dev *pdev_in = rdev->en_dev->pdev;
+ int tmp_bus_num, bus_num = pdev_in->bus->number;
+ int tmp_dev_num, dev_num = PCI_SLOT(pdev_in->devfn);
+ int tmp_func_num, func_num = PCI_FUNC(pdev_in->devfn);
+ struct bnxt_re_dev *tmp_rdev;
+
+ rcu_read_lock();
+ list_for_each_entry_rcu(tmp_rdev, &bnxt_re_dev_list, list) {
+ tmp_bus_num = tmp_rdev->en_dev->pdev->bus->number;
+ tmp_dev_num = PCI_SLOT(tmp_rdev->en_dev->pdev->devfn);
+ tmp_func_num = PCI_FUNC(tmp_rdev->en_dev->pdev->devfn);
+
+ if (bus_num == tmp_bus_num && dev_num == tmp_dev_num &&
+ func_num != tmp_func_num) {
+ rcu_read_unlock();
+ return tmp_rdev;
+ }
+ }
+ rcu_read_unlock();
+ return NULL;
+}
+
+
+int bnxt_re_schedule_work(struct bnxt_re_dev *rdev, unsigned long event,
+ struct ifnet *vlan_dev,
+ struct ifnet *netdev,
+ struct auxiliary_device *adev)
+{
+ struct bnxt_re_work *re_work;
+
+ /* Allocate for the deferred task */
+ re_work = kzalloc(sizeof(*re_work), GFP_KERNEL);
+ if (!re_work)
+ return -ENOMEM;
+
+ re_work->rdev = rdev;
+ re_work->event = event;
+ re_work->vlan_dev = vlan_dev;
+ re_work->adev = adev;
+ INIT_WORK(&re_work->work, bnxt_re_task);
+ if (rdev)
+ atomic_inc(&rdev->sched_count);
+ re_work->netdev = netdev;
+ queue_work(bnxt_re_wq, &re_work->work);
+
+ return 0;
+}
+
+
+int bnxt_re_get_slot_pf_count(struct bnxt_re_dev *rdev)
+{
+ struct pci_dev *pdev_in = rdev->en_dev->pdev;
+ int tmp_bus_num, bus_num = pdev_in->bus->number;
+ int tmp_dev_num, dev_num = PCI_SLOT(pdev_in->devfn);
+ struct bnxt_re_dev *tmp_rdev;
+ int pf_cnt = 0;
+
+ rcu_read_lock();
+ list_for_each_entry_rcu(tmp_rdev, &bnxt_re_dev_list, list) {
+ tmp_bus_num = tmp_rdev->en_dev->pdev->bus->number;
+ tmp_dev_num = PCI_SLOT(tmp_rdev->en_dev->pdev->devfn);
+
+ if (bus_num == tmp_bus_num && dev_num == tmp_dev_num)
+ pf_cnt++;
+ }
+ rcu_read_unlock();
+ return pf_cnt;
+}
+
+/* Handle all deferred netevents tasks */
+static void bnxt_re_task(struct work_struct *work)
+{
+ struct bnxt_re_en_dev_info *en_info;
+ struct auxiliary_device *aux_dev;
+ struct bnxt_re_work *re_work;
+ struct bnxt_re_dev *rdev;
+
+ re_work = container_of(work, struct bnxt_re_work, work);
+
+ mutex_lock(&bnxt_re_mutex);
+ rdev = re_work->rdev;
+
+ /*
+ * If the previous rdev is deleted due to bond creation
+ * do not handle the event
+ */
+ if (!bnxt_re_is_rdev_valid(rdev))
+ goto exit;
+
+ /* Ignore the event, if the device is not registred with IB stack. This
+ * is to avoid handling any event while the device is added/removed.
+ */
+ if (rdev && !test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) {
+ dev_dbg(rdev_to_dev(rdev), "%s: Ignoring netdev event 0x%lx",
+ __func__, re_work->event);
+ goto done;
+ }
+
+ /* Extra check to silence coverity. We shouldn't handle any event
+ * when rdev is NULL.
+ */
+ if (!rdev)
+ goto exit;
+
+ dev_dbg(rdev_to_dev(rdev), "Scheduled work for event 0x%lx",
+ re_work->event);
+
+ switch (re_work->event) {
+ case NETDEV_UP:
+ bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1,
+ IB_EVENT_PORT_ACTIVE);
+ bnxt_re_net_register_async_event(rdev);
+ break;
+
+ case NETDEV_DOWN:
+ bnxt_qplib_dbr_pacing_set_primary_pf(rdev->chip_ctx, 0);
+ bnxt_re_stop_all_nonqp1_nonshadow_qps(rdev);
+ bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1,
+ IB_EVENT_PORT_ERR);
+ break;
+
+ case NETDEV_CHANGE:
+ if (bnxt_re_get_link_state(rdev) == IB_PORT_DOWN) {
+ bnxt_re_stop_all_nonqp1_nonshadow_qps(rdev);
+ bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1,
+ IB_EVENT_PORT_ERR);
+ break;
+ } else if (bnxt_re_get_link_state(rdev) == IB_PORT_ACTIVE) {
+ bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1,
+ IB_EVENT_PORT_ACTIVE);
+ }
+
+ /* temporarily disable the check for SR2 */
+ if (!bnxt_qplib_query_cc_param(&rdev->qplib_res,
+ &rdev->cc_param) &&
+ !_is_chip_p7(rdev->chip_ctx)) {
+ /*
+ * Disable CC for 10G speed
+ * for non p5 devices
+ */
+ if (rdev->sl_espeed == SPEED_10000 &&
+ !_is_chip_gen_p5_p7(rdev->chip_ctx)) {
+ if (rdev->cc_param.enable)
+ bnxt_re_clear_cc(rdev);
+ } else {
+ if (!rdev->cc_param.enable &&
+ rdev->cc_param.admin_enable)
+ bnxt_re_setup_cc(rdev);
+ }
+ }
+ break;
+
+ case NETDEV_UNREGISTER:
+ bnxt_re_stopqps_and_ib_uninit(rdev);
+ aux_dev = rdev->adev;
+ if (re_work->adev)
+ goto done;
+
+ bnxt_re_remove_device(rdev, BNXT_RE_COMPLETE_REMOVE, aux_dev);
+
+ break;
+
+ default:
+ break;
+ }
+done:
+ if (rdev) {
+ /* memory barrier to guarantee task completion
+ * before decrementing sched count
+ */
+ mmiowb();
+ atomic_dec(&rdev->sched_count);
+ }
+exit:
+ if (re_work->adev && re_work->event == NETDEV_UNREGISTER) {
+ en_info = auxiliary_get_drvdata(re_work->adev);
+ en_info->ib_uninit_done = true;
+ wake_up(&en_info->waitq);
+ }
+ kfree(re_work);
+ mutex_unlock(&bnxt_re_mutex);
+}
+
+/*
+ "Notifier chain callback can be invoked for the same chain from
+ different CPUs at the same time".
+
+ For cases when the netdev is already present, our call to the
+ register_netdevice_notifier() will actually get the rtnl_lock()
+ before sending NETDEV_REGISTER and (if up) NETDEV_UP
+ events.
+
+ But for cases when the netdev is not already present, the notifier
+ chain is subjected to be invoked from different CPUs simultaneously.
+
+ This is protected by the netdev_mutex.
+*/
+static int bnxt_re_netdev_event(struct notifier_block *notifier,
+ unsigned long event, void *ptr)
+{
+ struct ifnet *real_dev, *netdev;
+ struct bnxt_re_dev *rdev = NULL;
+
+ netdev = netdev_notifier_info_to_ifp(ptr);
+ real_dev = rdma_vlan_dev_real_dev(netdev);
+ if (!real_dev)
+ real_dev = netdev;
+ /* In case of bonding,this will be bond's rdev */
+ rdev = bnxt_re_from_netdev(real_dev);
+
+ if (!rdev)
+ goto exit;
+
+ dev_info(rdev_to_dev(rdev), "%s: Event = %s (0x%lx), rdev %s (real_dev %s)\n",
+ __func__, bnxt_re_netevent(event), event,
+ rdev ? rdev->netdev ? rdev->netdev->if_dname : "->netdev = NULL" : "= NULL",
+ (real_dev == netdev) ? "= netdev" : real_dev->if_dname);
+
+ if (!test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags))
+ goto exit;
+
+ bnxt_re_hold(rdev);
+
+ if (real_dev != netdev) {
+ switch (event) {
+ case NETDEV_UP:
+ bnxt_re_schedule_work(rdev, event, netdev,
+ NULL, NULL);
+ break;
+ case NETDEV_DOWN:
+ break;
+ default:
+ break;
+ }
+ goto done;
+ }
+
+ switch (event) {
+ case NETDEV_CHANGEADDR:
+ if (!_is_chip_gen_p5_p7(rdev->chip_ctx))
+ bnxt_re_update_shadow_ah(rdev);
+ bnxt_qplib_get_guid(rdev->dev_addr,
+ (u8 *)&rdev->ibdev.node_guid);
+ break;
+
+ case NETDEV_CHANGE:
+ bnxt_re_get_link_speed(rdev);
+ bnxt_re_schedule_work(rdev, event, NULL, NULL, NULL);
+ break;
+ case NETDEV_UNREGISTER:
+ /* netdev notifier will call NETDEV_UNREGISTER again later since
+ * we are still holding the reference to the netdev
+ */
+
+ /*
+ * Workaround to avoid ib_unregister hang. Check for module
+ * reference and dont free up the device if the reference
+ * is non zero. Checking only for PF functions.
+ */
+
+ if (rdev) {
+ dev_info(rdev_to_dev(rdev),
+ "bnxt_re:Unreg recvd when module refcnt > 0");
+ dev_info(rdev_to_dev(rdev),
+ "bnxt_re:Close all apps using bnxt_re devs");
+ dev_info(rdev_to_dev(rdev),
+ "bnxt_re:Remove the configfs entry created for the device");
+ dev_info(rdev_to_dev(rdev),
+ "bnxt_re:Refer documentation for details");
+ goto done;
+ }
+
+ if (atomic_read(&rdev->sched_count) > 0)
+ goto done;
+ if (!rdev->unreg_sched) {
+ bnxt_re_schedule_work(rdev, NETDEV_UNREGISTER,
+ NULL, NULL, NULL);
+ rdev->unreg_sched = true;
+ goto done;
+ }
+
+ break;
+ default:
+ break;
+ }
+done:
+ if (rdev)
+ bnxt_re_put(rdev);
+exit:
+ return NOTIFY_DONE;
+}
+
+static struct notifier_block bnxt_re_netdev_notifier = {
+ .notifier_call = bnxt_re_netdev_event
+};
+
+static void bnxt_re_remove_base_interface(struct bnxt_re_dev *rdev,
+ struct auxiliary_device *adev)
+{
+ bnxt_re_stopqps_and_ib_uninit(rdev);
+ bnxt_re_remove_device(rdev, BNXT_RE_COMPLETE_REMOVE, adev);
+ auxiliary_set_drvdata(adev, NULL);
+}
+
+/*
+ * bnxt_re_remove - Removes the roce aux device
+ * @adev - aux device pointer
+ *
+ * This function removes the roce device. This gets
+ * called in the mod exit path and pci unbind path.
+ * If the rdev is bond interace, destroys the lag
+ * in module exit path, and in pci unbind case
+ * destroys the lag and recreates other base interface.
+ * If the device is already removed in error recovery
+ * path, it just unregister with the L2.
+ */
+static void bnxt_re_remove(struct auxiliary_device *adev)
+{
+ struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(adev);
+ struct bnxt_en_dev *en_dev;
+ struct bnxt_re_dev *rdev;
+ bool primary_dev = false;
+ bool secondary_dev = false;
+
+ if (!en_info)
+ return;
+
+ mutex_lock(&bnxt_re_mutex);
+ en_dev = en_info->en_dev;
+
+ rdev = en_info->rdev;
+
+ if (rdev && bnxt_re_is_rdev_valid(rdev)) {
+ if (pci_channel_offline(rdev->rcfw.pdev))
+ set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags);
+
+ if (test_bit(BNXT_RE_FLAG_EN_DEV_PRIMARY_DEV, &en_info->flags))
+ primary_dev = true;
+ if (test_bit(BNXT_RE_FLAG_EN_DEV_SECONDARY_DEV, &en_info->flags))
+ secondary_dev = true;
+
+ /*
+ * en_dev_info of primary device and secondary device have the
+ * same rdev pointer when LAG is configured. This rdev pointer
+ * is rdev of bond interface.
+ */
+ if (!primary_dev && !secondary_dev) {
+ /* removal of non bond interface */
+ bnxt_re_remove_base_interface(rdev, adev);
+ } else {
+ /*
+ * removal of bond primary/secondary interface. In this
+ * case bond device is already removed, so rdev->binfo
+ * is NULL.
+ */
+ auxiliary_set_drvdata(adev, NULL);
+ }
+ } else {
+ /* device is removed from ulp stop, unregister the net dev */
+ if (test_bit(BNXT_RE_FLAG_EN_DEV_NETDEV_REG, &en_info->flags)) {
+ rtnl_lock();
+ en_dev->en_ops->bnxt_unregister_device(en_dev,
+ BNXT_ROCE_ULP);
+ rtnl_unlock();
+ }
+ }
+ mutex_unlock(&bnxt_re_mutex);
+ return;
+}
+
+/* wrapper for all external user context callers */
+void _bnxt_re_remove(struct auxiliary_device *adev)
+{
+ bnxt_re_remove(adev);
+}
+
+static void bnxt_re_ib_init_2(struct bnxt_re_dev *rdev)
+{
+ int rc;
+
+ rc = bnxt_re_get_device_stats(rdev);
+ if (rc)
+ dev_err(rdev_to_dev(rdev),
+ "Failed initial device stat query");
+
+ bnxt_re_net_register_async_event(rdev);
+}
+
+static int bnxt_re_probe(struct auxiliary_device *adev,
+ const struct auxiliary_device_id *id)
+{
+ struct bnxt_aux_dev *aux_dev =
+ container_of(adev, struct bnxt_aux_dev, aux_dev);
+ struct bnxt_re_en_dev_info *en_info;
+ struct bnxt_en_dev *en_dev = NULL;
+ struct bnxt_re_dev *rdev;
+ int rc = -ENODEV;
+
+ if (aux_dev)
+ en_dev = aux_dev->edev;
+
+ if (!en_dev)
+ return rc;
+
+ if (en_dev->ulp_version != BNXT_ULP_VERSION) {
+ pr_err("%s: probe error: bnxt_en ulp version magic %x is not compatible!\n",
+ ROCE_DRV_MODULE_NAME, en_dev->ulp_version);
+ return -EINVAL;
+ }
+
+ en_info = kzalloc(sizeof(*en_info), GFP_KERNEL);
+ if (!en_info)
+ return -ENOMEM;
+ memset(en_info, 0, sizeof(struct bnxt_re_en_dev_info));
+ en_info->en_dev = en_dev;
+ auxiliary_set_drvdata(adev, en_info);
+
+ mutex_lock(&bnxt_re_mutex);
+ rc = bnxt_re_add_device(&rdev, en_dev->net,
+ BNXT_RE_GSI_MODE_ALL,
+ BNXT_RE_COMPLETE_INIT,
+ BNXT_QPLIB_WQE_MODE_STATIC,
+ BNXT_RE_MSIX_FROM_MOD_PARAM, adev);
+ if (rc) {
+ mutex_unlock(&bnxt_re_mutex);
+ return rc;
+ }
+
+ rc = bnxt_re_ib_init(rdev);
+ if (rc)
+ goto err;
+
+ bnxt_re_ib_init_2(rdev);
+
+ dev_dbg(rdev_to_dev(rdev), "%s: adev: %p\n", __func__, adev);
+ rdev->adev = adev;
+
+ mutex_unlock(&bnxt_re_mutex);
+
+ return 0;
+
+err:
+ mutex_unlock(&bnxt_re_mutex);
+ bnxt_re_remove(adev);
+
+ return rc;
+}
+
+static const struct auxiliary_device_id bnxt_re_id_table[] = {
+ { .name = BNXT_ADEV_NAME ".rdma", },
+ {},
+};
+
+MODULE_DEVICE_TABLE(auxiliary, bnxt_re_id_table);
+
+static struct auxiliary_driver bnxt_re_driver = {
+ .name = "rdma",
+ .probe = bnxt_re_probe,
+ .remove = bnxt_re_remove,
+ .id_table = bnxt_re_id_table,
+};
+
+static int __init bnxt_re_mod_init(void)
+{
+ int rc = 0;
+
+ pr_info("%s: %s", ROCE_DRV_MODULE_NAME, drv_version);
+
+ bnxt_re_wq = create_singlethread_workqueue("bnxt_re");
+ if (!bnxt_re_wq)
+ return -ENOMEM;
+
+ rc = bnxt_re_register_netdevice_notifier(&bnxt_re_netdev_notifier);
+ if (rc) {
+ pr_err("%s: Cannot register to netdevice_notifier",
+ ROCE_DRV_MODULE_NAME);
+ goto err_netdev;
+ }
+
+ INIT_LIST_HEAD(&bnxt_re_dev_list);
+
+ rc = auxiliary_driver_register(&bnxt_re_driver);
+ if (rc) {
+ pr_err("%s: Failed to register auxiliary driver\n",
+ ROCE_DRV_MODULE_NAME);
+ goto err_auxdrv;
+ }
+
+ return 0;
+
+err_auxdrv:
+ bnxt_re_unregister_netdevice_notifier(&bnxt_re_netdev_notifier);
+
+err_netdev:
+ destroy_workqueue(bnxt_re_wq);
+
+ return rc;
+}
+
+static void __exit bnxt_re_mod_exit(void)
+{
+ gmod_exit = 1;
+ auxiliary_driver_unregister(&bnxt_re_driver);
+
+ bnxt_re_unregister_netdevice_notifier(&bnxt_re_netdev_notifier);
+
+ if (bnxt_re_wq)
+ destroy_workqueue(bnxt_re_wq);
+}
+
+module_init(bnxt_re_mod_init);
+module_exit(bnxt_re_mod_exit);
diff --git a/sys/dev/bnxt/bnxt_re/qplib_fp.c b/sys/dev/bnxt/bnxt_re/qplib_fp.c
new file mode 100644
index 000000000000..438f7ddd468e
--- /dev/null
+++ b/sys/dev/bnxt/bnxt_re/qplib_fp.c
@@ -0,0 +1,3544 @@
+/*
+ * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term
+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Description: Fast Path Operators
+ */
+
+#include <linux/interrupt.h>
+#include <linux/spinlock.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+#include <linux/pci.h>
+#include <linux/delay.h>
+#include <linux/if_ether.h>
+#include <linux/hardirq.h>
+#include <rdma/ib_mad.h>
+
+#include "hsi_struct_def.h"
+#include "qplib_tlv.h"
+#include "qplib_res.h"
+#include "qplib_rcfw.h"
+#include "qplib_sp.h"
+#include "qplib_fp.h"
+#include "ib_verbs.h"
+
+static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp);
+
+static void bnxt_re_legacy_cancel_phantom_processing(struct bnxt_qplib_qp *qp)
+{
+ qp->sq.condition = false;
+ qp->sq.legacy_send_phantom = false;
+ qp->sq.single = false;
+}
+
+static void __bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp)
+{
+ struct bnxt_qplib_cq *scq, *rcq;
+
+ scq = qp->scq;
+ rcq = qp->rcq;
+
+ if (!qp->sq.flushed) {
+ dev_dbg(&scq->hwq.pdev->dev,
+ "QPLIB: FP: Adding to SQ Flush list = %p\n",
+ qp);
+ bnxt_re_legacy_cancel_phantom_processing(qp);
+ list_add_tail(&qp->sq_flush, &scq->sqf_head);
+ qp->sq.flushed = true;
+ }
+ if (!qp->srq) {
+ if (!qp->rq.flushed) {
+ dev_dbg(&rcq->hwq.pdev->dev,
+ "QPLIB: FP: Adding to RQ Flush list = %p\n",
+ qp);
+ list_add_tail(&qp->rq_flush, &rcq->rqf_head);
+ qp->rq.flushed = true;
+ }
+ }
+}
+
+static void bnxt_qplib_acquire_cq_flush_locks(struct bnxt_qplib_qp *qp)
+ __acquires(&qp->scq->flush_lock) __acquires(&qp->rcq->flush_lock)
+{
+ /* Interrupts are already disabled in calling functions */
+ spin_lock(&qp->scq->flush_lock);
+ if (qp->scq == qp->rcq)
+ __acquire(&qp->rcq->flush_lock);
+ else
+ spin_lock(&qp->rcq->flush_lock);
+}
+
+static void bnxt_qplib_release_cq_flush_locks(struct bnxt_qplib_qp *qp)
+ __releases(&qp->scq->flush_lock) __releases(&qp->rcq->flush_lock)
+{
+ if (qp->scq == qp->rcq)
+ __release(&qp->rcq->flush_lock);
+ else
+ spin_unlock(&qp->rcq->flush_lock);
+ spin_unlock(&qp->scq->flush_lock);
+}
+
+void bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp)
+{
+
+ bnxt_qplib_acquire_cq_flush_locks(qp);
+ __bnxt_qplib_add_flush_qp(qp);
+ bnxt_qplib_release_cq_flush_locks(qp);
+}
+
+static void __bnxt_qplib_del_flush_qp(struct bnxt_qplib_qp *qp)
+{
+ if (qp->sq.flushed) {
+ qp->sq.flushed = false;
+ list_del(&qp->sq_flush);
+ }
+ if (!qp->srq) {
+ if (qp->rq.flushed) {
+ qp->rq.flushed = false;
+ list_del(&qp->rq_flush);
+ }
+ }
+}
+
+void bnxt_qplib_clean_qp(struct bnxt_qplib_qp *qp)
+{
+
+ bnxt_qplib_acquire_cq_flush_locks(qp);
+ __clean_cq(qp->scq, (u64)(unsigned long)qp);
+ qp->sq.hwq.prod = 0;
+ qp->sq.hwq.cons = 0;
+ qp->sq.swq_start = 0;
+ qp->sq.swq_last = 0;
+ __clean_cq(qp->rcq, (u64)(unsigned long)qp);
+ qp->rq.hwq.prod = 0;
+ qp->rq.hwq.cons = 0;
+ qp->rq.swq_start = 0;
+ qp->rq.swq_last = 0;
+
+ __bnxt_qplib_del_flush_qp(qp);
+ bnxt_qplib_release_cq_flush_locks(qp);
+}
+
+static void bnxt_qpn_cqn_sched_task(struct work_struct *work)
+{
+ struct bnxt_qplib_nq_work *nq_work =
+ container_of(work, struct bnxt_qplib_nq_work, work);
+
+ struct bnxt_qplib_cq *cq = nq_work->cq;
+ struct bnxt_qplib_nq *nq = nq_work->nq;
+
+ if (cq && nq) {
+ spin_lock_bh(&cq->compl_lock);
+ if (nq->cqn_handler) {
+ dev_dbg(&nq->res->pdev->dev,
+ "%s:Trigger cq = %p event nq = %p\n",
+ __func__, cq, nq);
+ nq->cqn_handler(nq, cq);
+ }
+ spin_unlock_bh(&cq->compl_lock);
+ }
+ kfree(nq_work);
+}
+
+static void bnxt_qplib_put_hdr_buf(struct pci_dev *pdev,
+ struct bnxt_qplib_hdrbuf *buf)
+{
+ dma_free_coherent(&pdev->dev, buf->len, buf->va, buf->dma_map);
+ kfree(buf);
+}
+
+static void *bnxt_qplib_get_hdr_buf(struct pci_dev *pdev, u32 step, u32 cnt)
+{
+ struct bnxt_qplib_hdrbuf *hdrbuf;
+ u32 len;
+
+ hdrbuf = kmalloc(sizeof(*hdrbuf), GFP_KERNEL);
+ if (!hdrbuf)
+ return NULL;
+
+ len = ALIGN((step * cnt), PAGE_SIZE);
+ hdrbuf->va = dma_alloc_coherent(&pdev->dev, len,
+ &hdrbuf->dma_map, GFP_KERNEL);
+ if (!hdrbuf->va)
+ goto out;
+
+ hdrbuf->len = len;
+ hdrbuf->step = step;
+ return hdrbuf;
+out:
+ kfree(hdrbuf);
+ return NULL;
+}
+
+void bnxt_qplib_free_hdr_buf(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_qp *qp)
+{
+ if (qp->rq_hdr_buf) {
+ bnxt_qplib_put_hdr_buf(res->pdev, qp->rq_hdr_buf);
+ qp->rq_hdr_buf = NULL;
+ }
+
+ if (qp->sq_hdr_buf) {
+ bnxt_qplib_put_hdr_buf(res->pdev, qp->sq_hdr_buf);
+ qp->sq_hdr_buf = NULL;
+ }
+}
+
+int bnxt_qplib_alloc_hdr_buf(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_qp *qp, u32 sstep, u32 rstep)
+{
+ struct pci_dev *pdev;
+ int rc = 0;
+
+ pdev = res->pdev;
+ if (sstep) {
+ qp->sq_hdr_buf = bnxt_qplib_get_hdr_buf(pdev, sstep,
+ qp->sq.max_wqe);
+ if (!qp->sq_hdr_buf) {
+ dev_err(&pdev->dev, "QPLIB: Failed to get sq_hdr_buf\n");
+ return -ENOMEM;
+ }
+ }
+
+ if (rstep) {
+ qp->rq_hdr_buf = bnxt_qplib_get_hdr_buf(pdev, rstep,
+ qp->rq.max_wqe);
+ if (!qp->rq_hdr_buf) {
+ rc = -ENOMEM;
+ dev_err(&pdev->dev, "QPLIB: Failed to get rq_hdr_buf\n");
+ goto fail;
+ }
+ }
+
+ return 0;
+fail:
+ bnxt_qplib_free_hdr_buf(res, qp);
+ return rc;
+}
+
+/*
+ * clean_nq - Invalidate cqe from given nq.
+ * @cq - Completion queue
+ *
+ * Traverse whole notification queue and invalidate any completion
+ * associated cq handler provided by caller.
+ * Note - This function traverse the hardware queue but do not update
+ * consumer index. Invalidated cqe(marked from this function) will be
+ * ignored from actual completion of notification queue.
+ */
+static void clean_nq(struct bnxt_qplib_cq *cq)
+{
+ struct bnxt_qplib_hwq *nq_hwq = NULL;
+ struct bnxt_qplib_nq *nq = NULL;
+ struct nq_base *hw_nqe = NULL;
+ struct nq_cn *nqcne = NULL;
+ u32 peek_flags, peek_cons;
+ u64 q_handle;
+ u32 type;
+ int i;
+
+ nq = cq->nq;
+ nq_hwq = &nq->hwq;
+
+ spin_lock_bh(&nq_hwq->lock);
+ peek_flags = nq->nq_db.dbinfo.flags;
+ peek_cons = nq_hwq->cons;
+ for (i = 0; i < nq_hwq->max_elements; i++) {
+ hw_nqe = bnxt_qplib_get_qe(nq_hwq, peek_cons, NULL);
+ if (!NQE_CMP_VALID(hw_nqe, peek_flags))
+ break;
+
+ /* The valid test of the entry must be done first
+ * before reading any further.
+ */
+ dma_rmb();
+ type = le16_to_cpu(hw_nqe->info10_type) &
+ NQ_BASE_TYPE_MASK;
+
+ /* Processing only NQ_BASE_TYPE_CQ_NOTIFICATION */
+ if (type == NQ_BASE_TYPE_CQ_NOTIFICATION) {
+ nqcne = (struct nq_cn *)hw_nqe;
+
+ q_handle = le32_to_cpu(nqcne->cq_handle_low);
+ q_handle |= (u64)le32_to_cpu(nqcne->cq_handle_high) << 32;
+ if (q_handle == (u64)cq) {
+ nqcne->cq_handle_low = 0;
+ nqcne->cq_handle_high = 0;
+ cq->cnq_events++;
+ }
+ }
+ bnxt_qplib_hwq_incr_cons(nq_hwq->max_elements, &peek_cons,
+ 1, &peek_flags);
+ }
+ spin_unlock_bh(&nq_hwq->lock);
+}
+
+/*
+ * Wait for receiving all NQEs for this CQ.
+ * clean_nq is tried 100 times, each time clean_cq
+ * loops upto budget times. budget is based on the
+ * number of CQs shared by that NQ. So any NQE from
+ * CQ would be already in the NQ.
+ */
+static void __wait_for_all_nqes(struct bnxt_qplib_cq *cq, u16 cnq_events)
+{
+ u32 retry_cnt = 100;
+ u16 total_events;
+
+ if (!cnq_events) {
+ clean_nq(cq);
+ return;
+ }
+ while (retry_cnt--) {
+ total_events = cq->cnq_events;
+
+ /* Increment total_events by 1 if any CREQ event received with CQ notification */
+ if (cq->is_cq_err_event)
+ total_events++;
+
+ if (cnq_events == total_events) {
+ dev_dbg(&cq->nq->res->pdev->dev,
+ "QPLIB: NQ cleanup - Received all NQ events\n");
+ return;
+ }
+ msleep(1);
+ clean_nq(cq);
+ }
+}
+
+static void bnxt_qplib_service_nq(unsigned long data)
+{
+ struct bnxt_qplib_nq *nq = (struct bnxt_qplib_nq *)data;
+ struct bnxt_qplib_hwq *nq_hwq = &nq->hwq;
+ int budget = nq->budget;
+ struct bnxt_qplib_res *res;
+ struct bnxt_qplib_cq *cq;
+ struct pci_dev *pdev;
+ struct nq_base *nqe;
+ u32 hw_polled = 0;
+ u64 q_handle;
+ u32 type;
+
+ res = nq->res;
+ pdev = res->pdev;
+
+ spin_lock_bh(&nq_hwq->lock);
+ /* Service the NQ until empty or budget expired */
+ while (budget--) {
+ nqe = bnxt_qplib_get_qe(nq_hwq, nq_hwq->cons, NULL);
+ if (!NQE_CMP_VALID(nqe, nq->nq_db.dbinfo.flags))
+ break;
+ /* The valid test of the entry must be done first before
+ * reading any further.
+ */
+ dma_rmb();
+ type = le16_to_cpu(nqe->info10_type) & NQ_BASE_TYPE_MASK;
+ switch (type) {
+ case NQ_BASE_TYPE_CQ_NOTIFICATION:
+ {
+ struct nq_cn *nqcne = (struct nq_cn *)nqe;
+
+ q_handle = le32_to_cpu(nqcne->cq_handle_low);
+ q_handle |= (u64)le32_to_cpu(nqcne->cq_handle_high) << 32;
+ cq = (struct bnxt_qplib_cq *)q_handle;
+ if (!cq)
+ break;
+ cq->toggle = (le16_to_cpu(nqe->info10_type) & NQ_CN_TOGGLE_MASK) >> NQ_CN_TOGGLE_SFT;
+ cq->dbinfo.toggle = cq->toggle;
+ bnxt_qplib_armen_db(&cq->dbinfo,
+ DBC_DBC_TYPE_CQ_ARMENA);
+ spin_lock_bh(&cq->compl_lock);
+ atomic_set(&cq->arm_state, 0) ;
+ if (!nq->cqn_handler(nq, (cq)))
+ nq->stats.num_cqne_processed++;
+ else
+ dev_warn(&pdev->dev,
+ "QPLIB: cqn - type 0x%x not handled\n",
+ type);
+ cq->cnq_events++;
+ spin_unlock_bh(&cq->compl_lock);
+ break;
+ }
+ case NQ_BASE_TYPE_SRQ_EVENT:
+ {
+ struct bnxt_qplib_srq *srq;
+ struct nq_srq_event *nqsrqe =
+ (struct nq_srq_event *)nqe;
+
+ q_handle = le32_to_cpu(nqsrqe->srq_handle_low);
+ q_handle |= (u64)le32_to_cpu(nqsrqe->srq_handle_high) << 32;
+ srq = (struct bnxt_qplib_srq *)q_handle;
+ bnxt_qplib_armen_db(&srq->dbinfo,
+ DBC_DBC_TYPE_SRQ_ARMENA);
+ if (!nq->srqn_handler(nq,
+ (struct bnxt_qplib_srq *)q_handle,
+ nqsrqe->event))
+ nq->stats.num_srqne_processed++;
+ else
+ dev_warn(&pdev->dev,
+ "QPLIB: SRQ event 0x%x not handled\n",
+ nqsrqe->event);
+ break;
+ }
+ default:
+ dev_warn(&pdev->dev,
+ "QPLIB: nqe with opcode = 0x%x not handled\n",
+ type);
+ break;
+ }
+ hw_polled++;
+ bnxt_qplib_hwq_incr_cons(nq_hwq->max_elements, &nq_hwq->cons,
+ 1, &nq->nq_db.dbinfo.flags);
+ }
+ nqe = bnxt_qplib_get_qe(nq_hwq, nq_hwq->cons, NULL);
+ if (!NQE_CMP_VALID(nqe, nq->nq_db.dbinfo.flags)) {
+ nq->stats.num_nq_rearm++;
+ bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, res->cctx, true);
+ } else if (nq->requested) {
+ bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, res->cctx, true);
+ nq->stats.num_tasklet_resched++;
+ }
+ dev_dbg(&pdev->dev, "QPLIB: cqn/srqn/dbqn \n");
+ if (hw_polled >= 0)
+ dev_dbg(&pdev->dev,
+ "QPLIB: serviced %llu/%llu/%llu budget 0x%x reaped 0x%x\n",
+ nq->stats.num_cqne_processed, nq->stats.num_srqne_processed,
+ nq->stats.num_dbqne_processed, budget, hw_polled);
+ dev_dbg(&pdev->dev,
+ "QPLIB: resched_cnt = %llu arm_count = %llu\n",
+ nq->stats.num_tasklet_resched, nq->stats.num_nq_rearm);
+ spin_unlock_bh(&nq_hwq->lock);
+}
+
+static irqreturn_t bnxt_qplib_nq_irq(int irq, void *dev_instance)
+{
+ struct bnxt_qplib_nq *nq = dev_instance;
+ struct bnxt_qplib_hwq *nq_hwq = &nq->hwq;
+ u32 sw_cons;
+
+ /* Prefetch the NQ element */
+ sw_cons = HWQ_CMP(nq_hwq->cons, nq_hwq);
+ if (sw_cons >= 0)
+ prefetch(bnxt_qplib_get_qe(nq_hwq, sw_cons, NULL));
+
+ bnxt_qplib_service_nq((unsigned long)nq);
+
+ return IRQ_HANDLED;
+}
+
+void bnxt_qplib_nq_stop_irq(struct bnxt_qplib_nq *nq, bool kill)
+{
+ struct bnxt_qplib_res *res;
+
+ if (!nq->requested)
+ return;
+
+ nq->requested = false;
+ res = nq->res;
+ /* Mask h/w interrupt */
+ bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, res->cctx, false);
+ /* Sync with last running IRQ handler */
+ synchronize_irq(nq->msix_vec);
+ free_irq(nq->msix_vec, nq);
+ kfree(nq->name);
+ nq->name = NULL;
+}
+
+void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq)
+{
+ if (nq->cqn_wq) {
+ destroy_workqueue(nq->cqn_wq);
+ nq->cqn_wq = NULL;
+ }
+ /* Make sure the HW is stopped! */
+ bnxt_qplib_nq_stop_irq(nq, true);
+
+ nq->nq_db.reg.bar_reg = NULL;
+ nq->nq_db.db = NULL;
+
+ nq->cqn_handler = NULL;
+ nq->srqn_handler = NULL;
+ nq->msix_vec = 0;
+}
+
+int bnxt_qplib_nq_start_irq(struct bnxt_qplib_nq *nq, int nq_indx,
+ int msix_vector, bool need_init)
+{
+ struct bnxt_qplib_res *res;
+ int rc;
+
+ res = nq->res;
+ if (nq->requested)
+ return -EFAULT;
+
+ nq->msix_vec = msix_vector;
+ nq->name = kasprintf(GFP_KERNEL, "bnxt_re-nq-%d@pci:%s\n",
+ nq_indx, pci_name(res->pdev));
+ if (!nq->name)
+ return -ENOMEM;
+ rc = request_irq(nq->msix_vec, bnxt_qplib_nq_irq, 0, nq->name, nq);
+ if (rc) {
+ kfree(nq->name);
+ nq->name = NULL;
+ return rc;
+ }
+ nq->requested = true;
+ bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, res->cctx, true);
+
+ return rc;
+}
+
+static void bnxt_qplib_map_nq_db(struct bnxt_qplib_nq *nq, u32 reg_offt)
+{
+ struct bnxt_qplib_reg_desc *dbreg;
+ struct bnxt_qplib_nq_db *nq_db;
+ struct bnxt_qplib_res *res;
+
+ nq_db = &nq->nq_db;
+ res = nq->res;
+ dbreg = &res->dpi_tbl.ucreg;
+
+ nq_db->reg.bar_id = dbreg->bar_id;
+ nq_db->reg.bar_base = dbreg->bar_base;
+ nq_db->reg.bar_reg = dbreg->bar_reg + reg_offt;
+ nq_db->reg.len = _is_chip_gen_p5_p7(res->cctx) ? sizeof(u64) :
+ sizeof(u32);
+
+ nq_db->dbinfo.db = nq_db->reg.bar_reg;
+ nq_db->dbinfo.hwq = &nq->hwq;
+ nq_db->dbinfo.xid = nq->ring_id;
+ nq_db->dbinfo.seed = nq->ring_id;
+ nq_db->dbinfo.flags = 0;
+ spin_lock_init(&nq_db->dbinfo.lock);
+ nq_db->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID;
+ nq_db->dbinfo.res = nq->res;
+
+ return;
+}
+
+int bnxt_qplib_enable_nq(struct bnxt_qplib_nq *nq, int nq_idx,
+ int msix_vector, int bar_reg_offset,
+ cqn_handler_t cqn_handler,
+ srqn_handler_t srqn_handler)
+{
+ struct pci_dev *pdev;
+ int rc;
+
+ pdev = nq->res->pdev;
+ nq->cqn_handler = cqn_handler;
+ nq->srqn_handler = srqn_handler;
+ nq->load = 0;
+ mutex_init(&nq->lock);
+
+ /* Have a task to schedule CQ notifiers in post send case */
+ nq->cqn_wq = create_singlethread_workqueue("bnxt_qplib_nq\n");
+ if (!nq->cqn_wq)
+ return -ENOMEM;
+
+ bnxt_qplib_map_nq_db(nq, bar_reg_offset);
+ rc = bnxt_qplib_nq_start_irq(nq, nq_idx, msix_vector, true);
+ if (rc) {
+ dev_err(&pdev->dev,
+ "QPLIB: Failed to request irq for nq-idx %d\n", nq_idx);
+ goto fail;
+ }
+ dev_dbg(&pdev->dev, "QPLIB: NQ max = 0x%x\n", nq->hwq.max_elements);
+
+ return 0;
+fail:
+ bnxt_qplib_disable_nq(nq);
+ return rc;
+}
+
+void bnxt_qplib_free_nq_mem(struct bnxt_qplib_nq *nq)
+{
+ if (nq->hwq.max_elements) {
+ bnxt_qplib_free_hwq(nq->res, &nq->hwq);
+ nq->hwq.max_elements = 0;
+ }
+}
+
+int bnxt_qplib_alloc_nq_mem(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_nq *nq)
+{
+ struct bnxt_qplib_hwq_attr hwq_attr = {};
+ struct bnxt_qplib_sg_info sginfo = {};
+
+ nq->res = res;
+ if (!nq->hwq.max_elements ||
+ nq->hwq.max_elements > BNXT_QPLIB_NQE_MAX_CNT)
+ nq->hwq.max_elements = BNXT_QPLIB_NQE_MAX_CNT;
+
+ sginfo.pgsize = PAGE_SIZE;
+ sginfo.pgshft = PAGE_SHIFT;
+ hwq_attr.res = res;
+ hwq_attr.sginfo = &sginfo;
+ hwq_attr.depth = nq->hwq.max_elements;
+ hwq_attr.stride = sizeof(struct nq_base);
+ hwq_attr.type = _get_hwq_type(res);
+ if (bnxt_qplib_alloc_init_hwq(&nq->hwq, &hwq_attr)) {
+ dev_err(&res->pdev->dev, "QPLIB: FP NQ allocation failed\n");
+ return -ENOMEM;
+ }
+ nq->budget = 8;
+ return 0;
+}
+
+/* SRQ */
+static int __qplib_destroy_srq(struct bnxt_qplib_rcfw *rcfw,
+ struct bnxt_qplib_srq *srq)
+{
+ struct creq_destroy_srq_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct cmdq_destroy_srq req = {};
+ /* Configure the request */
+ req.srq_cid = cpu_to_le32(srq->id);
+ bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DESTROY_SRQ,
+ sizeof(req));
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), 0);
+ return bnxt_qplib_rcfw_send_message(rcfw, &msg);
+}
+
+int bnxt_qplib_destroy_srq(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_srq *srq)
+{
+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+ int rc;
+
+ rc = __qplib_destroy_srq(rcfw, srq);
+ if (rc)
+ return rc;
+ bnxt_qplib_free_hwq(res, &srq->hwq);
+ kfree(srq->swq);
+ return 0;
+}
+
+int bnxt_qplib_create_srq(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_srq *srq)
+{
+ struct bnxt_qplib_hwq_attr hwq_attr = {};
+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+ struct creq_create_srq_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct cmdq_create_srq req = {};
+ u16 pg_sz_lvl = 0;
+ u16 srq_size;
+ int rc, idx;
+
+ hwq_attr.res = res;
+ hwq_attr.sginfo = &srq->sginfo;
+ hwq_attr.depth = srq->max_wqe;
+ hwq_attr.stride = srq->wqe_size;
+ hwq_attr.type = HWQ_TYPE_QUEUE;
+ rc = bnxt_qplib_alloc_init_hwq(&srq->hwq, &hwq_attr);
+ if (rc)
+ goto exit;
+ /* Configure the request */
+ req.dpi = cpu_to_le32(srq->dpi->dpi);
+ req.srq_handle = cpu_to_le64(srq);
+ srq_size = min_t(u32, srq->hwq.depth, U16_MAX);
+ req.srq_size = cpu_to_le16(srq_size);
+ pg_sz_lvl |= (_get_base_pg_size(&srq->hwq) <<
+ CMDQ_CREATE_SRQ_PG_SIZE_SFT);
+ pg_sz_lvl |= (srq->hwq.level & CMDQ_CREATE_SRQ_LVL_MASK);
+ req.pg_size_lvl = cpu_to_le16(pg_sz_lvl);
+ req.pbl = cpu_to_le64(_get_base_addr(&srq->hwq));
+ req.pd_id = cpu_to_le32(srq->pd->id);
+ req.eventq_id = cpu_to_le16(srq->eventq_hw_ring_id);
+ bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_CREATE_SRQ,
+ sizeof(req));
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+ if (rc)
+ goto fail;
+ if (!srq->is_user) {
+ srq->swq = kcalloc(srq->hwq.depth, sizeof(*srq->swq),
+ GFP_KERNEL);
+ if (!srq->swq)
+ goto srq_fail;
+ srq->start_idx = 0;
+ srq->last_idx = srq->hwq.depth - 1;
+ for (idx = 0; idx < srq->hwq.depth; idx++)
+ srq->swq[idx].next_idx = idx + 1;
+ srq->swq[srq->last_idx].next_idx = -1;
+ }
+
+ spin_lock_init(&srq->lock);
+ srq->id = le32_to_cpu(resp.xid);
+ srq->cctx = res->cctx;
+ srq->dbinfo.hwq = &srq->hwq;
+ srq->dbinfo.xid = srq->id;
+ srq->dbinfo.db = srq->dpi->dbr;
+ srq->dbinfo.max_slot = 1;
+ srq->dbinfo.priv_db = res->dpi_tbl.priv_db;
+ srq->dbinfo.flags = 0;
+ spin_lock_init(&srq->dbinfo.lock);
+ srq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID;
+ srq->dbinfo.shadow_key_arm_ena = BNXT_QPLIB_DBR_KEY_INVALID;
+ srq->dbinfo.res = res;
+ srq->dbinfo.seed = srq->id;
+ if (srq->threshold)
+ bnxt_qplib_armen_db(&srq->dbinfo, DBC_DBC_TYPE_SRQ_ARMENA);
+ srq->arm_req = false;
+ return 0;
+srq_fail:
+ __qplib_destroy_srq(rcfw, srq);
+fail:
+ bnxt_qplib_free_hwq(res, &srq->hwq);
+exit:
+ return rc;
+}
+
+int bnxt_qplib_modify_srq(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_srq *srq)
+{
+ struct bnxt_qplib_hwq *srq_hwq = &srq->hwq;
+ u32 avail = 0;
+
+ avail = __bnxt_qplib_get_avail(srq_hwq);
+ if (avail <= srq->threshold) {
+ srq->arm_req = false;
+ bnxt_qplib_srq_arm_db(&srq->dbinfo);
+ } else {
+ /* Deferred arming */
+ srq->arm_req = true;
+ }
+ return 0;
+}
+
+int bnxt_qplib_query_srq(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_srq *srq)
+{
+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+ struct creq_query_srq_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct creq_query_srq_resp_sb *sb;
+ struct bnxt_qplib_rcfw_sbuf sbuf;
+ struct cmdq_query_srq req = {};
+ int rc = 0;
+
+ bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_QUERY_SRQ,
+ sizeof(req));
+ sbuf.size = ALIGN(sizeof(*sb), BNXT_QPLIB_CMDQE_UNITS);
+ sbuf.sb = dma_zalloc_coherent(&rcfw->pdev->dev, sbuf.size,
+ &sbuf.dma_addr, GFP_KERNEL);
+ if (!sbuf.sb)
+ return -ENOMEM;
+ req.resp_size = sbuf.size / BNXT_QPLIB_CMDQE_UNITS;
+ req.srq_cid = cpu_to_le32(srq->id);
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req),
+ sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+ /* TODO: What to do with the query? */
+ dma_free_coherent(&rcfw->pdev->dev, sbuf.size,
+ sbuf.sb, sbuf.dma_addr);
+
+ return rc;
+}
+
+int bnxt_qplib_post_srq_recv(struct bnxt_qplib_srq *srq,
+ struct bnxt_qplib_swqe *wqe)
+{
+ struct bnxt_qplib_hwq *srq_hwq = &srq->hwq;
+ struct sq_sge *hw_sge;
+ struct rq_wqe *srqe;
+ int i, rc = 0, next;
+ u32 avail;
+
+ spin_lock(&srq_hwq->lock);
+ if (srq->start_idx == srq->last_idx) {
+ dev_err(&srq_hwq->pdev->dev, "QPLIB: FP: SRQ (0x%x) is full!\n",
+ srq->id);
+ rc = -EINVAL;
+ spin_unlock(&srq_hwq->lock);
+ goto done;
+ }
+ next = srq->start_idx;
+ srq->start_idx = srq->swq[next].next_idx;
+ spin_unlock(&srq_hwq->lock);
+
+ srqe = bnxt_qplib_get_qe(srq_hwq, srq_hwq->prod, NULL);
+ memset(srqe, 0, srq->wqe_size);
+ /* Calculate wqe_size and data_len */
+ for (i = 0, hw_sge = (struct sq_sge *)srqe->data;
+ i < wqe->num_sge; i++, hw_sge++) {
+ hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr);
+ hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey);
+ hw_sge->size = cpu_to_le32(wqe->sg_list[i].size);
+ }
+ srqe->wqe_type = wqe->type;
+ srqe->flags = wqe->flags;
+ srqe->wqe_size = wqe->num_sge +
+ ((offsetof(typeof(*srqe), data) + 15) >> 4);
+ if (!wqe->num_sge)
+ srqe->wqe_size++;
+ srqe->wr_id |= cpu_to_le32((u32)next);
+ srq->swq[next].wr_id = wqe->wr_id;
+ bnxt_qplib_hwq_incr_prod(&srq->dbinfo, srq_hwq, srq->dbinfo.max_slot);
+ /* retaining srq_hwq->cons for this logic actually the lock is only
+ * required to read srq_hwq->cons.
+ */
+ spin_lock(&srq_hwq->lock);
+ avail = __bnxt_qplib_get_avail(srq_hwq);
+ spin_unlock(&srq_hwq->lock);
+ /* Ring DB */
+ bnxt_qplib_ring_prod_db(&srq->dbinfo, DBC_DBC_TYPE_SRQ);
+ if (srq->arm_req && avail <= srq->threshold) {
+ srq->arm_req = false;
+ bnxt_qplib_srq_arm_db(&srq->dbinfo);
+ }
+done:
+ return rc;
+}
+
+/* QP */
+static int __qplib_destroy_qp(struct bnxt_qplib_rcfw *rcfw,
+ struct bnxt_qplib_qp *qp)
+{
+ struct creq_destroy_qp_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct cmdq_destroy_qp req = {};
+
+ req.qp_cid = cpu_to_le32(qp->id);
+ bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DESTROY_QP,
+ sizeof(req));
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), 0);
+ return bnxt_qplib_rcfw_send_message(rcfw, &msg);
+}
+
+static int bnxt_qplib_alloc_init_swq(struct bnxt_qplib_q *que)
+{
+ int rc = 0;
+ int indx;
+
+ que->swq = kcalloc(que->max_wqe, sizeof(*que->swq), GFP_KERNEL);
+ if (!que->swq) {
+ rc = -ENOMEM;
+ goto out;
+ }
+
+ que->swq_start = 0;
+ que->swq_last = que->max_wqe - 1;
+ for (indx = 0; indx < que->max_wqe; indx++)
+ que->swq[indx].next_idx = indx + 1;
+ que->swq[que->swq_last].next_idx = 0; /* Make it circular */
+ que->swq_last = 0;
+out:
+ return rc;
+}
+
+static struct bnxt_qplib_swq *bnxt_qplib_get_swqe(struct bnxt_qplib_q *que,
+ u32 *swq_idx)
+{
+ u32 idx;
+
+ idx = que->swq_start;
+ if (swq_idx)
+ *swq_idx = idx;
+ return &que->swq[idx];
+}
+
+static void bnxt_qplib_swq_mod_start(struct bnxt_qplib_q *que, u32 idx)
+{
+ que->swq_start = que->swq[idx].next_idx;
+}
+
+static u32 bnxt_qplib_get_stride(void)
+{
+ return sizeof(struct sq_sge);
+}
+
+static u32 bnxt_qplib_get_depth(struct bnxt_qplib_q *que)
+{
+ u8 stride;
+
+ stride = bnxt_qplib_get_stride();
+ return (que->wqe_size * que->max_wqe) / stride;
+}
+
+static u32 _set_sq_size(struct bnxt_qplib_q *que, u8 wqe_mode)
+{
+ /* For Variable mode supply number of 16B slots */
+ return (wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?
+ que->max_wqe : bnxt_qplib_get_depth(que);
+}
+
+static u32 _set_sq_max_slot(u8 wqe_mode)
+{
+ /* for static mode index divisor is 8 */
+ return (wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?
+ sizeof(struct sq_send) / sizeof(struct sq_sge) : 1;
+}
+
+static u32 _set_rq_max_slot(struct bnxt_qplib_q *que)
+{
+ return (que->wqe_size / sizeof(struct sq_sge));
+}
+
+int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
+{
+ struct bnxt_qplib_hwq_attr hwq_attr = {};
+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+ struct creq_create_qp1_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct bnxt_qplib_q *sq = &qp->sq;
+ struct bnxt_qplib_q *rq = &qp->rq;
+ struct cmdq_create_qp1 req = {};
+ struct bnxt_qplib_reftbl *tbl;
+ unsigned long flag;
+ u8 pg_sz_lvl = 0;
+ u32 qp_flags = 0;
+ int rc;
+
+ /* General */
+ req.type = qp->type;
+ req.dpi = cpu_to_le32(qp->dpi->dpi);
+ req.qp_handle = cpu_to_le64(qp->qp_handle);
+ /* SQ */
+ hwq_attr.res = res;
+ hwq_attr.sginfo = &sq->sginfo;
+ hwq_attr.stride = bnxt_qplib_get_stride();
+ hwq_attr.depth = bnxt_qplib_get_depth(sq);
+ hwq_attr.type = HWQ_TYPE_QUEUE;
+ rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr);
+ if (rc)
+ goto exit;
+
+ req.sq_size = cpu_to_le32(_set_sq_size(sq, qp->wqe_mode));
+ req.sq_pbl = cpu_to_le64(_get_base_addr(&sq->hwq));
+ pg_sz_lvl = _get_base_pg_size(&sq->hwq) <<
+ CMDQ_CREATE_QP1_SQ_PG_SIZE_SFT;
+ pg_sz_lvl |= ((sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK) <<
+ CMDQ_CREATE_QP1_SQ_LVL_SFT);
+ req.sq_pg_size_sq_lvl = pg_sz_lvl;
+ req.sq_fwo_sq_sge = cpu_to_le16(((0 << CMDQ_CREATE_QP1_SQ_FWO_SFT) &
+ CMDQ_CREATE_QP1_SQ_FWO_MASK) |
+ (sq->max_sge &
+ CMDQ_CREATE_QP1_SQ_SGE_MASK));
+ req.scq_cid = cpu_to_le32(qp->scq->id);
+
+ /* RQ */
+ if (!qp->srq) {
+ hwq_attr.res = res;
+ hwq_attr.sginfo = &rq->sginfo;
+ hwq_attr.stride = bnxt_qplib_get_stride();
+ hwq_attr.depth = bnxt_qplib_get_depth(rq);
+ hwq_attr.type = HWQ_TYPE_QUEUE;
+ rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr);
+ if (rc)
+ goto fail_sq;
+ req.rq_size = cpu_to_le32(rq->max_wqe);
+ req.rq_pbl = cpu_to_le64(_get_base_addr(&rq->hwq));
+ pg_sz_lvl = _get_base_pg_size(&rq->hwq) <<
+ CMDQ_CREATE_QP1_RQ_PG_SIZE_SFT;
+ pg_sz_lvl |= ((rq->hwq.level & CMDQ_CREATE_QP1_RQ_LVL_MASK) <<
+ CMDQ_CREATE_QP1_RQ_LVL_SFT);
+ req.rq_pg_size_rq_lvl = pg_sz_lvl;
+ req.rq_fwo_rq_sge =
+ cpu_to_le16(((0 << CMDQ_CREATE_QP1_RQ_FWO_SFT) &
+ CMDQ_CREATE_QP1_RQ_FWO_MASK) |
+ (rq->max_sge &
+ CMDQ_CREATE_QP1_RQ_SGE_MASK));
+ } else {
+ /* SRQ */
+ qp_flags |= CMDQ_CREATE_QP1_QP_FLAGS_SRQ_USED;
+ req.srq_cid = cpu_to_le32(qp->srq->id);
+ }
+ req.rcq_cid = cpu_to_le32(qp->rcq->id);
+
+ qp_flags |= CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE;
+ req.qp_flags = cpu_to_le32(qp_flags);
+ req.pd_id = cpu_to_le32(qp->pd->id);
+
+ bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_CREATE_QP1,
+ sizeof(req));
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+ if (rc)
+ goto fail_rq;
+
+ rc = bnxt_qplib_alloc_init_swq(sq);
+ if (rc)
+ goto sq_swq;
+
+ if (!qp->srq) {
+ rc = bnxt_qplib_alloc_init_swq(rq);
+ if (rc)
+ goto rq_swq;
+ }
+
+ qp->id = le32_to_cpu(resp.xid);
+ qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;
+ qp->cctx = res->cctx;
+ sq->dbinfo.hwq = &sq->hwq;
+ sq->dbinfo.xid = qp->id;
+ sq->dbinfo.db = qp->dpi->dbr;
+ sq->dbinfo.max_slot = _set_sq_max_slot(qp->wqe_mode);
+ sq->dbinfo.flags = 0;
+ spin_lock_init(&sq->dbinfo.lock);
+ sq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID;
+ sq->dbinfo.res = res;
+ if (rq->max_wqe) {
+ rq->dbinfo.hwq = &rq->hwq;
+ rq->dbinfo.xid = qp->id;
+ rq->dbinfo.db = qp->dpi->dbr;
+ rq->dbinfo.max_slot = _set_rq_max_slot(rq);
+ rq->dbinfo.flags = 0;
+ spin_lock_init(&rq->dbinfo.lock);
+ rq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID;
+ rq->dbinfo.res = res;
+ }
+
+ tbl = &res->reftbl.qpref;
+ spin_lock_irqsave(&tbl->lock, flag);
+ tbl->rec[tbl->max].xid = qp->id;
+ tbl->rec[tbl->max].handle = qp;
+ spin_unlock_irqrestore(&tbl->lock, flag);
+
+ return 0;
+rq_swq:
+ kfree(sq->swq);
+sq_swq:
+ __qplib_destroy_qp(rcfw, qp);
+fail_rq:
+ bnxt_qplib_free_hwq(res, &rq->hwq);
+fail_sq:
+ bnxt_qplib_free_hwq(res, &sq->hwq);
+exit:
+ return rc;
+}
+
+static void bnxt_qplib_init_psn_ptr(struct bnxt_qplib_qp *qp, int size)
+{
+ struct bnxt_qplib_hwq *sq_hwq;
+ struct bnxt_qplib_q *sq;
+ u64 fpsne, psn_pg;
+ u16 indx_pad = 0;
+
+ sq = &qp->sq;
+ sq_hwq = &sq->hwq;
+ /* First psn entry */
+ fpsne = (u64)bnxt_qplib_get_qe(sq_hwq, sq_hwq->depth, &psn_pg);
+ if (!IS_ALIGNED(fpsne, PAGE_SIZE))
+ indx_pad = (fpsne & ~PAGE_MASK) / size;
+ sq_hwq->pad_pgofft = indx_pad;
+ sq_hwq->pad_pg = (u64 *)psn_pg;
+ sq_hwq->pad_stride = size;
+}
+
+int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
+{
+ struct bnxt_qplib_hwq_attr hwq_attr = {};
+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+ struct bnxt_qplib_sg_info sginfo = {};
+ struct creq_create_qp_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct bnxt_qplib_q *sq = &qp->sq;
+ struct bnxt_qplib_q *rq = &qp->rq;
+ struct cmdq_create_qp req = {};
+ struct bnxt_qplib_reftbl *tbl;
+ struct bnxt_qplib_hwq *xrrq;
+ int rc, req_size, psn_sz;
+ unsigned long flag;
+ u8 pg_sz_lvl = 0;
+ u32 qp_flags = 0;
+ u32 qp_idx;
+ u16 nsge;
+ u32 sqsz;
+
+ qp->cctx = res->cctx;
+ if (res->dattr)
+ qp->dev_cap_flags = res->dattr->dev_cap_flags;
+ /* General */
+ req.type = qp->type;
+ req.dpi = cpu_to_le32(qp->dpi->dpi);
+ req.qp_handle = cpu_to_le64(qp->qp_handle);
+
+ /* SQ */
+ if (qp->type == CMDQ_CREATE_QP_TYPE_RC) {
+ psn_sz = _is_chip_gen_p5_p7(qp->cctx) ?
+ sizeof(struct sq_psn_search_ext) :
+ sizeof(struct sq_psn_search);
+ if (BNXT_RE_HW_RETX(qp->dev_cap_flags)) {
+ psn_sz = sizeof(struct sq_msn_search);
+ qp->msn = 0;
+ }
+ } else {
+ psn_sz = 0;
+ }
+
+ hwq_attr.res = res;
+ hwq_attr.sginfo = &sq->sginfo;
+ hwq_attr.stride = bnxt_qplib_get_stride();
+ hwq_attr.depth = bnxt_qplib_get_depth(sq);
+ hwq_attr.aux_stride = psn_sz;
+ hwq_attr.aux_depth = (psn_sz) ?
+ _set_sq_size(sq, qp->wqe_mode) : 0;
+ /* Update msn tbl size */
+ if (BNXT_RE_HW_RETX(qp->dev_cap_flags) && psn_sz) {
+ if (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
+ hwq_attr.aux_depth = roundup_pow_of_two(_set_sq_size(sq, qp->wqe_mode));
+ else
+ hwq_attr.aux_depth = roundup_pow_of_two(_set_sq_size(sq, qp->wqe_mode)) / 2;
+ qp->msn_tbl_sz = hwq_attr.aux_depth;
+ qp->msn = 0;
+ }
+ hwq_attr.type = HWQ_TYPE_QUEUE;
+ rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr);
+ if (rc)
+ goto exit;
+
+ sqsz = _set_sq_size(sq, qp->wqe_mode);
+ /* 0xffff is the max sq size hw limits to */
+ if (sqsz > BNXT_QPLIB_MAX_SQSZ) {
+ pr_err("QPLIB: FP: QP (0x%x) exceeds sq size %d\n", qp->id, sqsz);
+ goto fail_sq;
+ }
+ req.sq_size = cpu_to_le32(sqsz);
+ req.sq_pbl = cpu_to_le64(_get_base_addr(&sq->hwq));
+ pg_sz_lvl = _get_base_pg_size(&sq->hwq) <<
+ CMDQ_CREATE_QP_SQ_PG_SIZE_SFT;
+ pg_sz_lvl |= ((sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK) <<
+ CMDQ_CREATE_QP_SQ_LVL_SFT);
+ req.sq_pg_size_sq_lvl = pg_sz_lvl;
+ req.sq_fwo_sq_sge = cpu_to_le16(((0 << CMDQ_CREATE_QP_SQ_FWO_SFT) &
+ CMDQ_CREATE_QP_SQ_FWO_MASK) |
+ ((BNXT_RE_HW_RETX(qp->dev_cap_flags)) ?
+ BNXT_MSN_TBLE_SGE : sq->max_sge &
+ CMDQ_CREATE_QP_SQ_SGE_MASK));
+ req.scq_cid = cpu_to_le32(qp->scq->id);
+
+ /* RQ/SRQ */
+ if (!qp->srq) {
+ hwq_attr.res = res;
+ hwq_attr.sginfo = &rq->sginfo;
+ hwq_attr.stride = bnxt_qplib_get_stride();
+ hwq_attr.depth = bnxt_qplib_get_depth(rq);
+ hwq_attr.aux_stride = 0;
+ hwq_attr.aux_depth = 0;
+ hwq_attr.type = HWQ_TYPE_QUEUE;
+ rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr);
+ if (rc)
+ goto fail_sq;
+ req.rq_size = cpu_to_le32(rq->max_wqe);
+ req.rq_pbl = cpu_to_le64(_get_base_addr(&rq->hwq));
+ pg_sz_lvl = _get_base_pg_size(&rq->hwq) <<
+ CMDQ_CREATE_QP_RQ_PG_SIZE_SFT;
+ pg_sz_lvl |= ((rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK) <<
+ CMDQ_CREATE_QP_RQ_LVL_SFT);
+ req.rq_pg_size_rq_lvl = pg_sz_lvl;
+ nsge = (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?
+ res->dattr->max_qp_sges : rq->max_sge;
+ req.rq_fwo_rq_sge =
+ cpu_to_le16(((0 << CMDQ_CREATE_QP_RQ_FWO_SFT) &
+ CMDQ_CREATE_QP_RQ_FWO_MASK) |
+ (nsge & CMDQ_CREATE_QP_RQ_SGE_MASK));
+ } else {
+ qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_SRQ_USED;
+ req.srq_cid = cpu_to_le32(qp->srq->id);
+ }
+ req.rcq_cid = cpu_to_le32(qp->rcq->id);
+
+ qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE;
+ qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED;
+ if (qp->sig_type)
+ qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION;
+ if (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE)
+ qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_VARIABLE_SIZED_WQE_ENABLED;
+ if (res->cctx->modes.te_bypass)
+ qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_OPTIMIZED_TRANSMIT_ENABLED;
+ if (res->dattr &&
+ bnxt_ext_stats_supported(qp->cctx, res->dattr->dev_cap_flags, res->is_vf))
+ qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_EXT_STATS_ENABLED;
+ req.qp_flags = cpu_to_le32(qp_flags);
+
+ /* ORRQ and IRRQ */
+ if (psn_sz) {
+ xrrq = &qp->orrq;
+ xrrq->max_elements =
+ ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic);
+ req_size = xrrq->max_elements *
+ BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE + PAGE_SIZE - 1;
+ req_size &= ~(PAGE_SIZE - 1);
+ sginfo.pgsize = req_size;
+ sginfo.pgshft = PAGE_SHIFT;
+
+ hwq_attr.res = res;
+ hwq_attr.sginfo = &sginfo;
+ hwq_attr.depth = xrrq->max_elements;
+ hwq_attr.stride = BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE;
+ hwq_attr.aux_stride = 0;
+ hwq_attr.aux_depth = 0;
+ hwq_attr.type = HWQ_TYPE_CTX;
+ rc = bnxt_qplib_alloc_init_hwq(xrrq, &hwq_attr);
+ if (rc)
+ goto fail_rq;
+ req.orrq_addr = cpu_to_le64(_get_base_addr(xrrq));
+
+ xrrq = &qp->irrq;
+ xrrq->max_elements = IRD_LIMIT_TO_IRRQ_SLOTS(
+ qp->max_dest_rd_atomic);
+ req_size = xrrq->max_elements *
+ BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE + PAGE_SIZE - 1;
+ req_size &= ~(PAGE_SIZE - 1);
+ sginfo.pgsize = req_size;
+ hwq_attr.depth = xrrq->max_elements;
+ hwq_attr.stride = BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE;
+ rc = bnxt_qplib_alloc_init_hwq(xrrq, &hwq_attr);
+ if (rc)
+ goto fail_orrq;
+ req.irrq_addr = cpu_to_le64(_get_base_addr(xrrq));
+ }
+ req.pd_id = cpu_to_le32(qp->pd->id);
+
+ bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_CREATE_QP,
+ sizeof(req));
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+ if (rc)
+ goto fail;
+
+ if (!qp->is_user) {
+ rc = bnxt_qplib_alloc_init_swq(sq);
+ if (rc)
+ goto swq_sq;
+ if (!qp->srq) {
+ rc = bnxt_qplib_alloc_init_swq(rq);
+ if (rc)
+ goto swq_rq;
+ }
+ if (psn_sz)
+ bnxt_qplib_init_psn_ptr(qp, psn_sz);
+ }
+ qp->id = le32_to_cpu(resp.xid);
+ qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;
+ INIT_LIST_HEAD(&qp->sq_flush);
+ INIT_LIST_HEAD(&qp->rq_flush);
+
+ sq->dbinfo.hwq = &sq->hwq;
+ sq->dbinfo.xid = qp->id;
+ sq->dbinfo.db = qp->dpi->dbr;
+ sq->dbinfo.max_slot = _set_sq_max_slot(qp->wqe_mode);
+ sq->dbinfo.flags = 0;
+ spin_lock_init(&sq->dbinfo.lock);
+ sq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID;
+ sq->dbinfo.res = res;
+ sq->dbinfo.seed = qp->id;
+ if (rq->max_wqe) {
+ rq->dbinfo.hwq = &rq->hwq;
+ rq->dbinfo.xid = qp->id;
+ rq->dbinfo.db = qp->dpi->dbr;
+ rq->dbinfo.max_slot = _set_rq_max_slot(rq);
+ rq->dbinfo.flags = 0;
+ spin_lock_init(&rq->dbinfo.lock);
+ rq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID;
+ rq->dbinfo.res = res;
+ rq->dbinfo.seed = qp->id;
+ }
+
+ tbl = &res->reftbl.qpref;
+ qp_idx = map_qp_id_to_tbl_indx(qp->id, tbl);
+ spin_lock_irqsave(&tbl->lock, flag);
+ tbl->rec[qp_idx].xid = qp->id;
+ tbl->rec[qp_idx].handle = qp;
+ spin_unlock_irqrestore(&tbl->lock, flag);
+
+ return 0;
+swq_rq:
+ kfree(sq->swq);
+swq_sq:
+ __qplib_destroy_qp(rcfw, qp);
+fail:
+ bnxt_qplib_free_hwq(res, &qp->irrq);
+fail_orrq:
+ bnxt_qplib_free_hwq(res, &qp->orrq);
+fail_rq:
+ bnxt_qplib_free_hwq(res, &rq->hwq);
+fail_sq:
+ bnxt_qplib_free_hwq(res, &sq->hwq);
+exit:
+ return rc;
+}
+
+static void __filter_modify_flags(struct bnxt_qplib_qp *qp)
+{
+ switch (qp->cur_qp_state) {
+ case CMDQ_MODIFY_QP_NEW_STATE_RESET:
+ switch (qp->state) {
+ case CMDQ_MODIFY_QP_NEW_STATE_INIT:
+ break;
+ default:
+ break;
+ }
+ break;
+ case CMDQ_MODIFY_QP_NEW_STATE_INIT:
+ switch (qp->state) {
+ case CMDQ_MODIFY_QP_NEW_STATE_RTR:
+ if (!(qp->modify_flags &
+ CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU)) {
+ qp->modify_flags |=
+ CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
+ qp->path_mtu = CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
+ }
+ qp->modify_flags &=
+ ~CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID;
+ /* Bono FW requires the max_dest_rd_atomic to be >= 1 */
+ if (qp->max_dest_rd_atomic < 1)
+ qp->max_dest_rd_atomic = 1;
+ qp->modify_flags &= ~CMDQ_MODIFY_QP_MODIFY_MASK_SRC_MAC;
+ /* Bono FW 20.6.5 requires SGID_INDEX to be configured */
+ if (!(qp->modify_flags &
+ CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX)) {
+ qp->modify_flags |=
+ CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX;
+ qp->ah.sgid_index = 0;
+ }
+ break;
+ default:
+ break;
+ }
+ break;
+ case CMDQ_MODIFY_QP_NEW_STATE_RTR:
+ switch (qp->state) {
+ case CMDQ_MODIFY_QP_NEW_STATE_RTS:
+ /* Bono FW requires the max_rd_atomic to be >= 1 */
+ if (qp->max_rd_atomic < 1)
+ qp->max_rd_atomic = 1;
+ qp->modify_flags &=
+ ~(CMDQ_MODIFY_QP_MODIFY_MASK_PKEY |
+ CMDQ_MODIFY_QP_MODIFY_MASK_DGID |
+ CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL |
+ CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX |
+ CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT |
+ CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS |
+ CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC |
+ CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU |
+ CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN |
+ CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER |
+ CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC |
+ CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID);
+ break;
+ default:
+ break;
+ }
+ break;
+ case CMDQ_MODIFY_QP_NEW_STATE_RTS:
+ break;
+ case CMDQ_MODIFY_QP_NEW_STATE_SQD:
+ break;
+ case CMDQ_MODIFY_QP_NEW_STATE_SQE:
+ break;
+ case CMDQ_MODIFY_QP_NEW_STATE_ERR:
+ break;
+ default:
+ break;
+ }
+}
+
+int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
+{
+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+ struct creq_modify_qp_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct cmdq_modify_qp req = {};
+ bool ppp_requested = false;
+ u32 temp32[4];
+ u32 bmask;
+ int rc;
+
+ bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_MODIFY_QP,
+ sizeof(req));
+
+ /* Filter out the qp_attr_mask based on the state->new transition */
+ __filter_modify_flags(qp);
+ bmask = qp->modify_flags;
+ req.modify_mask = cpu_to_le32(qp->modify_flags);
+ req.qp_cid = cpu_to_le32(qp->id);
+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_STATE) {
+ req.network_type_en_sqd_async_notify_new_state =
+ (qp->state & CMDQ_MODIFY_QP_NEW_STATE_MASK) |
+ (qp->en_sqd_async_notify == true ?
+ CMDQ_MODIFY_QP_EN_SQD_ASYNC_NOTIFY : 0);
+ if (__can_request_ppp(qp)) {
+ req.path_mtu_pingpong_push_enable =
+ CMDQ_MODIFY_QP_PINGPONG_PUSH_ENABLE;
+ req.pingpong_push_dpi = qp->ppp.dpi;
+ ppp_requested = true;
+ }
+ }
+ req.network_type_en_sqd_async_notify_new_state |= qp->nw_type;
+
+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS) {
+ req.access = qp->access;
+ }
+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PKEY)
+ req.pkey = IB_DEFAULT_PKEY_FULL;
+
+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_QKEY) {
+ req.qkey = cpu_to_le32(qp->qkey);
+ }
+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DGID) {
+ memcpy(temp32, qp->ah.dgid.data, sizeof(struct bnxt_qplib_gid));
+ req.dgid[0] = cpu_to_le32(temp32[0]);
+ req.dgid[1] = cpu_to_le32(temp32[1]);
+ req.dgid[2] = cpu_to_le32(temp32[2]);
+ req.dgid[3] = cpu_to_le32(temp32[3]);
+ }
+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL) {
+ req.flow_label = cpu_to_le32(qp->ah.flow_label);
+ }
+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX) {
+ req.sgid_index = cpu_to_le16(res->sgid_tbl.hw_id[qp->ah.sgid_index]);
+ }
+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT) {
+ req.hop_limit = qp->ah.hop_limit;
+ }
+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS) {
+ req.traffic_class = qp->ah.traffic_class;
+ }
+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC) {
+ memcpy(req.dest_mac, qp->ah.dmac, 6);
+ }
+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU) {
+ req.path_mtu_pingpong_push_enable = qp->path_mtu;
+ }
+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT) {
+ req.timeout = qp->timeout;
+ }
+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT) {
+ req.retry_cnt = qp->retry_cnt;
+ }
+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY) {
+ req.rnr_retry = qp->rnr_retry;
+ }
+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER) {
+ req.min_rnr_timer = qp->min_rnr_timer;
+ }
+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN) {
+ req.rq_psn = cpu_to_le32(qp->rq.psn);
+ }
+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN) {
+ req.sq_psn = cpu_to_le32(qp->sq.psn);
+ }
+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC) {
+ req.max_rd_atomic =
+ ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic);
+ }
+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC) {
+ req.max_dest_rd_atomic =
+ IRD_LIMIT_TO_IRRQ_SLOTS(qp->max_dest_rd_atomic);
+ }
+ req.sq_size = cpu_to_le32(qp->sq.hwq.max_elements);
+ req.rq_size = cpu_to_le32(qp->rq.hwq.max_elements);
+ req.sq_sge = cpu_to_le16(qp->sq.max_sge);
+ req.rq_sge = cpu_to_le16(qp->rq.max_sge);
+ req.max_inline_data = cpu_to_le32(qp->max_inline_data);
+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID)
+ req.dest_qp_id = cpu_to_le32(qp->dest_qpn);
+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_ENABLE_CC)
+ req.enable_cc = cpu_to_le16(CMDQ_MODIFY_QP_ENABLE_CC);
+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TOS_ECN)
+ req.tos_dscp_tos_ecn =
+ ((qp->tos_ecn << CMDQ_MODIFY_QP_TOS_ECN_SFT) &
+ CMDQ_MODIFY_QP_TOS_ECN_MASK);
+ if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TOS_DSCP)
+ req.tos_dscp_tos_ecn |=
+ ((qp->tos_dscp << CMDQ_MODIFY_QP_TOS_DSCP_SFT) &
+ CMDQ_MODIFY_QP_TOS_DSCP_MASK);
+ req.vlan_pcp_vlan_dei_vlan_id = cpu_to_le16(qp->vlan_id);
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), 0);
+ msg.qp_state = qp->state;
+
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+ if (rc == -ETIMEDOUT && (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR)) {
+ qp->cur_qp_state = qp->state;
+ return 0;
+ } else if (rc) {
+ return rc;
+ }
+ if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_RTR)
+ qp->lag_src_mac = be32_to_cpu(resp.lag_src_mac);
+
+ if (ppp_requested)
+ qp->ppp.st_idx_en = resp.pingpong_push_state_index_enabled;
+
+ qp->cur_qp_state = qp->state;
+ return 0;
+}
+
+int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
+{
+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+ struct creq_query_qp_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct bnxt_qplib_rcfw_sbuf sbuf;
+ struct creq_query_qp_resp_sb *sb;
+ struct cmdq_query_qp req = {};
+ u32 temp32[4];
+ int i, rc;
+
+ sbuf.size = ALIGN(sizeof(*sb), BNXT_QPLIB_CMDQE_UNITS);
+ sbuf.sb = dma_zalloc_coherent(&rcfw->pdev->dev, sbuf.size,
+ &sbuf.dma_addr, GFP_KERNEL);
+ if (!sbuf.sb)
+ return -ENOMEM;
+ sb = sbuf.sb;
+
+ bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_QUERY_QP,
+ sizeof(req));
+ req.qp_cid = cpu_to_le32(qp->id);
+ req.resp_size = sbuf.size / BNXT_QPLIB_CMDQE_UNITS;
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req),
+ sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+ if (rc)
+ goto bail;
+
+ /* Extract the context from the side buffer */
+ qp->state = sb->en_sqd_async_notify_state &
+ CREQ_QUERY_QP_RESP_SB_STATE_MASK;
+ qp->cur_qp_state = qp->state;
+ qp->en_sqd_async_notify = sb->en_sqd_async_notify_state &
+ CREQ_QUERY_QP_RESP_SB_EN_SQD_ASYNC_NOTIFY ?
+ true : false;
+ qp->access = sb->access;
+ qp->pkey_index = le16_to_cpu(sb->pkey);
+ qp->qkey = le32_to_cpu(sb->qkey);
+
+ temp32[0] = le32_to_cpu(sb->dgid[0]);
+ temp32[1] = le32_to_cpu(sb->dgid[1]);
+ temp32[2] = le32_to_cpu(sb->dgid[2]);
+ temp32[3] = le32_to_cpu(sb->dgid[3]);
+ memcpy(qp->ah.dgid.data, temp32, sizeof(qp->ah.dgid.data));
+
+ qp->ah.flow_label = le32_to_cpu(sb->flow_label);
+
+ qp->ah.sgid_index = 0;
+ for (i = 0; i < res->sgid_tbl.max; i++) {
+ if (res->sgid_tbl.hw_id[i] == le16_to_cpu(sb->sgid_index)) {
+ qp->ah.sgid_index = i;
+ break;
+ }
+ }
+ if (i == res->sgid_tbl.max)
+ dev_dbg(&res->pdev->dev,
+ "QPLIB: SGID not found qp->id = 0x%x sgid_index = 0x%x\n",
+ qp->id, le16_to_cpu(sb->sgid_index));
+
+ qp->ah.hop_limit = sb->hop_limit;
+ qp->ah.traffic_class = sb->traffic_class;
+ memcpy(qp->ah.dmac, sb->dest_mac, ETH_ALEN);
+ qp->ah.vlan_id = le16_to_cpu(sb->path_mtu_dest_vlan_id) &
+ CREQ_QUERY_QP_RESP_SB_VLAN_ID_MASK >>
+ CREQ_QUERY_QP_RESP_SB_VLAN_ID_SFT;
+ qp->path_mtu = le16_to_cpu(sb->path_mtu_dest_vlan_id) &
+ CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK;
+ qp->timeout = sb->timeout;
+ qp->retry_cnt = sb->retry_cnt;
+ qp->rnr_retry = sb->rnr_retry;
+ qp->min_rnr_timer = sb->min_rnr_timer;
+ qp->rq.psn = le32_to_cpu(sb->rq_psn);
+ qp->max_rd_atomic = ORRQ_SLOTS_TO_ORD_LIMIT(sb->max_rd_atomic);
+ qp->sq.psn = le32_to_cpu(sb->sq_psn);
+ qp->max_dest_rd_atomic =
+ IRRQ_SLOTS_TO_IRD_LIMIT(sb->max_dest_rd_atomic);
+ qp->sq.max_wqe = qp->sq.hwq.max_elements;
+ qp->rq.max_wqe = qp->rq.hwq.max_elements;
+ qp->sq.max_sge = le16_to_cpu(sb->sq_sge);
+ qp->rq.max_sge = le16_to_cpu(sb->rq_sge);
+ qp->max_inline_data = le32_to_cpu(sb->max_inline_data);
+ qp->dest_qpn = le32_to_cpu(sb->dest_qp_id);
+ memcpy(qp->smac, sb->src_mac, ETH_ALEN);
+ qp->vlan_id = le16_to_cpu(sb->vlan_pcp_vlan_dei_vlan_id);
+ qp->port_id = le16_to_cpu(sb->port_id);
+bail:
+ dma_free_coherent(&rcfw->pdev->dev, sbuf.size,
+ sbuf.sb, sbuf.dma_addr);
+ return rc;
+}
+
+
+static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp)
+{
+ struct bnxt_qplib_hwq *cq_hwq = &cq->hwq;
+ u32 peek_flags, peek_cons;
+ struct cq_base *hw_cqe;
+ int i;
+
+ peek_flags = cq->dbinfo.flags;
+ peek_cons = cq_hwq->cons;
+ for (i = 0; i < cq_hwq->depth; i++) {
+ hw_cqe = bnxt_qplib_get_qe(cq_hwq, peek_cons, NULL);
+ if (CQE_CMP_VALID(hw_cqe, peek_flags)) {
+ dma_rmb();
+ switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) {
+ case CQ_BASE_CQE_TYPE_REQ:
+ case CQ_BASE_CQE_TYPE_TERMINAL:
+ {
+ struct cq_req *cqe = (struct cq_req *)hw_cqe;
+
+ if (qp == le64_to_cpu(cqe->qp_handle))
+ cqe->qp_handle = 0;
+ break;
+ }
+ case CQ_BASE_CQE_TYPE_RES_RC:
+ case CQ_BASE_CQE_TYPE_RES_UD:
+ case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
+ {
+ struct cq_res_rc *cqe = (struct cq_res_rc *)hw_cqe;
+
+ if (qp == le64_to_cpu(cqe->qp_handle))
+ cqe->qp_handle = 0;
+ break;
+ }
+ default:
+ break;
+ }
+ }
+ bnxt_qplib_hwq_incr_cons(cq_hwq->depth, &peek_cons,
+ 1, &peek_flags);
+ }
+}
+
+int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_qp *qp)
+{
+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+ struct bnxt_qplib_reftbl *tbl;
+ unsigned long flags;
+ u32 qp_idx;
+ int rc;
+
+ tbl = &res->reftbl.qpref;
+ qp_idx = map_qp_id_to_tbl_indx(qp->id, tbl);
+ spin_lock_irqsave(&tbl->lock, flags);
+ tbl->rec[qp_idx].xid = BNXT_QPLIB_QP_ID_INVALID;
+ tbl->rec[qp_idx].handle = NULL;
+ spin_unlock_irqrestore(&tbl->lock, flags);
+
+ rc = __qplib_destroy_qp(rcfw, qp);
+ if (rc) {
+ spin_lock_irqsave(&tbl->lock, flags);
+ tbl->rec[qp_idx].xid = qp->id;
+ tbl->rec[qp_idx].handle = qp;
+ spin_unlock_irqrestore(&tbl->lock, flags);
+ return rc;
+ }
+
+ return 0;
+}
+
+void bnxt_qplib_free_qp_res(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_qp *qp)
+{
+ if (qp->irrq.max_elements)
+ bnxt_qplib_free_hwq(res, &qp->irrq);
+ if (qp->orrq.max_elements)
+ bnxt_qplib_free_hwq(res, &qp->orrq);
+
+ if (!qp->is_user)
+ kfree(qp->rq.swq);
+ bnxt_qplib_free_hwq(res, &qp->rq.hwq);
+
+ if (!qp->is_user)
+ kfree(qp->sq.swq);
+ bnxt_qplib_free_hwq(res, &qp->sq.hwq);
+}
+
+void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp,
+ struct bnxt_qplib_sge *sge)
+{
+ struct bnxt_qplib_q *sq = &qp->sq;
+ struct bnxt_qplib_hdrbuf *buf;
+ u32 sw_prod;
+
+ memset(sge, 0, sizeof(*sge));
+
+ buf = qp->sq_hdr_buf;
+ if (buf) {
+ sw_prod = sq->swq_start;
+ sge->addr = (dma_addr_t)(buf->dma_map + sw_prod * buf->step);
+ sge->lkey = 0xFFFFFFFF;
+ sge->size = buf->step;
+ return buf->va + sw_prod * sge->size;
+ }
+ return NULL;
+}
+
+u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp)
+{
+ struct bnxt_qplib_q *rq = &qp->rq;
+
+ return rq->swq_start;
+}
+
+void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp,
+ struct bnxt_qplib_sge *sge)
+{
+ struct bnxt_qplib_q *rq = &qp->rq;
+ struct bnxt_qplib_hdrbuf *buf;
+ u32 sw_prod;
+
+ memset(sge, 0, sizeof(*sge));
+
+ buf = qp->rq_hdr_buf;
+ if (buf) {
+ sw_prod = rq->swq_start;
+ sge->addr = (dma_addr_t)(buf->dma_map + sw_prod * buf->step);
+ sge->lkey = 0xFFFFFFFF;
+ sge->size = buf->step;
+ return buf->va + sw_prod * sge->size;
+ }
+ return NULL;
+}
+
+/* Fil the MSN table into the next psn row */
+static void bnxt_qplib_fill_msn_search(struct bnxt_qplib_qp *qp,
+ struct bnxt_qplib_swqe *wqe,
+ struct bnxt_qplib_swq *swq)
+{
+ struct sq_msn_search *msns;
+ u32 start_psn, next_psn;
+ u16 start_idx;
+
+ msns = (struct sq_msn_search *)swq->psn_search;
+ msns->start_idx_next_psn_start_psn = 0;
+
+ start_psn = swq->start_psn;
+ next_psn = swq->next_psn;
+ start_idx = swq->slot_idx;
+ msns->start_idx_next_psn_start_psn |=
+ bnxt_re_update_msn_tbl(start_idx, next_psn, start_psn);
+ pr_debug("QP_LIB MSN %d START_IDX %u NEXT_PSN %u START_PSN %u\n",
+ qp->msn,
+ (u16)
+ cpu_to_le16(BNXT_RE_MSN_IDX(msns->start_idx_next_psn_start_psn)),
+ (u32)
+ cpu_to_le32(BNXT_RE_MSN_NPSN(msns->start_idx_next_psn_start_psn)),
+ (u32)
+ cpu_to_le32(BNXT_RE_MSN_SPSN(msns->start_idx_next_psn_start_psn)));
+ qp->msn++;
+ qp->msn %= qp->msn_tbl_sz;
+}
+
+static void bnxt_qplib_fill_psn_search(struct bnxt_qplib_qp *qp,
+ struct bnxt_qplib_swqe *wqe,
+ struct bnxt_qplib_swq *swq)
+{
+ struct sq_psn_search_ext *psns_ext;
+ struct sq_psn_search *psns;
+ u32 flg_npsn;
+ u32 op_spsn;
+
+ if (!swq->psn_search)
+ return;
+
+ /* Handle MSN differently on cap flags */
+ if (BNXT_RE_HW_RETX(qp->dev_cap_flags)) {
+ bnxt_qplib_fill_msn_search(qp, wqe, swq);
+ return;
+ }
+ psns = (struct sq_psn_search *)swq->psn_search;
+ psns_ext = (struct sq_psn_search_ext *)swq->psn_search;
+
+ op_spsn = ((swq->start_psn << SQ_PSN_SEARCH_START_PSN_SFT) &
+ SQ_PSN_SEARCH_START_PSN_MASK);
+ op_spsn |= ((wqe->type << SQ_PSN_SEARCH_OPCODE_SFT) &
+ SQ_PSN_SEARCH_OPCODE_MASK);
+ flg_npsn = ((swq->next_psn << SQ_PSN_SEARCH_NEXT_PSN_SFT) &
+ SQ_PSN_SEARCH_NEXT_PSN_MASK);
+
+ if (_is_chip_gen_p5_p7(qp->cctx)) {
+ psns_ext->opcode_start_psn = cpu_to_le32(op_spsn);
+ psns_ext->flags_next_psn = cpu_to_le32(flg_npsn);
+ psns_ext->start_slot_idx = cpu_to_le16(swq->slot_idx);
+ } else {
+ psns->opcode_start_psn = cpu_to_le32(op_spsn);
+ psns->flags_next_psn = cpu_to_le32(flg_npsn);
+ }
+}
+
+static u16 _calc_ilsize(struct bnxt_qplib_swqe *wqe)
+{
+ u16 size = 0;
+ int indx;
+
+ for (indx = 0; indx < wqe->num_sge; indx++)
+ size += wqe->sg_list[indx].size;
+ return size;
+}
+
+static int bnxt_qplib_put_inline(struct bnxt_qplib_qp *qp,
+ struct bnxt_qplib_swqe *wqe,
+ u32 *sw_prod)
+{
+ struct bnxt_qplib_hwq *sq_hwq;
+ int len, t_len, offt = 0;
+ int t_cplen = 0, cplen;
+ bool pull_dst = true;
+ void *il_dst = NULL;
+ void *il_src = NULL;
+ int indx;
+
+ sq_hwq = &qp->sq.hwq;
+ t_len = 0;
+ for (indx = 0; indx < wqe->num_sge; indx++) {
+ len = wqe->sg_list[indx].size;
+ il_src = (void *)wqe->sg_list[indx].addr;
+ t_len += len;
+ if (t_len > qp->max_inline_data)
+ goto bad;
+ while (len) {
+ if (pull_dst) {
+ pull_dst = false;
+ il_dst = bnxt_qplib_get_qe(sq_hwq, ((*sw_prod) %
+ sq_hwq->depth), NULL);
+ (*sw_prod)++;
+ t_cplen = 0;
+ offt = 0;
+ }
+ cplen = min_t(int, len, sizeof(struct sq_sge));
+ cplen = min_t(int, cplen,
+ (sizeof(struct sq_sge) - offt));
+ memcpy(il_dst, il_src, cplen);
+ t_cplen += cplen;
+ il_src += cplen;
+ il_dst += cplen;
+ offt += cplen;
+ len -= cplen;
+ if (t_cplen == sizeof(struct sq_sge))
+ pull_dst = true;
+ }
+ }
+
+ return t_len;
+bad:
+ return -ENOMEM;
+}
+
+static int bnxt_qplib_put_sges(struct bnxt_qplib_hwq *sq_hwq,
+ struct bnxt_qplib_sge *ssge,
+ u32 nsge, u32 *sw_prod)
+{
+ struct sq_sge *dsge;
+ int indx, len = 0;
+
+ for (indx = 0; indx < nsge; indx++, (*sw_prod)++) {
+ dsge = bnxt_qplib_get_qe(sq_hwq, ((*sw_prod) % sq_hwq->depth), NULL);
+ dsge->va_or_pa = cpu_to_le64(ssge[indx].addr);
+ dsge->l_key = cpu_to_le32(ssge[indx].lkey);
+ dsge->size = cpu_to_le32(ssge[indx].size);
+ len += ssge[indx].size;
+ }
+ return len;
+}
+
+static u16 _calculate_wqe_byte(struct bnxt_qplib_qp *qp,
+ struct bnxt_qplib_swqe *wqe, u16 *wqe_byte)
+{
+ u16 wqe_size;
+ u32 ilsize;
+ u16 nsge;
+
+ nsge = wqe->num_sge;
+ if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) {
+ ilsize = _calc_ilsize(wqe);
+ wqe_size = (ilsize > qp->max_inline_data) ?
+ qp->max_inline_data : ilsize;
+ wqe_size = ALIGN(wqe_size, sizeof(struct sq_sge));
+ } else {
+ wqe_size = nsge * sizeof(struct sq_sge);
+ }
+ /* Adding sq_send_hdr is a misnomer, for rq also hdr size is same. */
+ wqe_size += sizeof(struct sq_send_hdr);
+ if (wqe_byte)
+ *wqe_byte = wqe_size;
+ return wqe_size / sizeof(struct sq_sge);
+}
+
+static u16 _translate_q_full_delta(struct bnxt_qplib_q *que, u16 wqe_bytes)
+{
+ /* For Cu/Wh delta = 128, stride = 16, wqe_bytes = 128
+ * For Gen-p5 B/C mode delta = 0, stride = 16, wqe_bytes = 128.
+ * For Gen-p5 delta = 0, stride = 16, 32 <= wqe_bytes <= 512.
+ * when 8916 is disabled.
+ */
+ return (que->q_full_delta * wqe_bytes) / que->hwq.element_size;
+}
+
+static void bnxt_qplib_pull_psn_buff(struct bnxt_qplib_qp *qp, struct bnxt_qplib_q *sq,
+ struct bnxt_qplib_swq *swq, bool hw_retx)
+{
+ struct bnxt_qplib_hwq *sq_hwq;
+ u32 pg_num, pg_indx;
+ void *buff;
+ u32 tail;
+
+ sq_hwq = &sq->hwq;
+ if (!sq_hwq->pad_pg)
+ return;
+
+ tail = swq->slot_idx / sq->dbinfo.max_slot;
+ if (hw_retx)
+ tail %= qp->msn_tbl_sz;
+ pg_num = (tail + sq_hwq->pad_pgofft) / (PAGE_SIZE / sq_hwq->pad_stride);
+ pg_indx = (tail + sq_hwq->pad_pgofft) % (PAGE_SIZE / sq_hwq->pad_stride);
+ buff = (void *)(sq_hwq->pad_pg[pg_num] + pg_indx * sq_hwq->pad_stride);
+ /* the start ptr for buff is same ie after the SQ */
+ swq->psn_search = buff;
+}
+
+void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp)
+{
+ struct bnxt_qplib_q *sq = &qp->sq;
+
+ bnxt_qplib_ring_prod_db(&sq->dbinfo, DBC_DBC_TYPE_SQ);
+}
+
+int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp,
+ struct bnxt_qplib_swqe *wqe)
+{
+ struct bnxt_qplib_nq_work *nq_work = NULL;
+ int i, rc = 0, data_len = 0, pkt_num = 0;
+ struct bnxt_qplib_q *sq = &qp->sq;
+ struct bnxt_qplib_hwq *sq_hwq;
+ struct bnxt_qplib_swq *swq;
+ bool sch_handler = false;
+ u16 slots_needed;
+ void *base_hdr;
+ void *ext_hdr;
+ __le32 temp32;
+ u16 qfd_slots;
+ u8 wqe_slots;
+ u16 wqe_size;
+ u32 sw_prod;
+ u32 wqe_idx;
+
+ sq_hwq = &sq->hwq;
+ if (qp->state != CMDQ_MODIFY_QP_NEW_STATE_RTS &&
+ qp->state != CMDQ_MODIFY_QP_NEW_STATE_ERR) {
+ dev_err(&sq_hwq->pdev->dev,
+ "QPLIB: FP: QP (0x%x) is in the 0x%x state\n",
+ qp->id, qp->state);
+ rc = -EINVAL;
+ goto done;
+ }
+
+ wqe_slots = _calculate_wqe_byte(qp, wqe, &wqe_size);
+ slots_needed = (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?
+ sq->dbinfo.max_slot : wqe_slots;
+ qfd_slots = _translate_q_full_delta(sq, wqe_size);
+ if (bnxt_qplib_queue_full(sq_hwq, (slots_needed + qfd_slots))) {
+ dev_err(&sq_hwq->pdev->dev,
+ "QPLIB: FP: QP (0x%x) SQ is full!\n", qp->id);
+ dev_err(&sq_hwq->pdev->dev,
+ "QPLIB: prod = %#x cons = %#x qdepth = %#x delta = %#x slots = %#x\n",
+ HWQ_CMP(sq_hwq->prod, sq_hwq),
+ HWQ_CMP(sq_hwq->cons, sq_hwq),
+ sq_hwq->max_elements, qfd_slots, slots_needed);
+ dev_err(&sq_hwq->pdev->dev,
+ "QPLIB: phantom_wqe_cnt: %d phantom_cqe_cnt: %d\n",
+ sq->phantom_wqe_cnt, sq->phantom_cqe_cnt);
+ rc = -ENOMEM;
+ goto done;
+ }
+
+ sw_prod = sq_hwq->prod;
+ swq = bnxt_qplib_get_swqe(sq, &wqe_idx);
+ swq->slot_idx = sw_prod;
+ bnxt_qplib_pull_psn_buff(qp, sq, swq, BNXT_RE_HW_RETX(qp->dev_cap_flags));
+
+ swq->wr_id = wqe->wr_id;
+ swq->type = wqe->type;
+ swq->flags = wqe->flags;
+ swq->slots = slots_needed;
+ swq->start_psn = sq->psn & BTH_PSN_MASK;
+ if (qp->sig_type || wqe->flags & BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP)
+ swq->flags |= SQ_SEND_FLAGS_SIGNAL_COMP;
+
+ dev_dbg(&sq_hwq->pdev->dev,
+ "QPLIB: FP: QP(0x%x) post SQ wr_id[%d] = 0x%llx\n",
+ qp->id, wqe_idx, swq->wr_id);
+ if (qp->cur_qp_state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
+ sch_handler = true;
+ dev_dbg(&sq_hwq->pdev->dev,
+ "%s Error QP. Scheduling for poll_cq\n", __func__);
+ goto queue_err;
+ }
+
+ base_hdr = bnxt_qplib_get_qe(sq_hwq, sw_prod, NULL);
+ sw_prod++;
+ ext_hdr = bnxt_qplib_get_qe(sq_hwq, (sw_prod % sq_hwq->depth), NULL);
+ sw_prod++;
+ memset(base_hdr, 0, sizeof(struct sq_sge));
+ memset(ext_hdr, 0, sizeof(struct sq_sge));
+
+ if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE)
+ data_len = bnxt_qplib_put_inline(qp, wqe, &sw_prod);
+ else
+ data_len = bnxt_qplib_put_sges(sq_hwq, wqe->sg_list,
+ wqe->num_sge, &sw_prod);
+ if (data_len < 0)
+ goto queue_err;
+ /* Specifics */
+ switch (wqe->type) {
+ case BNXT_QPLIB_SWQE_TYPE_SEND:
+ if (qp->type == CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE ||
+ qp->type == CMDQ_CREATE_QP1_TYPE_GSI) {
+ /* Assemble info for Raw Ethertype QPs */
+ struct sq_send_raweth_qp1_hdr *sqe = base_hdr;
+ struct sq_raw_ext_hdr *ext_sqe = ext_hdr;
+
+ sqe->wqe_type = wqe->type;
+ sqe->flags = wqe->flags;
+ sqe->wqe_size = wqe_slots;
+ sqe->cfa_action = cpu_to_le16(wqe->rawqp1.cfa_action);
+ sqe->lflags = cpu_to_le16(wqe->rawqp1.lflags);
+ sqe->length = cpu_to_le32(data_len);
+ ext_sqe->cfa_meta = cpu_to_le32((wqe->rawqp1.cfa_meta &
+ SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK) <<
+ SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT);
+
+ dev_dbg(&sq_hwq->pdev->dev,
+ "QPLIB: FP: RAW/QP1 Send WQE:\n"
+ "\twqe_type = 0x%x\n"
+ "\tflags = 0x%x\n"
+ "\twqe_size = 0x%x\n"
+ "\tlflags = 0x%x\n"
+ "\tcfa_action = 0x%x\n"
+ "\tlength = 0x%x\n"
+ "\tcfa_meta = 0x%x\n",
+ sqe->wqe_type, sqe->flags, sqe->wqe_size,
+ sqe->lflags, sqe->cfa_action,
+ sqe->length, ext_sqe->cfa_meta);
+ break;
+ }
+ fallthrough;
+ case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM:
+ fallthrough;
+ case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV:
+ {
+ struct sq_send_hdr *sqe = base_hdr;
+ struct sq_ud_ext_hdr *ext_sqe = ext_hdr;
+
+ sqe->wqe_type = wqe->type;
+ sqe->flags = wqe->flags;
+ sqe->wqe_size = wqe_slots;
+ sqe->inv_key_or_imm_data = cpu_to_le32(wqe->send.inv_key);
+ if (qp->type == CMDQ_CREATE_QP_TYPE_UD ||
+ qp->type == CMDQ_CREATE_QP_TYPE_GSI) {
+ sqe->q_key = cpu_to_le32(wqe->send.q_key);
+ sqe->length = cpu_to_le32(data_len);
+ ext_sqe->dst_qp = cpu_to_le32(
+ wqe->send.dst_qp & SQ_SEND_DST_QP_MASK);
+ ext_sqe->avid = cpu_to_le32(wqe->send.avid &
+ SQ_SEND_AVID_MASK);
+ sq->psn = (sq->psn + 1) & BTH_PSN_MASK;
+ } else {
+ sqe->length = cpu_to_le32(data_len);
+ if (qp->mtu)
+ pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
+ if (!pkt_num)
+ pkt_num = 1;
+ sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;
+ }
+ dev_dbg(&sq_hwq->pdev->dev,
+ "QPLIB: FP: Send WQE:\n"
+ "\twqe_type = 0x%x\n"
+ "\tflags = 0x%x\n"
+ "\twqe_size = 0x%x\n"
+ "\tinv_key/immdata = 0x%x\n"
+ "\tq_key = 0x%x\n"
+ "\tdst_qp = 0x%x\n"
+ "\tlength = 0x%x\n"
+ "\tavid = 0x%x\n",
+ sqe->wqe_type, sqe->flags, sqe->wqe_size,
+ sqe->inv_key_or_imm_data, sqe->q_key, ext_sqe->dst_qp,
+ sqe->length, ext_sqe->avid);
+ break;
+ }
+ case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE:
+ /* fall-thru */
+ case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM:
+ /* fall-thru */
+ case BNXT_QPLIB_SWQE_TYPE_RDMA_READ:
+ {
+ struct sq_rdma_hdr *sqe = base_hdr;
+ struct sq_rdma_ext_hdr *ext_sqe = ext_hdr;
+
+ sqe->wqe_type = wqe->type;
+ sqe->flags = wqe->flags;
+ sqe->wqe_size = wqe_slots;
+ sqe->imm_data = cpu_to_le32(wqe->rdma.inv_key);
+ sqe->length = cpu_to_le32((u32)data_len);
+ ext_sqe->remote_va = cpu_to_le64(wqe->rdma.remote_va);
+ ext_sqe->remote_key = cpu_to_le32(wqe->rdma.r_key);
+ if (qp->mtu)
+ pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
+ if (!pkt_num)
+ pkt_num = 1;
+ sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;
+
+ dev_dbg(&sq_hwq->pdev->dev,
+ "QPLIB: FP: RDMA WQE:\n"
+ "\twqe_type = 0x%x\n"
+ "\tflags = 0x%x\n"
+ "\twqe_size = 0x%x\n"
+ "\timmdata = 0x%x\n"
+ "\tlength = 0x%x\n"
+ "\tremote_va = 0x%llx\n"
+ "\tremote_key = 0x%x\n",
+ sqe->wqe_type, sqe->flags, sqe->wqe_size,
+ sqe->imm_data, sqe->length, ext_sqe->remote_va,
+ ext_sqe->remote_key);
+ break;
+ }
+ case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP:
+ /* fall-thru */
+ case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD:
+ {
+ struct sq_atomic_hdr *sqe = base_hdr;
+ struct sq_atomic_ext_hdr *ext_sqe = ext_hdr;
+
+ sqe->wqe_type = wqe->type;
+ sqe->flags = wqe->flags;
+ sqe->remote_key = cpu_to_le32(wqe->atomic.r_key);
+ sqe->remote_va = cpu_to_le64(wqe->atomic.remote_va);
+ ext_sqe->swap_data = cpu_to_le64(wqe->atomic.swap_data);
+ ext_sqe->cmp_data = cpu_to_le64(wqe->atomic.cmp_data);
+ if (qp->mtu)
+ pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
+ if (!pkt_num)
+ pkt_num = 1;
+ sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;
+ break;
+ }
+ case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV:
+ {
+ struct sq_localinvalidate_hdr *sqe = base_hdr;
+
+ sqe->wqe_type = wqe->type;
+ sqe->flags = wqe->flags;
+ sqe->inv_l_key = cpu_to_le32(wqe->local_inv.inv_l_key);
+
+ dev_dbg(&sq_hwq->pdev->dev,
+ "QPLIB: FP: LOCAL INV WQE:\n"
+ "\twqe_type = 0x%x\n"
+ "\tflags = 0x%x\n"
+ "\tinv_l_key = 0x%x\n",
+ sqe->wqe_type, sqe->flags, sqe->inv_l_key);
+ break;
+ }
+ case BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR:
+ {
+ struct sq_fr_pmr_hdr *sqe = base_hdr;
+ struct sq_fr_pmr_ext_hdr *ext_sqe = ext_hdr;
+
+ sqe->wqe_type = wqe->type;
+ sqe->flags = wqe->flags;
+ sqe->access_cntl = wqe->frmr.access_cntl |
+ SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE;
+ sqe->zero_based_page_size_log =
+ (wqe->frmr.pg_sz_log & SQ_FR_PMR_PAGE_SIZE_LOG_MASK) <<
+ SQ_FR_PMR_PAGE_SIZE_LOG_SFT |
+ (wqe->frmr.zero_based == true ? SQ_FR_PMR_ZERO_BASED : 0);
+ sqe->l_key = cpu_to_le32(wqe->frmr.l_key);
+ /* TODO: OFED only provides length of MR up to 32-bits for FRMR */
+ temp32 = cpu_to_le32(wqe->frmr.length);
+ memcpy(sqe->length, &temp32, sizeof(wqe->frmr.length));
+ sqe->numlevels_pbl_page_size_log =
+ ((wqe->frmr.pbl_pg_sz_log <<
+ SQ_FR_PMR_PBL_PAGE_SIZE_LOG_SFT) &
+ SQ_FR_PMR_PBL_PAGE_SIZE_LOG_MASK) |
+ ((wqe->frmr.levels << SQ_FR_PMR_NUMLEVELS_SFT) &
+ SQ_FR_PMR_NUMLEVELS_MASK);
+ if (!wqe->frmr.levels && !wqe->frmr.pbl_ptr) {
+ ext_sqe->pblptr = cpu_to_le64(wqe->frmr.page_list[0]);
+ } else {
+ for (i = 0; i < wqe->frmr.page_list_len; i++)
+ wqe->frmr.pbl_ptr[i] = cpu_to_le64(
+ wqe->frmr.page_list[i] |
+ PTU_PTE_VALID);
+ ext_sqe->pblptr = cpu_to_le64(wqe->frmr.pbl_dma_ptr);
+ }
+ ext_sqe->va = cpu_to_le64(wqe->frmr.va);
+ dev_dbg(&sq_hwq->pdev->dev,
+ "QPLIB: FP: FRMR WQE:\n"
+ "\twqe_type = 0x%x\n"
+ "\tflags = 0x%x\n"
+ "\taccess_cntl = 0x%x\n"
+ "\tzero_based_page_size_log = 0x%x\n"
+ "\tl_key = 0x%x\n"
+ "\tlength = 0x%x\n"
+ "\tnumlevels_pbl_page_size_log = 0x%x\n"
+ "\tpblptr = 0x%llx\n"
+ "\tva = 0x%llx\n",
+ sqe->wqe_type, sqe->flags, sqe->access_cntl,
+ sqe->zero_based_page_size_log, sqe->l_key,
+ *(u32 *)sqe->length, sqe->numlevels_pbl_page_size_log,
+ ext_sqe->pblptr, ext_sqe->va);
+ break;
+ }
+ case BNXT_QPLIB_SWQE_TYPE_BIND_MW:
+ {
+ struct sq_bind_hdr *sqe = base_hdr;
+ struct sq_bind_ext_hdr *ext_sqe = ext_hdr;
+
+ sqe->wqe_type = wqe->type;
+ sqe->flags = wqe->flags;
+ sqe->access_cntl = wqe->bind.access_cntl;
+ sqe->mw_type_zero_based = wqe->bind.mw_type |
+ (wqe->bind.zero_based == true ? SQ_BIND_ZERO_BASED : 0);
+ sqe->parent_l_key = cpu_to_le32(wqe->bind.parent_l_key);
+ sqe->l_key = cpu_to_le32(wqe->bind.r_key);
+ ext_sqe->va = cpu_to_le64(wqe->bind.va);
+ ext_sqe->length_lo = cpu_to_le32(wqe->bind.length);
+ dev_dbg(&sq_hwq->pdev->dev,
+ "QPLIB: FP: BIND WQE:\n"
+ "\twqe_type = 0x%x\n"
+ "\tflags = 0x%x\n"
+ "\taccess_cntl = 0x%x\n"
+ "\tmw_type_zero_based = 0x%x\n"
+ "\tparent_l_key = 0x%x\n"
+ "\tl_key = 0x%x\n"
+ "\tva = 0x%llx\n"
+ "\tlength = 0x%x\n",
+ sqe->wqe_type, sqe->flags, sqe->access_cntl,
+ sqe->mw_type_zero_based, sqe->parent_l_key,
+ sqe->l_key, sqe->va, ext_sqe->length_lo);
+ break;
+ }
+ default:
+ /* Bad wqe, return error */
+ rc = -EINVAL;
+ goto done;
+ }
+ swq->next_psn = sq->psn & BTH_PSN_MASK;
+ bnxt_qplib_fill_psn_search(qp, wqe, swq);
+
+queue_err:
+ bnxt_qplib_swq_mod_start(sq, wqe_idx);
+ bnxt_qplib_hwq_incr_prod(&sq->dbinfo, sq_hwq, swq->slots);
+ qp->wqe_cnt++;
+done:
+ if (sch_handler) {
+ nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC);
+ if (nq_work) {
+ nq_work->cq = qp->scq;
+ nq_work->nq = qp->scq->nq;
+ INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task);
+ queue_work(qp->scq->nq->cqn_wq, &nq_work->work);
+ } else {
+ dev_err(&sq->hwq.pdev->dev,
+ "QPLIB: FP: Failed to allocate SQ nq_work!\n");
+ rc = -ENOMEM;
+ }
+ }
+ return rc;
+}
+
+void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp)
+{
+ struct bnxt_qplib_q *rq = &qp->rq;
+
+ bnxt_qplib_ring_prod_db(&rq->dbinfo, DBC_DBC_TYPE_RQ);
+}
+
+void bnxt_re_handle_cqn(struct bnxt_qplib_cq *cq)
+{
+ struct bnxt_qplib_nq *nq;
+
+ if (!(cq && cq->nq))
+ return;
+
+ nq = cq->nq;
+ spin_lock_bh(&cq->compl_lock);
+ if (nq->cqn_handler) {
+ dev_dbg(&nq->res->pdev->dev,
+ "%s:Trigger cq = %p event nq = %p\n",
+ __func__, cq, nq);
+ nq->cqn_handler(nq, cq);
+ }
+ spin_unlock_bh(&cq->compl_lock);
+}
+
+int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp,
+ struct bnxt_qplib_swqe *wqe)
+{
+ struct bnxt_qplib_nq_work *nq_work = NULL;
+ struct bnxt_qplib_q *rq = &qp->rq;
+ struct bnxt_qplib_hwq *rq_hwq;
+ struct bnxt_qplib_swq *swq;
+ bool sch_handler = false;
+ struct rq_wqe_hdr *base_hdr;
+ struct rq_ext_hdr *ext_hdr;
+ struct sq_sge *dsge;
+ u8 wqe_slots;
+ u32 wqe_idx;
+ u32 sw_prod;
+ int rc = 0;
+
+ rq_hwq = &rq->hwq;
+ if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_RESET) {
+ dev_err(&rq_hwq->pdev->dev,
+ "QPLIB: FP: QP (0x%x) is in the 0x%x state\n",
+ qp->id, qp->state);
+ rc = -EINVAL;
+ goto done;
+ }
+
+ wqe_slots = _calculate_wqe_byte(qp, wqe, NULL);
+ if (bnxt_qplib_queue_full(rq_hwq, rq->dbinfo.max_slot)) {
+ dev_err(&rq_hwq->pdev->dev,
+ "QPLIB: FP: QP (0x%x) RQ is full!\n", qp->id);
+ rc = -EINVAL;
+ goto done;
+ }
+
+ swq = bnxt_qplib_get_swqe(rq, &wqe_idx);
+ swq->wr_id = wqe->wr_id;
+ swq->slots = rq->dbinfo.max_slot;
+ dev_dbg(&rq_hwq->pdev->dev,
+ "QPLIB: FP: post RQ wr_id[%d] = 0x%llx\n",
+ wqe_idx, swq->wr_id);
+ if (qp->cur_qp_state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
+ sch_handler = true;
+ dev_dbg(&rq_hwq->pdev->dev, "%s Error QP. Sched a flushed cmpl\n",
+ __func__);
+ goto queue_err;
+ }
+
+ sw_prod = rq_hwq->prod;
+ base_hdr = bnxt_qplib_get_qe(rq_hwq, sw_prod, NULL);
+ sw_prod++;
+ ext_hdr = bnxt_qplib_get_qe(rq_hwq, (sw_prod % rq_hwq->depth), NULL);
+ sw_prod++;
+ memset(base_hdr, 0, sizeof(struct sq_sge));
+ memset(ext_hdr, 0, sizeof(struct sq_sge));
+
+ if (!wqe->num_sge) {
+ dsge = bnxt_qplib_get_qe(rq_hwq, (sw_prod % rq_hwq->depth), NULL);
+ dsge->size = 0;
+ wqe_slots++;
+ } else {
+ bnxt_qplib_put_sges(rq_hwq, wqe->sg_list, wqe->num_sge, &sw_prod);
+ }
+ base_hdr->wqe_type = wqe->type;
+ base_hdr->flags = wqe->flags;
+ base_hdr->wqe_size = wqe_slots;
+ base_hdr->wr_id |= cpu_to_le32(wqe_idx);
+queue_err:
+ bnxt_qplib_swq_mod_start(rq, wqe_idx);
+ bnxt_qplib_hwq_incr_prod(&rq->dbinfo, &rq->hwq, swq->slots);
+done:
+ if (sch_handler) {
+ nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC);
+ if (nq_work) {
+ nq_work->cq = qp->rcq;
+ nq_work->nq = qp->rcq->nq;
+ INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task);
+ queue_work(qp->rcq->nq->cqn_wq, &nq_work->work);
+ } else {
+ dev_err(&rq->hwq.pdev->dev,
+ "QPLIB: FP: Failed to allocate RQ nq_work!\n");
+ rc = -ENOMEM;
+ }
+ }
+ return rc;
+}
+
+/* CQ */
+int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
+{
+ struct bnxt_qplib_hwq_attr hwq_attr = {};
+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+ struct creq_create_cq_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct cmdq_create_cq req = {};
+ struct bnxt_qplib_reftbl *tbl;
+ unsigned long flag;
+ u32 pg_sz_lvl = 0;
+ int rc;
+
+ hwq_attr.res = res;
+ hwq_attr.depth = cq->max_wqe;
+ hwq_attr.stride = sizeof(struct cq_base);
+ hwq_attr.type = HWQ_TYPE_QUEUE;
+ hwq_attr.sginfo = &cq->sginfo;
+ rc = bnxt_qplib_alloc_init_hwq(&cq->hwq, &hwq_attr);
+ if (rc)
+ goto exit;
+
+ bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_CREATE_CQ,
+ sizeof(req));
+
+ if (!cq->dpi) {
+ dev_err(&rcfw->pdev->dev,
+ "QPLIB: FP: CREATE_CQ failed due to NULL DPI\n");
+ return -EINVAL;
+ }
+ req.dpi = cpu_to_le32(cq->dpi->dpi);
+ req.cq_handle = cpu_to_le64(cq->cq_handle);
+
+ req.cq_size = cpu_to_le32(cq->max_wqe);
+ req.pbl = cpu_to_le64(_get_base_addr(&cq->hwq));
+ pg_sz_lvl = _get_base_pg_size(&cq->hwq) << CMDQ_CREATE_CQ_PG_SIZE_SFT;
+ pg_sz_lvl |= ((cq->hwq.level & CMDQ_CREATE_CQ_LVL_MASK) <<
+ CMDQ_CREATE_CQ_LVL_SFT);
+ req.pg_size_lvl = cpu_to_le32(pg_sz_lvl);
+
+ req.cq_fco_cnq_id = cpu_to_le32(
+ (cq->cnq_hw_ring_id & CMDQ_CREATE_CQ_CNQ_ID_MASK) <<
+ CMDQ_CREATE_CQ_CNQ_ID_SFT);
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+ if (rc)
+ goto fail;
+ cq->id = le32_to_cpu(resp.xid);
+ cq->period = BNXT_QPLIB_QUEUE_START_PERIOD;
+ init_waitqueue_head(&cq->waitq);
+ INIT_LIST_HEAD(&cq->sqf_head);
+ INIT_LIST_HEAD(&cq->rqf_head);
+ spin_lock_init(&cq->flush_lock);
+ spin_lock_init(&cq->compl_lock);
+
+ /* init dbinfo */
+ cq->cctx = res->cctx;
+ cq->dbinfo.hwq = &cq->hwq;
+ cq->dbinfo.xid = cq->id;
+ cq->dbinfo.db = cq->dpi->dbr;
+ cq->dbinfo.priv_db = res->dpi_tbl.priv_db;
+ cq->dbinfo.flags = 0;
+ cq->dbinfo.toggle = 0;
+ cq->dbinfo.res = res;
+ cq->dbinfo.seed = cq->id;
+ spin_lock_init(&cq->dbinfo.lock);
+ cq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID;
+ cq->dbinfo.shadow_key_arm_ena = BNXT_QPLIB_DBR_KEY_INVALID;
+
+ tbl = &res->reftbl.cqref;
+ spin_lock_irqsave(&tbl->lock, flag);
+ tbl->rec[GET_TBL_INDEX(cq->id, tbl)].xid = cq->id;
+ tbl->rec[GET_TBL_INDEX(cq->id, tbl)].handle = cq;
+ spin_unlock_irqrestore(&tbl->lock, flag);
+
+ bnxt_qplib_armen_db(&cq->dbinfo, DBC_DBC_TYPE_CQ_ARMENA);
+ return 0;
+
+fail:
+ bnxt_qplib_free_hwq(res, &cq->hwq);
+exit:
+ return rc;
+}
+
+int bnxt_qplib_modify_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
+{
+ /* TODO: Modify CQ threshold are passed to the HW via DBR */
+ return 0;
+}
+
+void bnxt_qplib_resize_cq_complete(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_cq *cq)
+{
+ bnxt_qplib_free_hwq(res, &cq->hwq);
+ memcpy(&cq->hwq, &cq->resize_hwq, sizeof(cq->hwq));
+ /* Reset only the cons bit in the flags */
+ cq->dbinfo.flags &= ~(1UL << BNXT_QPLIB_FLAG_EPOCH_CONS_SHIFT);
+
+ /* Tell HW to switch over to the new CQ */
+ if (!cq->resize_hwq.is_user)
+ bnxt_qplib_cq_coffack_db(&cq->dbinfo);
+}
+
+int bnxt_qplib_resize_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq,
+ int new_cqes)
+{
+ struct bnxt_qplib_hwq_attr hwq_attr = {};
+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+ struct creq_resize_cq_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct cmdq_resize_cq req = {};
+ u32 pgsz = 0, lvl = 0, nsz = 0;
+ struct bnxt_qplib_pbl *pbl;
+ u16 count = -1;
+ int rc;
+
+ bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_RESIZE_CQ,
+ sizeof(req));
+
+ hwq_attr.sginfo = &cq->sginfo;
+ hwq_attr.res = res;
+ hwq_attr.depth = new_cqes;
+ hwq_attr.stride = sizeof(struct cq_base);
+ hwq_attr.type = HWQ_TYPE_QUEUE;
+ rc = bnxt_qplib_alloc_init_hwq(&cq->resize_hwq, &hwq_attr);
+ if (rc)
+ return rc;
+
+ dev_dbg(&rcfw->pdev->dev, "QPLIB: FP: %s: pbl_lvl: %d\n", __func__,
+ cq->resize_hwq.level);
+ req.cq_cid = cpu_to_le32(cq->id);
+ pbl = &cq->resize_hwq.pbl[PBL_LVL_0];
+ pgsz = ((pbl->pg_size == ROCE_PG_SIZE_4K ? CMDQ_RESIZE_CQ_PG_SIZE_PG_4K :
+ pbl->pg_size == ROCE_PG_SIZE_8K ? CMDQ_RESIZE_CQ_PG_SIZE_PG_8K :
+ pbl->pg_size == ROCE_PG_SIZE_64K ? CMDQ_RESIZE_CQ_PG_SIZE_PG_64K :
+ pbl->pg_size == ROCE_PG_SIZE_2M ? CMDQ_RESIZE_CQ_PG_SIZE_PG_2M :
+ pbl->pg_size == ROCE_PG_SIZE_8M ? CMDQ_RESIZE_CQ_PG_SIZE_PG_8M :
+ pbl->pg_size == ROCE_PG_SIZE_1G ? CMDQ_RESIZE_CQ_PG_SIZE_PG_1G :
+ CMDQ_RESIZE_CQ_PG_SIZE_PG_4K) & CMDQ_RESIZE_CQ_PG_SIZE_MASK);
+ lvl = (cq->resize_hwq.level << CMDQ_RESIZE_CQ_LVL_SFT) &
+ CMDQ_RESIZE_CQ_LVL_MASK;
+ nsz = (new_cqes << CMDQ_RESIZE_CQ_NEW_CQ_SIZE_SFT) &
+ CMDQ_RESIZE_CQ_NEW_CQ_SIZE_MASK;
+ req.new_cq_size_pg_size_lvl = cpu_to_le32(nsz|pgsz|lvl);
+ req.new_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
+
+ if (!cq->resize_hwq.is_user)
+ set_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags);
+
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+ if (rc)
+ goto fail;
+
+ if (!cq->resize_hwq.is_user) {
+wait:
+ /* Wait here for the HW to switch the CQ over */
+ if (wait_event_interruptible_timeout(cq->waitq,
+ !test_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags),
+ msecs_to_jiffies(CQ_RESIZE_WAIT_TIME_MS)) ==
+ -ERESTARTSYS && count--)
+ goto wait;
+
+ if (test_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags)) {
+ dev_err(&rcfw->pdev->dev,
+ "QPLIB: FP: RESIZE_CQ timed out\n");
+ rc = -ETIMEDOUT;
+ goto fail;
+ }
+
+ bnxt_qplib_resize_cq_complete(res, cq);
+ }
+
+ return 0;
+fail:
+ if (!cq->resize_hwq.is_user) {
+ bnxt_qplib_free_hwq(res, &cq->resize_hwq);
+ clear_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags);
+ }
+ return rc;
+}
+
+void bnxt_qplib_free_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
+{
+ bnxt_qplib_free_hwq(res, &cq->hwq);
+}
+
+static void bnxt_qplib_sync_cq(struct bnxt_qplib_cq *cq)
+{
+ struct bnxt_qplib_nq *nq = cq->nq;
+ /* Flush any pending work and synchronize irq */
+ flush_workqueue(cq->nq->cqn_wq);
+ mutex_lock(&nq->lock);
+ if (nq->requested)
+ synchronize_irq(nq->msix_vec);
+ mutex_unlock(&nq->lock);
+}
+
+int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
+{
+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+ struct creq_destroy_cq_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct cmdq_destroy_cq req = {};
+ struct bnxt_qplib_reftbl *tbl;
+ u16 total_cnq_events;
+ unsigned long flag;
+ int rc;
+
+ tbl = &res->reftbl.cqref;
+ spin_lock_irqsave(&tbl->lock, flag);
+ tbl->rec[GET_TBL_INDEX(cq->id, tbl)].handle = NULL;
+ tbl->rec[GET_TBL_INDEX(cq->id, tbl)].xid = 0;
+ spin_unlock_irqrestore(&tbl->lock, flag);
+
+ bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DESTROY_CQ,
+ sizeof(req));
+
+ req.cq_cid = cpu_to_le32(cq->id);
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+ if (rc)
+ return rc;
+
+ total_cnq_events = le16_to_cpu(resp.total_cnq_events);
+ if (total_cnq_events >= 0)
+ dev_dbg(&rcfw->pdev->dev,
+ "%s: cq_id = 0x%x cq = 0x%p resp.total_cnq_events = 0x%x\n",
+ __func__, cq->id, cq, total_cnq_events);
+ __wait_for_all_nqes(cq, total_cnq_events);
+ bnxt_qplib_sync_cq(cq);
+ bnxt_qplib_free_hwq(res, &cq->hwq);
+ return 0;
+}
+
+static int __flush_sq(struct bnxt_qplib_q *sq, struct bnxt_qplib_qp *qp,
+ struct bnxt_qplib_cqe **pcqe, int *budget)
+{
+ struct bnxt_qplib_cqe *cqe;
+ u32 start, last;
+ int rc = 0;
+
+ /* Now complete all outstanding SQEs with FLUSHED_ERR */
+ start = sq->swq_start;
+ cqe = *pcqe;
+ while (*budget) {
+ last = sq->swq_last;
+ if (start == last) {
+ break;
+ }
+ /* Skip the FENCE WQE completions */
+ if (sq->swq[last].wr_id == BNXT_QPLIB_FENCE_WRID) {
+ bnxt_re_legacy_cancel_phantom_processing(qp);
+ goto skip_compl;
+ }
+
+ memset(cqe, 0, sizeof(*cqe));
+ cqe->status = CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR;
+ cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
+ cqe->qp_handle = (u64)qp;
+ cqe->wr_id = sq->swq[last].wr_id;
+ cqe->src_qp = qp->id;
+ cqe->type = sq->swq[last].type;
+ dev_dbg(&sq->hwq.pdev->dev,
+ "QPLIB: FP: CQ Processed terminal Req \n");
+ dev_dbg(&sq->hwq.pdev->dev,
+ "QPLIB: wr_id[%d] = 0x%llx with status 0x%x\n",
+ last, cqe->wr_id, cqe->status);
+ cqe++;
+ (*budget)--;
+skip_compl:
+ bnxt_qplib_hwq_incr_cons(sq->hwq.depth,
+ &sq->hwq.cons,
+ sq->swq[last].slots,
+ &sq->dbinfo.flags);
+ sq->swq_last = sq->swq[last].next_idx;
+ }
+ *pcqe = cqe;
+ if (!*budget && sq->swq_last != start)
+ /* Out of budget */
+ rc = -EAGAIN;
+ dev_dbg(&sq->hwq.pdev->dev, "QPLIB: FP: Flush SQ rc = 0x%x\n", rc);
+
+ return rc;
+}
+
+static int __flush_rq(struct bnxt_qplib_q *rq, struct bnxt_qplib_qp *qp,
+ struct bnxt_qplib_cqe **pcqe, int *budget)
+{
+ struct bnxt_qplib_cqe *cqe;
+ u32 start, last;
+ int opcode = 0;
+ int rc = 0;
+
+ switch (qp->type) {
+ case CMDQ_CREATE_QP1_TYPE_GSI:
+ opcode = CQ_BASE_CQE_TYPE_RES_RAWETH_QP1;
+ break;
+ case CMDQ_CREATE_QP_TYPE_RC:
+ opcode = CQ_BASE_CQE_TYPE_RES_RC;
+ break;
+ case CMDQ_CREATE_QP_TYPE_UD:
+ opcode = CQ_BASE_CQE_TYPE_RES_UD;
+ break;
+ }
+
+ /* Flush the rest of the RQ */
+ start = rq->swq_start;
+ cqe = *pcqe;
+ while (*budget) {
+ last = rq->swq_last;
+ if (last == start)
+ break;
+ memset(cqe, 0, sizeof(*cqe));
+ cqe->status =
+ CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR;
+ cqe->opcode = opcode;
+ cqe->qp_handle = (u64)qp;
+ cqe->wr_id = rq->swq[last].wr_id;
+ dev_dbg(&rq->hwq.pdev->dev, "QPLIB: FP: CQ Processed Res RC \n");
+ dev_dbg(&rq->hwq.pdev->dev,
+ "QPLIB: rq[%d] = 0x%llx with status 0x%x\n",
+ last, cqe->wr_id, cqe->status);
+ cqe++;
+ (*budget)--;
+ bnxt_qplib_hwq_incr_cons(rq->hwq.depth,
+ &rq->hwq.cons,
+ rq->swq[last].slots,
+ &rq->dbinfo.flags);
+ rq->swq_last = rq->swq[last].next_idx;
+ }
+ *pcqe = cqe;
+ if (!*budget && rq->swq_last != start)
+ /* Out of budget */
+ rc = -EAGAIN;
+
+ dev_dbg(&rq->hwq.pdev->dev, "QPLIB: FP: Flush RQ rc = 0x%x\n", rc);
+ return rc;
+}
+
+void bnxt_qplib_mark_qp_error(void *qp_handle)
+{
+ struct bnxt_qplib_qp *qp = qp_handle;
+
+ if (!qp)
+ return;
+
+ /* Must block new posting of SQ and RQ */
+ qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
+ qp->state = qp->cur_qp_state;
+
+ /* Add qp to flush list of the CQ */
+ if (!qp->is_user)
+ bnxt_qplib_add_flush_qp(qp);
+}
+
+/* Note: SQE is valid from sw_sq_cons up to cqe_sq_cons (exclusive)
+ * CQE is track from sw_cq_cons to max_element but valid only if VALID=1
+ */
+static int bnxt_re_legacy_do_wa9060(struct bnxt_qplib_qp *qp,
+ struct bnxt_qplib_cq *cq,
+ u32 cq_cons, u32 swq_last,
+ u32 cqe_sq_cons)
+{
+ struct bnxt_qplib_q *sq = &qp->sq;
+ struct bnxt_qplib_swq *swq;
+ u32 peek_sw_cq_cons, peek_sq_cons_idx, peek_flags;
+ struct cq_terminal *peek_term_hwcqe;
+ struct cq_req *peek_req_hwcqe;
+ struct bnxt_qplib_qp *peek_qp;
+ struct bnxt_qplib_q *peek_sq;
+ struct cq_base *peek_hwcqe;
+ int i, rc = 0;
+
+ /* Check for the psn_search marking before completing */
+ swq = &sq->swq[swq_last];
+ if (swq->psn_search &&
+ le32_to_cpu(swq->psn_search->flags_next_psn) & 0x80000000) {
+ /* Unmark */
+ swq->psn_search->flags_next_psn = cpu_to_le32
+ (le32_to_cpu(swq->psn_search->flags_next_psn)
+ & ~0x80000000);
+ dev_dbg(&cq->hwq.pdev->dev,
+ "FP: Process Req cq_cons=0x%x qp=0x%x sq cons sw=0x%x cqe=0x%x marked!\n",
+ cq_cons, qp->id, swq_last, cqe_sq_cons);
+ sq->condition = true;
+ sq->legacy_send_phantom = true;
+
+ /* TODO: Only ARM if the previous SQE is ARMALL */
+ bnxt_qplib_ring_db(&cq->dbinfo, DBC_DBC_TYPE_CQ_ARMALL);
+
+ rc = -EAGAIN;
+ goto out;
+ }
+ if (sq->condition == true) {
+ /* Peek at the completions */
+ peek_flags = cq->dbinfo.flags;
+ peek_sw_cq_cons = cq_cons;
+ i = cq->hwq.depth;
+ while (i--) {
+ peek_hwcqe = bnxt_qplib_get_qe(&cq->hwq,
+ peek_sw_cq_cons, NULL);
+ /* If the next hwcqe is VALID */
+ if (CQE_CMP_VALID(peek_hwcqe, peek_flags)) {
+ /* If the next hwcqe is a REQ */
+ dma_rmb();
+ switch (peek_hwcqe->cqe_type_toggle &
+ CQ_BASE_CQE_TYPE_MASK) {
+ case CQ_BASE_CQE_TYPE_REQ:
+ peek_req_hwcqe = (struct cq_req *)
+ peek_hwcqe;
+ peek_qp = (struct bnxt_qplib_qp *)
+ le64_to_cpu(
+ peek_req_hwcqe->qp_handle);
+ peek_sq = &peek_qp->sq;
+ peek_sq_cons_idx =
+ ((le16_to_cpu(
+ peek_req_hwcqe->sq_cons_idx)
+ - 1) % sq->max_wqe);
+ /* If the hwcqe's sq's wr_id matches */
+ if (peek_sq == sq &&
+ sq->swq[peek_sq_cons_idx].wr_id ==
+ BNXT_QPLIB_FENCE_WRID) {
+ /* Unbreak only if the phantom
+ comes back */
+ dev_dbg(&cq->hwq.pdev->dev,
+ "FP: Process Req qp=0x%x current sq cons sw=0x%x cqe=0x%x\n",
+ qp->id, swq_last,
+ cqe_sq_cons);
+ sq->condition = false;
+ sq->single = true;
+ sq->phantom_cqe_cnt++;
+ dev_dbg(&cq->hwq.pdev->dev,
+ "qp %#x condition restored at peek cq_cons=%#x sq_cons_idx %#x, phantom_cqe_cnt: %d unmark\n",
+ peek_qp->id,
+ peek_sw_cq_cons,
+ peek_sq_cons_idx,
+ sq->phantom_cqe_cnt);
+ rc = 0;
+ goto out;
+ }
+ break;
+
+ case CQ_BASE_CQE_TYPE_TERMINAL:
+ /* In case the QP has gone into the
+ error state */
+ peek_term_hwcqe = (struct cq_terminal *)
+ peek_hwcqe;
+ peek_qp = (struct bnxt_qplib_qp *)
+ le64_to_cpu(
+ peek_term_hwcqe->qp_handle);
+ if (peek_qp == qp) {
+ sq->condition = false;
+ rc = 0;
+ goto out;
+ }
+ break;
+ default:
+ break;
+ }
+ /* Valid but not the phantom, so keep looping */
+ } else {
+ /* Not valid yet, just exit and wait */
+ rc = -EINVAL;
+ goto out;
+ }
+ bnxt_qplib_hwq_incr_cons(cq->hwq.depth,
+ &peek_sw_cq_cons,
+ 1, &peek_flags);
+ }
+ dev_err(&cq->hwq.pdev->dev,
+ "Should not have come here! cq_cons=0x%x qp=0x%x sq cons sw=0x%x hw=0x%x\n",
+ cq_cons, qp->id, swq_last, cqe_sq_cons);
+ rc = -EINVAL;
+ }
+out:
+ return rc;
+}
+
+static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq,
+ struct cq_req *hwcqe,
+ struct bnxt_qplib_cqe **pcqe, int *budget,
+ u32 cq_cons, struct bnxt_qplib_qp **lib_qp)
+{
+ struct bnxt_qplib_qp *qp;
+ struct bnxt_qplib_q *sq;
+ struct bnxt_qplib_cqe *cqe;
+ u32 cqe_sq_cons;
+ struct bnxt_qplib_swq *swq;
+ int rc = 0;
+
+ qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle);
+ dev_dbg(&cq->hwq.pdev->dev, "FP: Process Req qp=0x%p\n", qp);
+ if (!qp) {
+ dev_err(&cq->hwq.pdev->dev,
+ "QPLIB: FP: Process Req qp is NULL\n");
+ return -EINVAL;
+ }
+ sq = &qp->sq;
+
+ cqe_sq_cons = le16_to_cpu(hwcqe->sq_cons_idx) % sq->max_wqe;
+ if (qp->sq.flushed) {
+ dev_dbg(&cq->hwq.pdev->dev,
+ "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
+ goto done;
+ }
+
+ /* Require to walk the sq's swq to fabricate CQEs for all previously
+ * signaled SWQEs due to CQE aggregation from the current sq cons
+ * to the cqe_sq_cons
+ */
+ cqe = *pcqe;
+ while (*budget) {
+ if (sq->swq_last == cqe_sq_cons)
+ /* Done */
+ break;
+
+ swq = &sq->swq[sq->swq_last];
+ memset(cqe, 0, sizeof(*cqe));
+ cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
+ cqe->qp_handle = (u64)qp;
+ cqe->src_qp = qp->id;
+ cqe->wr_id = swq->wr_id;
+
+ if (cqe->wr_id == BNXT_QPLIB_FENCE_WRID)
+ goto skip;
+
+ cqe->type = swq->type;
+
+ /* For the last CQE, check for status. For errors, regardless
+ * of the request being signaled or not, it must complete with
+ * the hwcqe error status
+ */
+ if (swq->next_idx == cqe_sq_cons &&
+ hwcqe->status != CQ_REQ_STATUS_OK) {
+ cqe->status = hwcqe->status;
+ dev_err(&cq->hwq.pdev->dev,
+ "QPLIB: FP: CQ Processed Req \n");
+ dev_err(&cq->hwq.pdev->dev,
+ "QPLIB: QP 0x%x wr_id[%d] = 0x%lx vendor type 0x%x with vendor status 0x%x\n",
+ cqe->src_qp, sq->swq_last, cqe->wr_id, cqe->type, cqe->status);
+ cqe++;
+ (*budget)--;
+ bnxt_qplib_mark_qp_error(qp);
+ } else {
+ /* Before we complete, do WA 9060 */
+ if (!_is_chip_gen_p5_p7(qp->cctx)) {
+ if (bnxt_re_legacy_do_wa9060(qp, cq, cq_cons,
+ sq->swq_last,
+ cqe_sq_cons)) {
+ *lib_qp = qp;
+ goto out;
+ }
+ }
+ if (swq->flags & SQ_SEND_FLAGS_SIGNAL_COMP) {
+
+ dev_dbg(&cq->hwq.pdev->dev,
+ "QPLIB: FP: CQ Processed Req \n");
+ dev_dbg(&cq->hwq.pdev->dev,
+ "QPLIB: wr_id[%d] = 0x%llx \n",
+ sq->swq_last, cqe->wr_id);
+ dev_dbg(&cq->hwq.pdev->dev,
+ "QPLIB: with status 0x%x\n", cqe->status);
+ cqe->status = CQ_REQ_STATUS_OK;
+ cqe++;
+ (*budget)--;
+ }
+ }
+skip:
+ bnxt_qplib_hwq_incr_cons(sq->hwq.depth, &sq->hwq.cons,
+ swq->slots, &sq->dbinfo.flags);
+ sq->swq_last = swq->next_idx;
+ if (sq->single == true)
+ break;
+ }
+out:
+ *pcqe = cqe;
+ if (sq->swq_last != cqe_sq_cons) {
+ /* Out of budget */
+ rc = -EAGAIN;
+ goto done;
+ }
+ /* Back to normal completion mode only after it has completed all of
+ the WC for this CQE */
+ sq->single = false;
+done:
+ return rc;
+}
+
+static void bnxt_qplib_release_srqe(struct bnxt_qplib_srq *srq, u32 tag)
+{
+ spin_lock(&srq->hwq.lock);
+ srq->swq[srq->last_idx].next_idx = (int)tag;
+ srq->last_idx = (int)tag;
+ srq->swq[srq->last_idx].next_idx = -1;
+ bnxt_qplib_hwq_incr_cons(srq->hwq.depth, &srq->hwq.cons,
+ srq->dbinfo.max_slot, &srq->dbinfo.flags);
+ spin_unlock(&srq->hwq.lock);
+}
+
+static int bnxt_qplib_cq_process_res_rc(struct bnxt_qplib_cq *cq,
+ struct cq_res_rc *hwcqe,
+ struct bnxt_qplib_cqe **pcqe,
+ int *budget)
+{
+ struct bnxt_qplib_srq *srq;
+ struct bnxt_qplib_cqe *cqe;
+ struct bnxt_qplib_qp *qp;
+ struct bnxt_qplib_q *rq;
+ u32 wr_id_idx;
+ int rc = 0;
+
+ qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle);
+ if (!qp) {
+ dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq RC qp is NULL\n");
+ return -EINVAL;
+ }
+ if (qp->rq.flushed) {
+ dev_dbg(&cq->hwq.pdev->dev,
+ "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
+ goto done;
+ }
+
+ cqe = *pcqe;
+ cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
+ cqe->length = le32_to_cpu(hwcqe->length);
+ cqe->invrkey = le32_to_cpu(hwcqe->imm_data_or_inv_r_key);
+ cqe->mr_handle = le64_to_cpu(hwcqe->mr_handle);
+ cqe->flags = le16_to_cpu(hwcqe->flags);
+ cqe->status = hwcqe->status;
+ cqe->qp_handle = (u64)(unsigned long)qp;
+
+ wr_id_idx = le32_to_cpu(hwcqe->srq_or_rq_wr_id) &
+ CQ_RES_RC_SRQ_OR_RQ_WR_ID_MASK;
+ if (cqe->flags & CQ_RES_RC_FLAGS_SRQ_SRQ) {
+ srq = qp->srq;
+ if (!srq) {
+ dev_err(&cq->hwq.pdev->dev,
+ "QPLIB: FP: SRQ used but not defined??\n");
+ return -EINVAL;
+ }
+ if (wr_id_idx > srq->hwq.depth - 1) {
+ dev_err(&cq->hwq.pdev->dev,
+ "QPLIB: FP: CQ Process RC \n");
+ dev_err(&cq->hwq.pdev->dev,
+ "QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x\n",
+ wr_id_idx, srq->hwq.depth);
+ return -EINVAL;
+ }
+ cqe->wr_id = srq->swq[wr_id_idx].wr_id;
+ bnxt_qplib_release_srqe(srq, wr_id_idx);
+ dev_dbg(&srq->hwq.pdev->dev,
+ "QPLIB: FP: CQ Processed RC SRQ wr_id[%d] = 0x%llx\n",
+ wr_id_idx, cqe->wr_id);
+ cqe++;
+ (*budget)--;
+ *pcqe = cqe;
+ } else {
+ rq = &qp->rq;
+ if (wr_id_idx > (rq->max_wqe - 1)) {
+ dev_err(&cq->hwq.pdev->dev,
+ "QPLIB: FP: CQ Process RC \n");
+ dev_err(&cq->hwq.pdev->dev,
+ "QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x\n",
+ wr_id_idx, rq->hwq.depth);
+ return -EINVAL;
+ }
+ if (wr_id_idx != rq->swq_last)
+ return -EINVAL;
+ cqe->wr_id = rq->swq[rq->swq_last].wr_id;
+ dev_dbg(&cq->hwq.pdev->dev,
+ "QPLIB: FP: CQ Processed RC RQ wr_id[%d] = 0x%llx\n",
+ rq->swq_last, cqe->wr_id);
+ cqe++;
+ (*budget)--;
+ bnxt_qplib_hwq_incr_cons(rq->hwq.depth, &rq->hwq.cons,
+ rq->swq[rq->swq_last].slots,
+ &rq->dbinfo.flags);
+ rq->swq_last = rq->swq[rq->swq_last].next_idx;
+ *pcqe = cqe;
+
+ if (hwcqe->status != CQ_RES_RC_STATUS_OK)
+ bnxt_qplib_mark_qp_error(qp);
+ }
+done:
+ return rc;
+}
+
+static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq,
+ struct cq_res_ud_v2 *hwcqe,
+ struct bnxt_qplib_cqe **pcqe,
+ int *budget)
+{
+ struct bnxt_qplib_srq *srq;
+ struct bnxt_qplib_cqe *cqe;
+ struct bnxt_qplib_qp *qp;
+ struct bnxt_qplib_q *rq;
+ u32 wr_id_idx;
+ int rc = 0;
+ u16 *smac;
+
+ qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle);
+ if (!qp) {
+ dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq UD qp is NULL\n");
+ return -EINVAL;
+ }
+ if (qp->rq.flushed) {
+ dev_dbg(&cq->hwq.pdev->dev,
+ "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
+ goto done;
+ }
+ cqe = *pcqe;
+ cqe->opcode = hwcqe->cqe_type_toggle & CQ_RES_UD_V2_CQE_TYPE_MASK;
+ cqe->length = le32_to_cpu((hwcqe->length & CQ_RES_UD_V2_LENGTH_MASK));
+ cqe->cfa_meta = le16_to_cpu(hwcqe->cfa_metadata0);
+ /* V2 format has metadata1 */
+ cqe->cfa_meta |= (((le32_to_cpu(hwcqe->src_qp_high_srq_or_rq_wr_id) &
+ CQ_RES_UD_V2_CFA_METADATA1_MASK) >>
+ CQ_RES_UD_V2_CFA_METADATA1_SFT) <<
+ BNXT_QPLIB_META1_SHIFT);
+ cqe->invrkey = le32_to_cpu(hwcqe->imm_data);
+ cqe->flags = le16_to_cpu(hwcqe->flags);
+ cqe->status = hwcqe->status;
+ cqe->qp_handle = (u64)(unsigned long)qp;
+ smac = (u16 *)cqe->smac;
+ smac[2] = ntohs(le16_to_cpu(hwcqe->src_mac[0]));
+ smac[1] = ntohs(le16_to_cpu(hwcqe->src_mac[1]));
+ smac[0] = ntohs(le16_to_cpu(hwcqe->src_mac[2]));
+ wr_id_idx = le32_to_cpu(hwcqe->src_qp_high_srq_or_rq_wr_id)
+ & CQ_RES_UD_V2_SRQ_OR_RQ_WR_ID_MASK;
+ cqe->src_qp = le16_to_cpu(hwcqe->src_qp_low) |
+ ((le32_to_cpu(
+ hwcqe->src_qp_high_srq_or_rq_wr_id) &
+ CQ_RES_UD_V2_SRC_QP_HIGH_MASK) >> 8);
+
+ if (cqe->flags & CQ_RES_UD_V2_FLAGS_SRQ) {
+ srq = qp->srq;
+ if (!srq) {
+ dev_err(&cq->hwq.pdev->dev,
+ "QPLIB: FP: SRQ used but not defined??\n");
+ return -EINVAL;
+ }
+ if (wr_id_idx > srq->hwq.depth - 1) {
+ dev_err(&cq->hwq.pdev->dev,
+ "QPLIB: FP: CQ Process UD \n");
+ dev_err(&cq->hwq.pdev->dev,
+ "QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x\n",
+ wr_id_idx, srq->hwq.depth);
+ return -EINVAL;
+ }
+ cqe->wr_id = srq->swq[wr_id_idx].wr_id;
+ bnxt_qplib_release_srqe(srq, wr_id_idx);
+ dev_dbg(&cq->hwq.pdev->dev,
+ "QPLIB: FP: CQ Processed UD SRQ wr_id[%d] = 0x%llx\n",
+ wr_id_idx, cqe->wr_id);
+ cqe++;
+ (*budget)--;
+ *pcqe = cqe;
+ } else {
+ rq = &qp->rq;
+ if (wr_id_idx > (rq->max_wqe - 1)) {
+ dev_err(&cq->hwq.pdev->dev,
+ "QPLIB: FP: CQ Process UD \n");
+ dev_err(&cq->hwq.pdev->dev,
+ "QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x\n",
+ wr_id_idx, rq->hwq.depth);
+ return -EINVAL;
+ }
+ if (rq->swq_last != wr_id_idx)
+ return -EINVAL;
+
+ cqe->wr_id = rq->swq[rq->swq_last].wr_id;
+ dev_dbg(&cq->hwq.pdev->dev,
+ "QPLIB: FP: CQ Processed UD RQ wr_id[%d] = 0x%llx\n",
+ rq->swq_last, cqe->wr_id);
+ cqe++;
+ (*budget)--;
+ bnxt_qplib_hwq_incr_cons(rq->hwq.depth, &rq->hwq.cons,
+ rq->swq[rq->swq_last].slots,
+ &rq->dbinfo.flags);
+ rq->swq_last = rq->swq[rq->swq_last].next_idx;
+ *pcqe = cqe;
+
+ if (hwcqe->status != CQ_RES_UD_V2_STATUS_OK)
+ bnxt_qplib_mark_qp_error(qp);
+ }
+done:
+ return rc;
+}
+
+bool bnxt_qplib_is_cq_empty(struct bnxt_qplib_cq *cq)
+{
+
+ struct cq_base *hw_cqe;
+ unsigned long flags;
+ bool rc = true;
+
+ spin_lock_irqsave(&cq->hwq.lock, flags);
+ hw_cqe = bnxt_qplib_get_qe(&cq->hwq, cq->hwq.cons, NULL);
+
+ /* Check for Valid bit. If the CQE is valid, return false */
+ rc = !CQE_CMP_VALID(hw_cqe, cq->dbinfo.flags);
+ spin_unlock_irqrestore(&cq->hwq.lock, flags);
+ return rc;
+}
+
+static int bnxt_qplib_cq_process_res_raweth_qp1(struct bnxt_qplib_cq *cq,
+ struct cq_res_raweth_qp1 *hwcqe,
+ struct bnxt_qplib_cqe **pcqe,
+ int *budget)
+{
+ struct bnxt_qplib_qp *qp;
+ struct bnxt_qplib_q *rq;
+ struct bnxt_qplib_srq *srq;
+ struct bnxt_qplib_cqe *cqe;
+ u32 wr_id_idx;
+ int rc = 0;
+
+ qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle);
+ if (!qp) {
+ dev_err(&cq->hwq.pdev->dev,
+ "QPLIB: process_cq Raw/QP1 qp is NULL\n");
+ return -EINVAL;
+ }
+ if (qp->rq.flushed) {
+ dev_dbg(&cq->hwq.pdev->dev,
+ "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
+ goto done;
+ }
+ cqe = *pcqe;
+ cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
+ cqe->flags = le16_to_cpu(hwcqe->flags);
+ cqe->qp_handle = (u64)(unsigned long)qp;
+
+ wr_id_idx = le32_to_cpu(hwcqe->raweth_qp1_payload_offset_srq_or_rq_wr_id)
+ & CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_MASK;
+ cqe->src_qp = qp->id;
+ if (qp->id == 1 && !cqe->length) {
+ /* Add workaround for the length misdetection */
+ cqe->length = 296;
+ } else {
+ cqe->length = le16_to_cpu(hwcqe->length);
+ }
+ cqe->pkey_index = qp->pkey_index;
+ memcpy(cqe->smac, qp->smac, 6);
+
+ cqe->raweth_qp1_flags = le16_to_cpu(hwcqe->raweth_qp1_flags);
+ cqe->raweth_qp1_flags2 = le32_to_cpu(hwcqe->raweth_qp1_flags2);
+ cqe->raweth_qp1_metadata = le32_to_cpu(hwcqe->raweth_qp1_metadata);
+
+ dev_dbg(&cq->hwq.pdev->dev,
+ "QPLIB: raweth_qp1_flags = 0x%x raweth_qp1_flags2 = 0x%x\n",
+ cqe->raweth_qp1_flags, cqe->raweth_qp1_flags2);
+
+ if (cqe->flags & CQ_RES_RAWETH_QP1_FLAGS_SRQ_SRQ) {
+ srq = qp->srq;
+ if (!srq) {
+ dev_err(&cq->hwq.pdev->dev,
+ "QPLIB: FP: SRQ used but not defined??\n");
+ return -EINVAL;
+ }
+ if (wr_id_idx > srq->hwq.depth - 1) {
+ dev_err(&cq->hwq.pdev->dev,
+ "QPLIB: FP: CQ Process Raw/QP1 \n");
+ dev_err(&cq->hwq.pdev->dev,
+ "QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x\n",
+ wr_id_idx, srq->hwq.depth);
+ return -EINVAL;
+ }
+ cqe->wr_id = srq->swq[wr_id_idx].wr_id;
+ dev_dbg(&cq->hwq.pdev->dev,
+ "QPLIB: FP: CQ Processed Raw/QP1 SRQ \n");
+ dev_dbg(&cq->hwq.pdev->dev,
+ "QPLIB: wr_id[%d] = 0x%llx with status = 0x%x\n",
+ wr_id_idx, cqe->wr_id, hwcqe->status);
+ cqe++;
+ (*budget)--;
+ srq->hwq.cons++;
+ *pcqe = cqe;
+ } else {
+ rq = &qp->rq;
+ if (wr_id_idx > (rq->max_wqe - 1)) {
+ dev_err(&cq->hwq.pdev->dev,
+ "QPLIB: FP: CQ Process Raw/QP1 RQ wr_id \n");
+ dev_err(&cq->hwq.pdev->dev,
+ "QPLIB: ix 0x%x exceeded RQ max 0x%x\n",
+ wr_id_idx, rq->max_wqe);
+ return -EINVAL;
+ }
+ if (wr_id_idx != rq->swq_last)
+ return -EINVAL;
+ cqe->wr_id = rq->swq[rq->swq_last].wr_id;
+ dev_dbg(&cq->hwq.pdev->dev,
+ "QPLIB: FP: CQ Processed Raw/QP1 RQ \n");
+ dev_dbg(&cq->hwq.pdev->dev,
+ "QPLIB: wr_id[%d] = 0x%llx with status = 0x%x\n",
+ wr_id_idx, cqe->wr_id, hwcqe->status);
+ cqe++;
+ (*budget)--;
+ bnxt_qplib_hwq_incr_cons(rq->hwq.depth, &rq->hwq.cons,
+ rq->swq[wr_id_idx].slots,
+ &rq->dbinfo.flags);
+ rq->swq_last = rq->swq[rq->swq_last].next_idx;
+ *pcqe = cqe;
+
+ if (hwcqe->status != CQ_RES_RC_STATUS_OK)
+ bnxt_qplib_mark_qp_error(qp);
+ }
+done:
+ return rc;
+}
+
+static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq,
+ struct cq_terminal *hwcqe,
+ struct bnxt_qplib_cqe **pcqe,
+ int *budget)
+{
+ struct bnxt_qplib_q *sq, *rq;
+ struct bnxt_qplib_cqe *cqe;
+ struct bnxt_qplib_qp *qp;
+ u32 swq_last;
+ u32 cqe_cons;
+ int rc = 0;
+
+ /* Check the Status */
+ if (hwcqe->status != CQ_TERMINAL_STATUS_OK)
+ dev_warn(&cq->hwq.pdev->dev,
+ "QPLIB: FP: CQ Process Terminal Error status = 0x%x\n",
+ hwcqe->status);
+
+ qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle);
+ if (!qp)
+ return -EINVAL;
+ dev_dbg(&cq->hwq.pdev->dev,
+ "QPLIB: FP: CQ Process terminal for qp (0x%x)\n", qp->id);
+
+ /* Terminal CQE requires all posted RQEs to complete with FLUSHED_ERR
+ * from the current rq->cons to the rq->prod regardless what the
+ * rq->cons the terminal CQE indicates.
+ */
+ bnxt_qplib_mark_qp_error(qp);
+
+ sq = &qp->sq;
+ rq = &qp->rq;
+
+ cqe_cons = le16_to_cpu(hwcqe->sq_cons_idx);
+ if (cqe_cons == 0xFFFF)
+ goto do_rq;
+
+ cqe_cons %= sq->max_wqe;
+ if (qp->sq.flushed) {
+ dev_dbg(&cq->hwq.pdev->dev,
+ "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
+ goto sq_done;
+ }
+
+ /* Terminal CQE can also include aggregated successful CQEs prior.
+ So we must complete all CQEs from the current sq's cons to the
+ cq_cons with status OK */
+ cqe = *pcqe;
+ while (*budget) {
+ /*sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);*/
+ swq_last = sq->swq_last;
+ if (swq_last == cqe_cons)
+ break;
+ if (sq->swq[swq_last].flags & SQ_SEND_FLAGS_SIGNAL_COMP) {
+ memset(cqe, 0, sizeof(*cqe));
+ cqe->status = CQ_REQ_STATUS_OK;
+ cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
+ cqe->qp_handle = (u64)qp;
+ cqe->src_qp = qp->id;
+ cqe->wr_id = sq->swq[swq_last].wr_id;
+ cqe->type = sq->swq[swq_last].type;
+ dev_dbg(&cq->hwq.pdev->dev,
+ "QPLIB: FP: CQ Processed terminal Req \n");
+ dev_dbg(&cq->hwq.pdev->dev,
+ "QPLIB: wr_id[%d] = 0x%llx with status 0x%x\n",
+ swq_last, cqe->wr_id, cqe->status);
+ cqe++;
+ (*budget)--;
+ }
+ bnxt_qplib_hwq_incr_cons(sq->hwq.depth, &sq->hwq.cons,
+ sq->swq[swq_last].slots,
+ &sq->dbinfo.flags);
+ sq->swq_last = sq->swq[swq_last].next_idx;
+ }
+ *pcqe = cqe;
+ if (!*budget && swq_last != cqe_cons) {
+ /* Out of budget */
+ rc = -EAGAIN;
+ goto sq_done;
+ }
+sq_done:
+ if (rc)
+ return rc;
+do_rq:
+ cqe_cons = le16_to_cpu(hwcqe->rq_cons_idx);
+ if (cqe_cons == 0xFFFF) {
+ goto done;
+ } else if (cqe_cons > (rq->max_wqe - 1)) {
+ dev_err(&cq->hwq.pdev->dev,
+ "QPLIB: FP: CQ Processed terminal \n");
+ dev_err(&cq->hwq.pdev->dev,
+ "QPLIB: reported rq_cons_idx 0x%x exceeds max 0x%x\n",
+ cqe_cons, rq->hwq.depth);
+ goto done;
+ }
+ if (qp->rq.flushed) {
+ dev_dbg(&cq->hwq.pdev->dev,
+ "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
+ rc = 0;
+ goto rq_done;
+ }
+
+rq_done:
+done:
+ return rc;
+}
+
+static int bnxt_qplib_cq_process_cutoff(struct bnxt_qplib_cq *cq,
+ struct cq_cutoff *hwcqe)
+{
+ /* Check the Status */
+ if (hwcqe->status != CQ_CUTOFF_STATUS_OK) {
+ dev_err(&cq->hwq.pdev->dev,
+ "QPLIB: FP: CQ Process Cutoff Error status = 0x%x\n",
+ hwcqe->status);
+ return -EINVAL;
+ }
+ clear_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags);
+ wake_up_interruptible(&cq->waitq);
+
+ dev_dbg(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Processed Cutoff\n");
+ return 0;
+}
+
+int bnxt_qplib_process_flush_list(struct bnxt_qplib_cq *cq,
+ struct bnxt_qplib_cqe *cqe,
+ int num_cqes)
+{
+ struct bnxt_qplib_qp *qp = NULL;
+ u32 budget = num_cqes;
+ unsigned long flags;
+
+ spin_lock_irqsave(&cq->flush_lock, flags);
+ list_for_each_entry(qp, &cq->sqf_head, sq_flush) {
+ dev_dbg(&cq->hwq.pdev->dev,
+ "QPLIB: FP: Flushing SQ QP= %p\n",
+ qp);
+ __flush_sq(&qp->sq, qp, &cqe, &budget);
+ }
+
+ list_for_each_entry(qp, &cq->rqf_head, rq_flush) {
+ dev_dbg(&cq->hwq.pdev->dev,
+ "QPLIB: FP: Flushing RQ QP= %p\n",
+ qp);
+ __flush_rq(&qp->rq, qp, &cqe, &budget);
+ }
+ spin_unlock_irqrestore(&cq->flush_lock, flags);
+
+ return num_cqes - budget;
+}
+
+int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe,
+ int num_cqes, struct bnxt_qplib_qp **lib_qp)
+{
+ struct cq_base *hw_cqe;
+ u32 hw_polled = 0;
+ int budget, rc = 0;
+ u8 type;
+
+ budget = num_cqes;
+
+ while (budget) {
+ hw_cqe = bnxt_qplib_get_qe(&cq->hwq, cq->hwq.cons, NULL);
+
+ /* Check for Valid bit */
+ if (!CQE_CMP_VALID(hw_cqe, cq->dbinfo.flags))
+ break;
+
+ /* The valid test of the entry must be done first before
+ * reading any further.
+ */
+ dma_rmb();
+ /* From the device's respective CQE format to qplib_wc*/
+ type = hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
+ switch (type) {
+ case CQ_BASE_CQE_TYPE_REQ:
+ rc = bnxt_qplib_cq_process_req(cq,
+ (struct cq_req *)hw_cqe, &cqe, &budget,
+ cq->hwq.cons, lib_qp);
+ break;
+ case CQ_BASE_CQE_TYPE_RES_RC:
+ rc = bnxt_qplib_cq_process_res_rc(cq,
+ (struct cq_res_rc *)hw_cqe, &cqe,
+ &budget);
+ break;
+ case CQ_BASE_CQE_TYPE_RES_UD:
+ rc = bnxt_qplib_cq_process_res_ud(cq,
+ (struct cq_res_ud_v2 *)hw_cqe,
+ &cqe, &budget);
+ break;
+ case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
+ rc = bnxt_qplib_cq_process_res_raweth_qp1(cq,
+ (struct cq_res_raweth_qp1 *)
+ hw_cqe, &cqe, &budget);
+ break;
+ case CQ_BASE_CQE_TYPE_TERMINAL:
+ rc = bnxt_qplib_cq_process_terminal(cq,
+ (struct cq_terminal *)hw_cqe,
+ &cqe, &budget);
+ break;
+ case CQ_BASE_CQE_TYPE_CUT_OFF:
+ bnxt_qplib_cq_process_cutoff(cq,
+ (struct cq_cutoff *)hw_cqe);
+ /* Done processing this CQ */
+ goto exit;
+ default:
+ dev_err(&cq->hwq.pdev->dev,
+ "QPLIB: process_cq unknown type 0x%x\n",
+ hw_cqe->cqe_type_toggle &
+ CQ_BASE_CQE_TYPE_MASK);
+ rc = -EINVAL;
+ break;
+ }
+ if (rc < 0) {
+ dev_dbg(&cq->hwq.pdev->dev,
+ "QPLIB: process_cqe rc = 0x%x\n", rc);
+ if (rc == -EAGAIN)
+ break;
+ /* Error while processing the CQE, just skip to the
+ next one */
+ if (type != CQ_BASE_CQE_TYPE_TERMINAL)
+ dev_err(&cq->hwq.pdev->dev,
+ "QPLIB: process_cqe error rc = 0x%x\n",
+ rc);
+ }
+ hw_polled++;
+ bnxt_qplib_hwq_incr_cons(cq->hwq.depth, &cq->hwq.cons,
+ 1, &cq->dbinfo.flags);
+ }
+ if (hw_polled)
+ bnxt_qplib_ring_db(&cq->dbinfo, DBC_DBC_TYPE_CQ);
+exit:
+ return num_cqes - budget;
+}
+
+void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type)
+{
+ cq->dbinfo.toggle = cq->toggle;
+ if (arm_type)
+ bnxt_qplib_ring_db(&cq->dbinfo, arm_type);
+ /* Using cq->arm_state variable to track whether to issue cq handler */
+ atomic_set(&cq->arm_state, 1);
+}
+
+void bnxt_qplib_flush_cqn_wq(struct bnxt_qplib_qp *qp)
+{
+ flush_workqueue(qp->scq->nq->cqn_wq);
+ if (qp->scq != qp->rcq)
+ flush_workqueue(qp->rcq->nq->cqn_wq);
+}
diff --git a/sys/dev/bnxt/bnxt_re/qplib_fp.h b/sys/dev/bnxt/bnxt_re/qplib_fp.h
new file mode 100644
index 000000000000..527c377f0aa5
--- /dev/null
+++ b/sys/dev/bnxt/bnxt_re/qplib_fp.h
@@ -0,0 +1,638 @@
+/*
+ * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term
+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Description: Fast Path Operators (header)
+ */
+
+#ifndef __BNXT_QPLIB_FP_H__
+#define __BNXT_QPLIB_FP_H__
+
+/* Temp header structures for SQ */
+struct sq_ud_ext_hdr {
+ __le32 dst_qp;
+ __le32 avid;
+ __le64 rsvd;
+};
+
+struct sq_raw_ext_hdr {
+ __le32 cfa_meta;
+ __le32 rsvd0;
+ __le64 rsvd1;
+};
+
+struct sq_rdma_ext_hdr {
+ __le64 remote_va;
+ __le32 remote_key;
+ __le32 rsvd;
+};
+
+struct sq_atomic_ext_hdr {
+ __le64 swap_data;
+ __le64 cmp_data;
+};
+
+struct sq_fr_pmr_ext_hdr {
+ __le64 pblptr;
+ __le64 va;
+};
+
+struct sq_bind_ext_hdr {
+ __le64 va;
+ __le32 length_lo;
+ __le32 length_hi;
+};
+
+struct rq_ext_hdr {
+ __le64 rsvd1;
+ __le64 rsvd2;
+};
+
+#define BNXT_QPLIB_ETHTYPE_ROCEV1 0x8915
+
+struct bnxt_qplib_srq {
+ struct bnxt_qplib_pd *pd;
+ struct bnxt_qplib_dpi *dpi;
+ struct bnxt_qplib_chip_ctx *cctx;
+ struct bnxt_qplib_cq *cq;
+ struct bnxt_qplib_swq *swq;
+ struct bnxt_qplib_hwq hwq;
+ struct bnxt_qplib_db_info dbinfo;
+ struct bnxt_qplib_sg_info sginfo;
+ u64 srq_handle;
+ u32 id;
+ u16 wqe_size;
+ u32 max_wqe;
+ u32 max_sge;
+ u32 threshold;
+ bool arm_req;
+ int start_idx;
+ int last_idx;
+ u16 eventq_hw_ring_id;
+ bool is_user;
+ spinlock_t lock;
+};
+
+struct bnxt_qplib_sge {
+ u64 addr;
+ u32 size;
+ u32 lkey;
+};
+
+/*
+ * Buffer space for ETH(14), IP or GRH(40), UDP header(8)
+ * and ib_bth + ib_deth (20).
+ * Max required is 82 when RoCE V2 is enabled
+ */
+
+/*
+ * RoCE V1 (38 bytes needed)
+ * +------------+----------+--------+--------+-------+
+ * |Eth-hdr(14B)| GRH (40B)|bth+deth| Mad | iCRC |
+ * | | supplied | 20B |payload | 4B |
+ * | | by user |supplied| 256B | |
+ * | | mad | |by user | |
+ * | | | | | |
+ * | sge 1 | sge 2 | sge 3 | sge 4 | sge 5 |
+ * +------------+----------+--------+--------+-------+
+ */
+
+/*
+ * RoCE V2-IPv4 (46 Bytes needed)
+ * +------------+----------+--------+--------+-------+
+ * |Eth-hdr(14B)| IP-hdr |UDP-hdr | Mad | iCRC |
+ * | | supplied | 8B |payload | 4B |
+ * | | by user |bth+deth| 256B | |
+ * | | mad lower| 20B |supplied| |
+ * | | 20B out | (sge 3)|by user | |
+ * | | of 40B | | | |
+ * | | grh space| | | |
+ * | sge 1 | sge 2 | sge 3 | sge 4 | sge 5 |
+ * +------------+----------+--------+--------+-------+
+ */
+
+/*
+ * RoCE V2-IPv6 (46 Bytes needed)
+ * +------------+----------+--------+--------+-------+
+ * |Eth-hdr(14B)| IPv6 |UDP-hdr | Mad | iCRC |
+ * | | supplied | 8B |payload | 4B |
+ * | | by user |bth+deth| 256B | |
+ * | | mad lower| 20B |supplied| |
+ * | | 40 bytes | |by user | |
+ * | | grh space| | | |
+ * | | | | | |
+ * | sge 1 | sge 2 | sge 3 | sge 4 | sge 5 |
+ * +------------+----------+--------+--------+-------+
+ */
+
+#define BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE 74
+#define BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2 86
+#define BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE 46
+#define BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE 14
+#define BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2 512
+#define BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 20
+#define BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 40
+#define BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE 20
+#define BNXT_QPLIB_MAX_SQSZ 0xFFFF
+
+struct bnxt_qplib_hdrbuf {
+ dma_addr_t dma_map;
+ void *va;
+ u32 len;
+ u32 step;
+};
+
+struct bnxt_qplib_swq {
+ u64 wr_id;
+ int next_idx;
+ u8 type;
+ u8 flags;
+ u32 start_psn;
+ u32 next_psn;
+ u32 slot_idx;
+ u8 slots;
+ /* WIP: make it void * to handle legacy also */
+ struct sq_psn_search *psn_search;
+ void *inline_data;
+};
+
+struct bnxt_qplib_swqe {
+ /* General */
+#define BNXT_QPLIB_FENCE_WRID 0x46454E43 /* "FENC" */
+#define BNXT_QPLIB_QP1_DUMMY_WRID 0x44554D59 /* "DUMY" */
+ u64 wr_id;
+ u8 reqs_type;
+ u8 type;
+#define BNXT_QPLIB_SWQE_TYPE_SEND 0
+#define BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM 1
+#define BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV 2
+#define BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE 4
+#define BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM 5
+#define BNXT_QPLIB_SWQE_TYPE_RDMA_READ 6
+#define BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP 8
+#define BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD 11
+#define BNXT_QPLIB_SWQE_TYPE_LOCAL_INV 12
+#define BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR 13
+#define BNXT_QPLIB_SWQE_TYPE_REG_MR 13
+#define BNXT_QPLIB_SWQE_TYPE_BIND_MW 14
+#define BNXT_QPLIB_SWQE_TYPE_RECV 128
+#define BNXT_QPLIB_SWQE_TYPE_RECV_RDMA_IMM 129
+ u8 flags;
+#define BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP (1 << 0)
+#define BNXT_QPLIB_SWQE_FLAGS_RD_ATOMIC_FENCE (1 << 1)
+#define BNXT_QPLIB_SWQE_FLAGS_UC_FENCE (1 << 2)
+#define BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT (1 << 3)
+#define BNXT_QPLIB_SWQE_FLAGS_INLINE (1 << 4)
+ struct bnxt_qplib_sge *sg_list;
+ int num_sge;
+
+ union {
+ /* Send, with imm, inval key */
+ struct {
+ union {
+ __be32 imm_data;
+ u32 inv_key;
+ };
+ u32 q_key;
+ u32 dst_qp;
+ u16 avid;
+ } send;
+
+ /* Send Raw Ethernet and QP1 */
+ struct {
+ u16 lflags;
+ u16 cfa_action;
+ u32 cfa_meta;
+ } rawqp1;
+
+ /* RDMA write, with imm, read */
+ struct {
+ union {
+ __be32 imm_data;
+ u32 inv_key;
+ };
+ u64 remote_va;
+ u32 r_key;
+ } rdma;
+
+ /* Atomic cmp/swap, fetch/add */
+ struct {
+ u64 remote_va;
+ u32 r_key;
+ u64 swap_data;
+ u64 cmp_data;
+ } atomic;
+
+ /* Local Invalidate */
+ struct {
+ u32 inv_l_key;
+ } local_inv;
+
+ /* FR-PMR */
+ struct {
+ u8 access_cntl;
+ u8 pg_sz_log;
+ bool zero_based;
+ u32 l_key;
+ u32 length;
+ u8 pbl_pg_sz_log;
+#define BNXT_QPLIB_SWQE_PAGE_SIZE_4K 0
+#define BNXT_QPLIB_SWQE_PAGE_SIZE_8K 1
+#define BNXT_QPLIB_SWQE_PAGE_SIZE_64K 4
+#define BNXT_QPLIB_SWQE_PAGE_SIZE_256K 6
+#define BNXT_QPLIB_SWQE_PAGE_SIZE_1M 8
+#define BNXT_QPLIB_SWQE_PAGE_SIZE_2M 9
+#define BNXT_QPLIB_SWQE_PAGE_SIZE_4M 10
+#define BNXT_QPLIB_SWQE_PAGE_SIZE_1G 18
+ u8 levels;
+#define PAGE_SHIFT_4K 12
+ __le64 *pbl_ptr;
+ dma_addr_t pbl_dma_ptr;
+ u64 *page_list;
+ u16 page_list_len;
+ u64 va;
+ } frmr;
+
+ /* Bind */
+ struct {
+ u8 access_cntl;
+#define BNXT_QPLIB_BIND_SWQE_ACCESS_LOCAL_WRITE (1 << 0)
+#define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_READ (1 << 1)
+#define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_WRITE (1 << 2)
+#define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_ATOMIC (1 << 3)
+#define BNXT_QPLIB_BIND_SWQE_ACCESS_WINDOW_BIND (1 << 4)
+ bool zero_based;
+ u8 mw_type;
+ u32 parent_l_key;
+ u32 r_key;
+ u64 va;
+ u32 length;
+ } bind;
+ };
+};
+
+struct bnxt_qplib_q {
+ struct bnxt_qplib_swq *swq;
+ struct bnxt_qplib_db_info dbinfo;
+ struct bnxt_qplib_sg_info sginfo;
+ struct bnxt_qplib_hwq hwq;
+ u32 max_wqe;
+ u16 max_sge;
+ u16 wqe_size;
+ u16 q_full_delta;
+ u32 psn;
+ bool condition;
+ bool single;
+ bool legacy_send_phantom;
+ u32 phantom_wqe_cnt;
+ u32 phantom_cqe_cnt;
+ u32 next_cq_cons;
+ bool flushed;
+ u32 swq_start;
+ u32 swq_last;
+};
+
+#define BNXT_QPLIB_PPP_REQ 0x1
+#define BNXT_QPLIB_PPP_ST_IDX_SHIFT 0x1
+
+struct bnxt_qplib_ppp {
+ u32 dpi;
+ u8 req;
+ u8 st_idx_en;
+};
+
+struct bnxt_qplib_qp {
+ struct bnxt_qplib_pd *pd;
+ struct bnxt_qplib_dpi *dpi;
+ struct bnxt_qplib_chip_ctx *cctx;
+ u64 qp_handle;
+#define BNXT_QPLIB_QP_ID_INVALID 0xFFFFFFFF
+ u32 id;
+ u8 type;
+ u8 sig_type;
+ u8 wqe_mode;
+ u8 state;
+ u8 cur_qp_state;
+ u8 is_user;
+ u64 modify_flags;
+ u32 max_inline_data;
+ u32 mtu;
+ u32 path_mtu;
+ bool en_sqd_async_notify;
+ u16 pkey_index;
+ u32 qkey;
+ u32 dest_qp_id;
+ u8 access;
+ u8 timeout;
+ u8 retry_cnt;
+ u8 rnr_retry;
+ u64 wqe_cnt;
+ u32 min_rnr_timer;
+ u32 max_rd_atomic;
+ u32 max_dest_rd_atomic;
+ u32 dest_qpn;
+ u8 smac[6];
+ u16 vlan_id;
+ u8 nw_type;
+ u16 port_id;
+ struct bnxt_qplib_ah ah;
+ struct bnxt_qplib_ppp ppp;
+
+#define BTH_PSN_MASK ((1 << 24) - 1)
+ /* SQ */
+ struct bnxt_qplib_q sq;
+ /* RQ */
+ struct bnxt_qplib_q rq;
+ /* SRQ */
+ struct bnxt_qplib_srq *srq;
+ /* CQ */
+ struct bnxt_qplib_cq *scq;
+ struct bnxt_qplib_cq *rcq;
+ /* IRRQ and ORRQ */
+ struct bnxt_qplib_hwq irrq;
+ struct bnxt_qplib_hwq orrq;
+ /* Header buffer for QP1 */
+ struct bnxt_qplib_hdrbuf *sq_hdr_buf;
+ struct bnxt_qplib_hdrbuf *rq_hdr_buf;
+
+ /* ToS */
+ u8 tos_ecn;
+ u8 tos_dscp;
+ /* To track the SQ and RQ flush list */
+ struct list_head sq_flush;
+ struct list_head rq_flush;
+ /* 4 bytes of QP's scrabled mac received from FW */
+ u32 lag_src_mac;
+ u32 msn;
+ u32 msn_tbl_sz;
+ /* get devflags in PI code */
+ u16 dev_cap_flags;
+};
+
+
+#define CQE_CMP_VALID(hdr, pass) \
+ (!!((hdr)->cqe_type_toggle & CQ_BASE_TOGGLE) == \
+ !(pass & BNXT_QPLIB_FLAG_EPOCH_CONS_MASK))
+
+static inline u32 __bnxt_qplib_get_avail(struct bnxt_qplib_hwq *hwq)
+{
+ int cons, prod, avail;
+
+ /* False full is possible retrying post-send makes sense */
+ cons = hwq->cons;
+ prod = hwq->prod;
+ avail = cons - prod;
+ if (cons <= prod)
+ avail += hwq->depth;
+ return avail;
+}
+
+static inline bool bnxt_qplib_queue_full(struct bnxt_qplib_hwq *hwq, u8 slots)
+{
+ return __bnxt_qplib_get_avail(hwq) <= slots;
+}
+
+struct bnxt_qplib_cqe {
+ u8 status;
+ u8 type;
+ u8 opcode;
+ u32 length;
+ /* Lower 16 is cfa_metadata0, Upper 16 is cfa_metadata1 */
+ u32 cfa_meta;
+#define BNXT_QPLIB_META1_SHIFT 16
+#define BNXT_QPLIB_CQE_CFA_META1_VALID 0x80000UL
+ u64 wr_id;
+ union {
+ __be32 immdata;
+ u32 invrkey;
+ };
+ u64 qp_handle;
+ u64 mr_handle;
+ u16 flags;
+ u8 smac[6];
+ u32 src_qp;
+ u16 raweth_qp1_flags;
+ u16 raweth_qp1_errors;
+ u16 raweth_qp1_cfa_code;
+ u32 raweth_qp1_flags2;
+ u32 raweth_qp1_metadata;
+ u8 raweth_qp1_payload_offset;
+ u16 pkey_index;
+};
+
+#define BNXT_QPLIB_QUEUE_START_PERIOD 0x01
+struct bnxt_qplib_cq {
+ struct bnxt_qplib_dpi *dpi;
+ struct bnxt_qplib_chip_ctx *cctx;
+ struct bnxt_qplib_nq *nq;
+ struct bnxt_qplib_db_info dbinfo;
+ struct bnxt_qplib_sg_info sginfo;
+ struct bnxt_qplib_hwq hwq;
+ struct bnxt_qplib_hwq resize_hwq;
+ struct list_head sqf_head;
+ struct list_head rqf_head;
+ u32 max_wqe;
+ u32 id;
+ u16 count;
+ u16 period;
+ u32 cnq_hw_ring_id;
+ u64 cq_handle;
+ atomic_t arm_state;
+#define CQ_RESIZE_WAIT_TIME_MS 500
+ unsigned long flags;
+#define CQ_FLAGS_RESIZE_IN_PROG 1
+ wait_queue_head_t waitq;
+ spinlock_t flush_lock; /* lock flush queue list */
+ spinlock_t compl_lock; /* synch CQ handlers */
+ u16 cnq_events;
+ bool is_cq_err_event;
+ bool destroyed;
+ u8 toggle;
+};
+
+#define BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE sizeof(struct xrrq_irrq)
+#define BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE sizeof(struct xrrq_orrq)
+#define IRD_LIMIT_TO_IRRQ_SLOTS(x) (2 * x + 2)
+#define IRRQ_SLOTS_TO_IRD_LIMIT(s) ((s >> 1) - 1)
+#define ORD_LIMIT_TO_ORRQ_SLOTS(x) (x + 1)
+#define ORRQ_SLOTS_TO_ORD_LIMIT(s) (s - 1)
+
+#define NQE_CMP_VALID(hdr, pass) \
+ (!!(le32_to_cpu((hdr)->info63_v & 0xffffffff) & NQ_BASE_V) == \
+ !(pass & BNXT_QPLIB_FLAG_EPOCH_CONS_MASK))
+
+#define BNXT_QPLIB_NQE_MAX_CNT (128 * 1024)
+
+/* MSN table print macros for debugging */
+#define BNXT_RE_MSN_IDX(m) (((m) & SQ_MSN_SEARCH_START_IDX_MASK) >> \
+ SQ_MSN_SEARCH_START_IDX_SFT)
+#define BNXT_RE_MSN_NPSN(m) (((m) & SQ_MSN_SEARCH_NEXT_PSN_MASK) >> \
+ SQ_MSN_SEARCH_NEXT_PSN_SFT)
+#define BNXT_RE_MSN_SPSN(m) (((m) & SQ_MSN_SEARCH_START_PSN_MASK) >> \
+ SQ_MSN_SEARCH_START_PSN_SFT)
+#define BNXT_MSN_TBLE_SGE 6
+
+struct bnxt_qplib_nq_stats {
+ u64 num_dbqne_processed;
+ u64 num_srqne_processed;
+ u64 num_cqne_processed;
+ u64 num_tasklet_resched;
+ u64 num_nq_rearm;
+};
+
+struct bnxt_qplib_nq_db {
+ struct bnxt_qplib_reg_desc reg;
+ void __iomem *db;
+ struct bnxt_qplib_db_info dbinfo;
+};
+
+typedef int (*cqn_handler_t)(struct bnxt_qplib_nq *nq,
+ struct bnxt_qplib_cq *cq);
+typedef int (*srqn_handler_t)(struct bnxt_qplib_nq *nq,
+ struct bnxt_qplib_srq *srq, u8 event);
+
+struct bnxt_qplib_nq {
+ struct bnxt_qplib_res *res;
+ struct bnxt_qplib_hwq hwq;
+ struct bnxt_qplib_nq_db nq_db;
+
+ char *name;
+ u16 ring_id;
+ int msix_vec;
+ bool requested;
+ int budget;
+ u32 load;
+ struct mutex lock;
+
+ cqn_handler_t cqn_handler;
+ srqn_handler_t srqn_handler;
+ struct workqueue_struct *cqn_wq;
+ struct bnxt_qplib_nq_stats stats;
+};
+
+struct bnxt_qplib_nq_work {
+ struct work_struct work;
+ struct bnxt_qplib_nq *nq;
+ struct bnxt_qplib_cq *cq;
+};
+
+static inline dma_addr_t
+bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp *qp, u32 index)
+{
+ struct bnxt_qplib_hdrbuf *buf;
+
+ buf = qp->rq_hdr_buf;
+ return (buf->dma_map + index * buf->step);
+}
+
+void bnxt_qplib_nq_stop_irq(struct bnxt_qplib_nq *nq, bool kill);
+void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq);
+int bnxt_qplib_nq_start_irq(struct bnxt_qplib_nq *nq, int nq_indx,
+ int msix_vector, bool need_init);
+int bnxt_qplib_enable_nq(struct bnxt_qplib_nq *nq, int nq_idx,
+ int msix_vector, int bar_reg_offset,
+ cqn_handler_t cqn_handler,
+ srqn_handler_t srq_handler);
+int bnxt_qplib_create_srq(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_srq *srq);
+int bnxt_qplib_modify_srq(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_srq *srq);
+int bnxt_qplib_query_srq(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_srq *srq);
+int bnxt_qplib_destroy_srq(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_srq *srq);
+int bnxt_qplib_post_srq_recv(struct bnxt_qplib_srq *srq,
+ struct bnxt_qplib_swqe *wqe);
+int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp);
+int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp);
+int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp);
+int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp);
+int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp);
+void bnxt_qplib_clean_qp(struct bnxt_qplib_qp *qp);
+void bnxt_qplib_free_qp_res(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp);
+void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp,
+ struct bnxt_qplib_sge *sge);
+void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp,
+ struct bnxt_qplib_sge *sge);
+u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp);
+void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp);
+int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp,
+ struct bnxt_qplib_swqe *wqe);
+void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp);
+int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp,
+ struct bnxt_qplib_swqe *wqe);
+int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq);
+int bnxt_qplib_modify_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq);
+int bnxt_qplib_resize_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq,
+ int new_cqes);
+void bnxt_qplib_resize_cq_complete(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_cq *cq);
+int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq);
+void bnxt_qplib_free_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq);
+int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe,
+ int num, struct bnxt_qplib_qp **qp);
+bool bnxt_qplib_is_cq_empty(struct bnxt_qplib_cq *cq);
+void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type);
+void bnxt_qplib_free_nq_mem(struct bnxt_qplib_nq *nq);
+int bnxt_qplib_alloc_nq_mem(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_nq *nq);
+void bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp);
+void bnxt_qplib_del_flush_qp(struct bnxt_qplib_qp *qp);
+int bnxt_qplib_process_flush_list(struct bnxt_qplib_cq *cq,
+ struct bnxt_qplib_cqe *cqe,
+ int num_cqes);
+void bnxt_qplib_flush_cqn_wq(struct bnxt_qplib_qp *qp);
+void bnxt_qplib_free_hdr_buf(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_qp *qp);
+int bnxt_qplib_alloc_hdr_buf(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_qp *qp, u32 slen, u32 rlen);
+
+static inline bool __can_request_ppp(struct bnxt_qplib_qp *qp)
+{
+ bool can_request = false;
+
+ if (qp->cur_qp_state == CMDQ_MODIFY_QP_NEW_STATE_RESET &&
+ qp->state == CMDQ_MODIFY_QP_NEW_STATE_INIT &&
+ qp->ppp.req &&
+ !(qp->ppp.st_idx_en &
+ CREQ_MODIFY_QP_RESP_PINGPONG_PUSH_ENABLED))
+ can_request = true;
+ return can_request;
+}
+
+/* MSN table update inlin */
+static inline uint64_t bnxt_re_update_msn_tbl(uint32_t st_idx, uint32_t npsn, uint32_t start_psn)
+{
+ return cpu_to_le64((((u64)(st_idx) << SQ_MSN_SEARCH_START_IDX_SFT) &
+ SQ_MSN_SEARCH_START_IDX_MASK) |
+ (((u64)(npsn) << SQ_MSN_SEARCH_NEXT_PSN_SFT) &
+ SQ_MSN_SEARCH_NEXT_PSN_MASK) |
+ (((start_psn) << SQ_MSN_SEARCH_START_PSN_SFT) &
+ SQ_MSN_SEARCH_START_PSN_MASK));
+}
+
+void bnxt_re_schedule_dbq_event(struct bnxt_qplib_res *res);
+#endif
diff --git a/sys/dev/bnxt/bnxt_re/qplib_rcfw.c b/sys/dev/bnxt/bnxt_re/qplib_rcfw.c
new file mode 100644
index 000000000000..7e3453a1e044
--- /dev/null
+++ b/sys/dev/bnxt/bnxt_re/qplib_rcfw.c
@@ -0,0 +1,1338 @@
+/*
+ * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term
+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Description: RDMA Controller HW interface
+ */
+
+#include <linux/interrupt.h>
+#include <linux/spinlock.h>
+#include <linux/dma-mapping.h>
+#include <linux/sched.h>
+#include <linux/pci.h>
+#include <linux/delay.h>
+#include <linux/hardirq.h>
+#include <linux/device.h>
+
+#include "hsi_struct_def.h"
+#include "qplib_tlv.h"
+#include "qplib_res.h"
+#include "qplib_sp.h"
+#include "qplib_rcfw.h"
+#include "bnxt_re.h"
+
+static void bnxt_qplib_service_creq(unsigned long data);
+
+int __check_cmdq_stall(struct bnxt_qplib_rcfw *rcfw,
+ u32 *cur_prod, u32 *cur_cons)
+{
+ struct bnxt_qplib_cmdq_ctx *cmdq;
+
+ cmdq = &rcfw->cmdq;
+
+ if (*cur_prod == cmdq->hwq.prod &&
+ *cur_cons == cmdq->hwq.cons)
+ /* No activity on CMDQ or CREQ. FW down */
+ return -ETIMEDOUT;
+
+ *cur_prod = cmdq->hwq.prod;
+ *cur_cons = cmdq->hwq.cons;
+ return 0;
+}
+
+static int bnxt_qplib_map_rc(u8 opcode)
+{
+ switch (opcode) {
+ case CMDQ_BASE_OPCODE_DESTROY_QP:
+ case CMDQ_BASE_OPCODE_DESTROY_SRQ:
+ case CMDQ_BASE_OPCODE_DESTROY_CQ:
+ case CMDQ_BASE_OPCODE_DEALLOCATE_KEY:
+ case CMDQ_BASE_OPCODE_DEREGISTER_MR:
+ case CMDQ_BASE_OPCODE_DELETE_GID:
+ case CMDQ_BASE_OPCODE_DESTROY_QP1:
+ case CMDQ_BASE_OPCODE_DESTROY_AH:
+ case CMDQ_BASE_OPCODE_DEINITIALIZE_FW:
+ case CMDQ_BASE_OPCODE_MODIFY_ROCE_CC:
+ case CMDQ_BASE_OPCODE_SET_LINK_AGGR_MODE:
+ return 0;
+ default:
+ return -ETIMEDOUT;
+ }
+}
+
+/**
+ * bnxt_re_is_fw_stalled - Check firmware health
+ * @rcfw - rcfw channel instance of rdev
+ * @cookie - cookie to track the command
+ *
+ * If firmware has not responded any rcfw command within
+ * rcfw->max_timeout, consider firmware as stalled.
+ *
+ * Returns:
+ * 0 if firmware is responding
+ * -ENODEV if firmware is not responding
+ */
+static int bnxt_re_is_fw_stalled(struct bnxt_qplib_rcfw *rcfw, u16 cookie)
+{
+ struct bnxt_qplib_cmdq_ctx *cmdq;
+ struct bnxt_qplib_crsqe *crsqe;
+
+ crsqe = &rcfw->crsqe_tbl[cookie];
+ cmdq = &rcfw->cmdq;
+
+ if (time_after(jiffies, cmdq->last_seen +
+ (rcfw->max_timeout * HZ))) {
+ dev_warn_ratelimited(&rcfw->pdev->dev,
+ "%s: FW STALL Detected. cmdq[%#x]=%#x waited (%ld > %d) msec active %d\n",
+ __func__, cookie, crsqe->opcode,
+ (long)jiffies_to_msecs(jiffies - cmdq->last_seen),
+ rcfw->max_timeout * 1000,
+ crsqe->is_in_used);
+ return -ENODEV;
+ }
+
+ return 0;
+}
+/**
+ * __wait_for_resp - Don't hold the cpu context and wait for response
+ * @rcfw - rcfw channel instance of rdev
+ * @cookie - cookie to track the command
+ *
+ * Wait for command completion in sleepable context.
+ *
+ * Returns:
+ * 0 if command is completed by firmware.
+ * Non zero error code for rest of the case.
+ */
+static int __wait_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie)
+{
+ struct bnxt_qplib_cmdq_ctx *cmdq;
+ struct bnxt_qplib_crsqe *crsqe;
+ unsigned long issue_time;
+ int ret;
+
+ cmdq = &rcfw->cmdq;
+ issue_time = jiffies;
+ crsqe = &rcfw->crsqe_tbl[cookie];
+
+ do {
+ if (RCFW_NO_FW_ACCESS(rcfw))
+ return bnxt_qplib_map_rc(crsqe->opcode);
+ if (test_bit(FIRMWARE_STALL_DETECTED, &cmdq->flags))
+ return -ETIMEDOUT;
+
+ /* Non zero means command completed */
+ ret = wait_event_timeout(cmdq->waitq,
+ !crsqe->is_in_used ||
+ RCFW_NO_FW_ACCESS(rcfw),
+ msecs_to_jiffies(rcfw->max_timeout * 1000));
+
+ if (!crsqe->is_in_used)
+ return 0;
+ /*
+ * Take care if interrupt miss or other cases like DBR drop
+ */
+ bnxt_qplib_service_creq((unsigned long)rcfw);
+ dev_warn_ratelimited(&rcfw->pdev->dev,
+ "Non-Blocking QPLIB: cmdq[%#x]=%#x waited (%lu) msec bit %d\n",
+ cookie, crsqe->opcode,
+ (long)jiffies_to_msecs(jiffies - issue_time),
+ crsqe->is_in_used);
+
+ if (!crsqe->is_in_used)
+ return 0;
+
+ ret = bnxt_re_is_fw_stalled(rcfw, cookie);
+ if (ret)
+ return ret;
+
+ } while (true);
+};
+
+/**
+ * __block_for_resp - hold the cpu context and wait for response
+ * @rcfw - rcfw channel instance of rdev
+ * @cookie - cookie to track the command
+ *
+ * This function will hold the cpu (non-sleepable context) and
+ * wait for command completion. Maximum holding interval is 8 second.
+ *
+ * Returns:
+ * -ETIMEOUT if command is not completed in specific time interval.
+ * 0 if command is completed by firmware.
+ */
+static int __block_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie)
+{
+ struct bnxt_qplib_cmdq_ctx *cmdq = &rcfw->cmdq;
+ struct bnxt_qplib_crsqe *crsqe;
+ unsigned long issue_time = 0;
+
+ issue_time = jiffies;
+ crsqe = &rcfw->crsqe_tbl[cookie];
+
+ do {
+ if (RCFW_NO_FW_ACCESS(rcfw))
+ return bnxt_qplib_map_rc(crsqe->opcode);
+ if (test_bit(FIRMWARE_STALL_DETECTED, &cmdq->flags))
+ return -ETIMEDOUT;
+
+ udelay(1);
+
+ /* Below call is must since there can be a deadlock
+ * if interrupt is mapped to the same cpu
+ */
+ bnxt_qplib_service_creq((unsigned long)rcfw);
+ if (!crsqe->is_in_used)
+ return 0;
+
+ } while (time_before(jiffies, issue_time + (8 * HZ)));
+
+ dev_warn_ratelimited(&rcfw->pdev->dev,
+ "Blocking QPLIB: cmdq[%#x]=%#x taken (%lu) msec",
+ cookie, crsqe->opcode,
+ (long)jiffies_to_msecs(jiffies - issue_time));
+
+ return -ETIMEDOUT;
+};
+
+/* __send_message_no_waiter - get cookie and post the message.
+ * @rcfw - rcfw channel instance of rdev
+ * @msg - qplib message internal
+ *
+ * This function will just post and don't bother about completion.
+ * Current design of this function is -
+ * user must hold the completion queue hwq->lock.
+ * user must have used existing completion and free the resources.
+ * this function will not check queue full condition.
+ * this function will explicitly set is_waiter_alive=false.
+ * current use case is - send destroy_ah if create_ah is return
+ * after waiter of create_ah is lost. It can be extended for other
+ * use case as well.
+ *
+ * Returns: Nothing
+ *
+ */
+static void __send_message_no_waiter(struct bnxt_qplib_rcfw *rcfw,
+ struct bnxt_qplib_cmdqmsg *msg)
+{
+ struct bnxt_qplib_cmdq_ctx *cmdq = &rcfw->cmdq;
+ struct bnxt_qplib_hwq *cmdq_hwq = &cmdq->hwq;
+ struct bnxt_qplib_crsqe *crsqe;
+ struct bnxt_qplib_cmdqe *cmdqe;
+ u32 sw_prod, cmdq_prod, bsize;
+ u16 cookie;
+ u8 *preq;
+
+ cookie = cmdq->seq_num & RCFW_MAX_COOKIE_VALUE;
+ __set_cmdq_base_cookie(msg->req, msg->req_sz, cpu_to_le16(cookie));
+ crsqe = &rcfw->crsqe_tbl[cookie];
+
+ /* Set cmd_size in terms of 16B slots in req. */
+ bsize = bnxt_qplib_set_cmd_slots(msg->req);
+ /* GET_CMD_SIZE would return number of slots in either case of tlv
+ * and non-tlv commands after call to bnxt_qplib_set_cmd_slots()
+ */
+ crsqe->send_timestamp = jiffies;
+ crsqe->is_internal_cmd = true;
+ crsqe->is_waiter_alive = false;
+ crsqe->is_in_used = true;
+ crsqe->req_size = __get_cmdq_base_cmd_size(msg->req, msg->req_sz);
+
+ preq = (u8 *)msg->req;
+ do {
+ /* Locate the next cmdq slot */
+ sw_prod = HWQ_CMP(cmdq_hwq->prod, cmdq_hwq);
+ cmdqe = bnxt_qplib_get_qe(cmdq_hwq, sw_prod, NULL);
+ /* Copy a segment of the req cmd to the cmdq */
+ memset(cmdqe, 0, sizeof(*cmdqe));
+ memcpy(cmdqe, preq, min_t(u32, bsize, sizeof(*cmdqe)));
+ preq += min_t(u32, bsize, sizeof(*cmdqe));
+ bsize -= min_t(u32, bsize, sizeof(*cmdqe));
+ cmdq_hwq->prod++;
+ } while (bsize > 0);
+ cmdq->seq_num++;
+
+ cmdq_prod = cmdq_hwq->prod & 0xFFFF;
+ atomic_inc(&rcfw->timeout_send);
+ /* ring CMDQ DB */
+ wmb();
+ writel(cmdq_prod, cmdq->cmdq_mbox.prod);
+ writel(RCFW_CMDQ_TRIG_VAL, cmdq->cmdq_mbox.db);
+}
+
+static int __send_message(struct bnxt_qplib_rcfw *rcfw,
+ struct bnxt_qplib_cmdqmsg *msg)
+{
+ u32 bsize, free_slots, required_slots;
+ struct bnxt_qplib_cmdq_ctx *cmdq;
+ struct bnxt_qplib_crsqe *crsqe;
+ struct bnxt_qplib_cmdqe *cmdqe;
+ struct bnxt_qplib_hwq *cmdq_hwq;
+ u32 sw_prod, cmdq_prod;
+ struct pci_dev *pdev;
+ unsigned long flags;
+ u16 cookie;
+ u8 opcode;
+ u8 *preq;
+
+ cmdq = &rcfw->cmdq;
+ cmdq_hwq = &cmdq->hwq;
+ pdev = rcfw->pdev;
+ opcode = __get_cmdq_base_opcode(msg->req, msg->req_sz);
+
+ /* Cmdq are in 16-byte units, each request can consume 1 or more
+ cmdqe */
+ spin_lock_irqsave(&cmdq_hwq->lock, flags);
+ required_slots = bnxt_qplib_get_cmd_slots(msg->req);
+ free_slots = HWQ_FREE_SLOTS(cmdq_hwq);
+ cookie = cmdq->seq_num & RCFW_MAX_COOKIE_VALUE;
+ crsqe = &rcfw->crsqe_tbl[cookie];
+
+ if (required_slots >= free_slots) {
+ dev_warn_ratelimited(&pdev->dev,
+ "QPLIB: RCFW: CMDQ is full req/free %d/%d!\n",
+ required_slots, free_slots);
+ rcfw->cmdq_full_dbg++;
+ spin_unlock_irqrestore(&cmdq_hwq->lock, flags);
+ return -EAGAIN;
+ }
+
+ if (crsqe->is_in_used)
+ panic("QPLIB: Cookie was not requested %d\n",
+ cookie);
+
+ if (msg->block)
+ cookie |= RCFW_CMD_IS_BLOCKING;
+ __set_cmdq_base_cookie(msg->req, msg->req_sz, cpu_to_le16(cookie));
+
+ /* Set cmd_size in terms of 16B slots in req. */
+ bsize = bnxt_qplib_set_cmd_slots(msg->req);
+ /* GET_CMD_SIZE would return number of slots in either case of tlv
+ * and non-tlv commands after call to bnxt_qplib_set_cmd_slots()
+ */
+ crsqe->send_timestamp = jiffies;
+ crsqe->free_slots = free_slots;
+ crsqe->resp = (struct creq_qp_event *)msg->resp;
+ crsqe->resp->cookie = cpu_to_le16(cookie);
+ crsqe->is_internal_cmd = false;
+ crsqe->is_waiter_alive = true;
+ crsqe->is_in_used = true;
+ crsqe->opcode = opcode;
+ crsqe->requested_qp_state = msg->qp_state;
+
+ crsqe->req_size = __get_cmdq_base_cmd_size(msg->req, msg->req_sz);
+ if (__get_cmdq_base_resp_size(msg->req, msg->req_sz) && msg->sb) {
+ struct bnxt_qplib_rcfw_sbuf *sbuf = msg->sb;
+
+ __set_cmdq_base_resp_addr(msg->req, msg->req_sz,
+ cpu_to_le64(sbuf->dma_addr));
+ __set_cmdq_base_resp_size(msg->req, msg->req_sz,
+ ALIGN(sbuf->size, BNXT_QPLIB_CMDQE_UNITS) /
+ BNXT_QPLIB_CMDQE_UNITS);
+ }
+
+ preq = (u8 *)msg->req;
+ do {
+ /* Locate the next cmdq slot */
+ sw_prod = HWQ_CMP(cmdq_hwq->prod, cmdq_hwq);
+ cmdqe = bnxt_qplib_get_qe(cmdq_hwq, sw_prod, NULL);
+ /* Copy a segment of the req cmd to the cmdq */
+ memset(cmdqe, 0, sizeof(*cmdqe));
+ memcpy(cmdqe, preq, min_t(u32, bsize, sizeof(*cmdqe)));
+ preq += min_t(u32, bsize, sizeof(*cmdqe));
+ bsize -= min_t(u32, bsize, sizeof(*cmdqe));
+ cmdq_hwq->prod++;
+ } while (bsize > 0);
+ cmdq->seq_num++;
+
+ cmdq_prod = cmdq_hwq->prod & 0xFFFF;
+ if (test_bit(FIRMWARE_FIRST_FLAG, &cmdq->flags)) {
+ /* The very first doorbell write
+ * is required to set this flag
+ * which prompts the FW to reset
+ * its internal pointers
+ */
+ cmdq_prod |= BIT(FIRMWARE_FIRST_FLAG);
+ clear_bit(FIRMWARE_FIRST_FLAG, &cmdq->flags);
+ }
+ /* ring CMDQ DB */
+ wmb();
+ writel(cmdq_prod, cmdq->cmdq_mbox.prod);
+ writel(RCFW_CMDQ_TRIG_VAL, cmdq->cmdq_mbox.db);
+
+ dev_dbg(&pdev->dev, "QPLIB: RCFW sent request with 0x%x 0x%x 0x%x\n",
+ cmdq_prod, cmdq_hwq->prod, crsqe->req_size);
+ dev_dbg(&pdev->dev,
+ "QPLIB: opcode 0x%x with cookie 0x%x at cmdq/crsq 0x%p/0x%p\n",
+ opcode,
+ __get_cmdq_base_cookie(msg->req, msg->req_sz),
+ cmdqe, crsqe);
+ spin_unlock_irqrestore(&cmdq_hwq->lock, flags);
+ /* Return the CREQ response pointer */
+ return 0;
+}
+
+/**
+ * __poll_for_resp - self poll completion for rcfw command
+ * @rcfw - rcfw channel instance of rdev
+ * @cookie - cookie to track the command
+ *
+ * It works same as __wait_for_resp except this function will
+ * do self polling in sort interval since interrupt is disabled.
+ * This function can not be called from non-sleepable context.
+ *
+ * Returns:
+ * -ETIMEOUT if command is not completed in specific time interval.
+ * 0 if command is completed by firmware.
+ */
+static int __poll_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie)
+{
+ struct bnxt_qplib_cmdq_ctx *cmdq = &rcfw->cmdq;
+ struct bnxt_qplib_crsqe *crsqe;
+ unsigned long issue_time;
+ int ret;
+
+ issue_time = jiffies;
+ crsqe = &rcfw->crsqe_tbl[cookie];
+
+ do {
+ if (RCFW_NO_FW_ACCESS(rcfw))
+ return bnxt_qplib_map_rc(crsqe->opcode);
+ if (test_bit(FIRMWARE_STALL_DETECTED, &cmdq->flags))
+ return -ETIMEDOUT;
+
+ usleep_range(1000, 1001);
+
+ bnxt_qplib_service_creq((unsigned long)rcfw);
+ if (!crsqe->is_in_used)
+ return 0;
+
+ if (jiffies_to_msecs(jiffies - issue_time) >
+ (rcfw->max_timeout * 1000)) {
+ dev_warn_ratelimited(&rcfw->pdev->dev,
+ "Self Polling QPLIB: cmdq[%#x]=%#x taken (%lu) msec",
+ cookie, crsqe->opcode,
+ (long)jiffies_to_msecs(jiffies - issue_time));
+ ret = bnxt_re_is_fw_stalled(rcfw, cookie);
+ if (ret)
+ return ret;
+ }
+ } while (true);
+
+};
+
+static int __send_message_basic_sanity(struct bnxt_qplib_rcfw *rcfw,
+ struct bnxt_qplib_cmdqmsg *msg, u8 opcode)
+{
+ struct bnxt_qplib_cmdq_ctx *cmdq;
+
+ cmdq = &rcfw->cmdq;
+
+ /* Prevent posting if f/w is not in a state to process */
+ if (RCFW_NO_FW_ACCESS(rcfw))
+ return -ENXIO;
+
+ if (test_bit(FIRMWARE_STALL_DETECTED, &cmdq->flags))
+ return -ETIMEDOUT;
+
+ if (test_bit(FIRMWARE_INITIALIZED_FLAG, &cmdq->flags) &&
+ opcode == CMDQ_BASE_OPCODE_INITIALIZE_FW) {
+ dev_err(&rcfw->pdev->dev, "QPLIB: RCFW already initialized!\n");
+ return -EINVAL;
+ }
+
+ if (!test_bit(FIRMWARE_INITIALIZED_FLAG, &cmdq->flags) &&
+ (opcode != CMDQ_BASE_OPCODE_QUERY_FUNC &&
+ opcode != CMDQ_BASE_OPCODE_INITIALIZE_FW &&
+ opcode != CMDQ_BASE_OPCODE_QUERY_VERSION)) {
+ dev_err(&rcfw->pdev->dev,
+ "QPLIB: RCFW not initialized, reject opcode 0x%x\n",
+ opcode);
+ return -ENOTSUPP;
+ }
+
+ return 0;
+}
+
+/* This function will just post and do not bother about completion */
+static void __destroy_timedout_ah(struct bnxt_qplib_rcfw *rcfw,
+ struct creq_create_ah_resp *create_ah_resp)
+{
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct cmdq_destroy_ah req = {};
+
+ bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DESTROY_AH,
+ sizeof(req));
+ req.ah_cid = create_ah_resp->xid;
+ msg.req = (struct cmdq_base *)&req;
+ msg.req_sz = sizeof(req);
+ __send_message_no_waiter(rcfw, &msg);
+ dev_warn_ratelimited(&rcfw->pdev->dev,
+ "From %s: ah_cid = %d timeout_send %d\n", __func__,
+ req.ah_cid,
+ atomic_read(&rcfw->timeout_send));
+}
+
+/**
+ * __bnxt_qplib_rcfw_send_message - qplib interface to send
+ * and complete rcfw command.
+ * @rcfw - rcfw channel instance of rdev
+ * @msg - qplib message internal
+ *
+ * This function does not account shadow queue depth. It will send
+ * all the command unconditionally as long as send queue is not full.
+ *
+ * Returns:
+ * 0 if command completed by firmware.
+ * Non zero if the command is not completed by firmware.
+ */
+static int __bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw,
+ struct bnxt_qplib_cmdqmsg *msg)
+{
+ struct bnxt_qplib_crsqe *crsqe;
+ struct creq_qp_event *event;
+ unsigned long flags;
+ u16 cookie;
+ int rc = 0;
+ u8 opcode;
+
+ opcode = __get_cmdq_base_opcode(msg->req, msg->req_sz);
+
+ rc = __send_message_basic_sanity(rcfw, msg, opcode);
+ if (rc)
+ return rc == -ENXIO ? bnxt_qplib_map_rc(opcode) : rc;
+
+ rc = __send_message(rcfw, msg);
+ if (rc)
+ return rc;
+
+ cookie = le16_to_cpu(__get_cmdq_base_cookie(msg->req,
+ msg->req_sz)) & RCFW_MAX_COOKIE_VALUE;
+
+
+ if (msg->block)
+ rc = __block_for_resp(rcfw, cookie);
+ else if (atomic_read(&rcfw->rcfw_intr_enabled))
+ rc = __wait_for_resp(rcfw, cookie);
+ else
+ rc = __poll_for_resp(rcfw, cookie);
+
+ if (rc) {
+ /* First check if it is FW stall.
+ * Use hwq.lock to avoid race with actual completion.
+ */
+ spin_lock_irqsave(&rcfw->cmdq.hwq.lock, flags);
+ crsqe = &rcfw->crsqe_tbl[cookie];
+ crsqe->is_waiter_alive = false;
+ if (rc == -ENODEV)
+ set_bit(FIRMWARE_STALL_DETECTED, &rcfw->cmdq.flags);
+ spin_unlock_irqrestore(&rcfw->cmdq.hwq.lock, flags);
+
+ return -ETIMEDOUT;
+ }
+
+ event = (struct creq_qp_event *)msg->resp;
+ if (event->status) {
+ /* failed with status */
+ dev_err(&rcfw->pdev->dev, "QPLIB: cmdq[%#x]=%#x (%s) status %d\n",
+ cookie, opcode, GET_OPCODE_TYPE(opcode), event->status);
+ rc = -EFAULT;
+ /*
+ * Workaround to avoid errors in the stack during bond
+ * creation and deletion.
+ * Disable error returned for ADD_GID/DEL_GID
+ */
+ if (opcode == CMDQ_BASE_OPCODE_ADD_GID ||
+ opcode == CMDQ_BASE_OPCODE_DELETE_GID)
+ rc = 0;
+ }
+
+ dev_dbg(&pdev->dev, "QPLIB: %s:%d - op 0x%x (%s), cookie 0x%x -- Return: e->status 0x%x, rc = 0x%x\n",
+ __func__, __LINE__, opcode, GET_OPCODE_TYPE(opcode), cookie, event->status, rc);
+ return rc;
+}
+
+/**
+ * bnxt_qplib_rcfw_send_message - qplib interface to send
+ * and complete rcfw command.
+ * @rcfw - rcfw channel instance of rdev
+ * @msg - qplib message internal
+ *
+ * Driver interact with Firmware through rcfw channel/slow path in two ways.
+ * a. Blocking rcfw command send. In this path, driver cannot hold
+ * the context for longer period since it is holding cpu until
+ * command is not completed.
+ * b. Non-blocking rcfw command send. In this path, driver can hold the
+ * context for longer period. There may be many pending command waiting
+ * for completion because of non-blocking nature.
+ *
+ * Driver will use shadow queue depth. Current queue depth of 8K
+ * (due to size of rcfw message it can be actual ~4K rcfw outstanding)
+ * is not optimal for rcfw command processing in firmware.
+ * RCFW_CMD_NON_BLOCKING_SHADOW_QD is defined as 64.
+ * Restrict at max 64 Non-Blocking rcfw commands.
+ * Do not allow more than 64 non-blocking command to the Firmware.
+ * Allow all blocking commands until there is no queue full.
+ *
+ * Returns:
+ * 0 if command completed by firmware.
+ * Non zero if the command is not completed by firmware.
+ */
+int bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw,
+ struct bnxt_qplib_cmdqmsg *msg)
+{
+ int ret;
+
+ if (!msg->block) {
+ down(&rcfw->rcfw_inflight);
+ ret = __bnxt_qplib_rcfw_send_message(rcfw, msg);
+ up(&rcfw->rcfw_inflight);
+ } else {
+ ret = __bnxt_qplib_rcfw_send_message(rcfw, msg);
+ }
+
+ return ret;
+}
+
+static void bnxt_re_add_perf_stats(struct bnxt_qplib_rcfw *rcfw,
+ struct bnxt_qplib_crsqe *crsqe)
+{
+ u32 latency_msec, dest_stats_id;
+ u64 *dest_stats_ptr = NULL;
+
+ latency_msec = jiffies_to_msecs(rcfw->cmdq.last_seen -
+ crsqe->send_timestamp);
+ if (latency_msec/1000 < RCFW_MAX_LATENCY_SEC_SLAB_INDEX)
+ rcfw->rcfw_lat_slab_sec[latency_msec/1000]++;
+
+ if (!rcfw->sp_perf_stats_enabled)
+ return;
+
+ if (latency_msec < RCFW_MAX_LATENCY_MSEC_SLAB_INDEX)
+ rcfw->rcfw_lat_slab_msec[latency_msec]++;
+
+ switch (crsqe->opcode) {
+ case CMDQ_BASE_OPCODE_CREATE_QP:
+ dest_stats_id = rcfw->qp_create_stats_id++;
+ dest_stats_id = dest_stats_id % RCFW_MAX_STAT_INDEX;
+ dest_stats_ptr = &rcfw->qp_create_stats[dest_stats_id];
+ break;
+ case CMDQ_BASE_OPCODE_DESTROY_QP:
+ dest_stats_id = rcfw->qp_destroy_stats_id++;
+ dest_stats_id = dest_stats_id % RCFW_MAX_STAT_INDEX;
+ dest_stats_ptr = &rcfw->qp_destroy_stats[dest_stats_id];
+ break;
+ case CMDQ_BASE_OPCODE_REGISTER_MR:
+ dest_stats_id = rcfw->mr_create_stats_id++;
+ dest_stats_id = dest_stats_id % RCFW_MAX_STAT_INDEX;
+ dest_stats_ptr = &rcfw->mr_create_stats[dest_stats_id];
+ break;
+ case CMDQ_BASE_OPCODE_DEREGISTER_MR:
+ case CMDQ_BASE_OPCODE_DEALLOCATE_KEY:
+ dest_stats_id = rcfw->mr_destroy_stats_id++;
+ dest_stats_id = dest_stats_id % RCFW_MAX_STAT_INDEX;
+ dest_stats_ptr = &rcfw->mr_destroy_stats[dest_stats_id];
+ break;
+ case CMDQ_BASE_OPCODE_MODIFY_QP:
+ if (crsqe->requested_qp_state != IB_QPS_ERR)
+ break;
+ dest_stats_id = rcfw->qp_modify_stats_id++;
+ dest_stats_id = dest_stats_id % RCFW_MAX_STAT_INDEX;
+ dest_stats_ptr = &rcfw->qp_modify_stats[dest_stats_id];
+ break;
+ default:
+ break;
+ }
+ if (dest_stats_ptr)
+ *dest_stats_ptr = max_t(unsigned long,
+ (rcfw->cmdq.last_seen - crsqe->send_timestamp), 1);
+
+}
+
+/* Completions */
+static int bnxt_qplib_process_qp_event(struct bnxt_qplib_rcfw *rcfw,
+ struct creq_qp_event *event,
+ u32 *num_wait)
+{
+ struct bnxt_qplib_hwq *cmdq_hwq = &rcfw->cmdq.hwq;
+ struct creq_cq_error_notification *cqerr;
+ struct creq_qp_error_notification *qperr;
+ struct bnxt_qplib_crsqe *crsqe;
+ struct bnxt_qplib_reftbl *tbl;
+ struct bnxt_qplib_qp *qp;
+ struct bnxt_qplib_cq *cq;
+ u16 cookie, blocked = 0;
+ struct pci_dev *pdev;
+ bool is_waiter_alive;
+ unsigned long flags;
+ u32 wait_cmds = 0;
+ u32 xid, qp_idx;
+ u32 req_size;
+ int rc = 0;
+
+ pdev = rcfw->pdev;
+ switch (event->event) {
+ case CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION:
+ tbl = &rcfw->res->reftbl.qpref;
+ qperr = (struct creq_qp_error_notification *)event;
+ xid = le32_to_cpu(qperr->xid);
+ qp_idx = map_qp_id_to_tbl_indx(xid, tbl);
+ spin_lock(&tbl->lock);
+ qp = tbl->rec[qp_idx].handle;
+ if (!qp) {
+ spin_unlock(&tbl->lock);
+ break;
+ }
+ bnxt_qplib_mark_qp_error(qp);
+ rc = rcfw->creq.aeq_handler(rcfw, event, qp);
+ spin_unlock(&tbl->lock);
+ /*
+ * Keeping these prints as debug to avoid flooding of log
+ * messages during modify QP to error state by applications
+ */
+ dev_dbg(&pdev->dev, "QPLIB: QP Error encountered!\n");
+ dev_dbg(&pdev->dev,
+ "QPLIB: qpid 0x%x, req_err=0x%x, resp_err=0x%x\n",
+ xid, qperr->req_err_state_reason,
+ qperr->res_err_state_reason);
+ break;
+ case CREQ_QP_EVENT_EVENT_CQ_ERROR_NOTIFICATION:
+ tbl = &rcfw->res->reftbl.cqref;
+ cqerr = (struct creq_cq_error_notification *)event;
+ xid = le32_to_cpu(cqerr->xid);
+ spin_lock(&tbl->lock);
+ cq = tbl->rec[GET_TBL_INDEX(xid, tbl)].handle;
+ if (!cq) {
+ spin_unlock(&tbl->lock);
+ break;
+ }
+ rc = rcfw->creq.aeq_handler(rcfw, event, cq);
+ spin_unlock(&tbl->lock);
+ dev_dbg(&pdev->dev, "QPLIB: CQ error encountered!\n");
+ break;
+ default:
+ /*
+ * Command Response
+ * cmdq hwq lock needs to be acquired to synchronize
+ * the command send and completion reaping. This function
+ * is always called with creq hwq lock held. So there is no
+ * chance of deadlock here as the locking is in correct sequence.
+ * Using the nested variant of spin_lock to annotate
+ */
+ spin_lock_irqsave_nested(&cmdq_hwq->lock, flags,
+ SINGLE_DEPTH_NESTING);
+ cookie = le16_to_cpu(event->cookie);
+ blocked = cookie & RCFW_CMD_IS_BLOCKING;
+ cookie &= RCFW_MAX_COOKIE_VALUE;
+
+ crsqe = &rcfw->crsqe_tbl[cookie];
+
+ bnxt_re_add_perf_stats(rcfw, crsqe);
+
+ if (WARN_ONCE(test_bit(FIRMWARE_STALL_DETECTED,
+ &rcfw->cmdq.flags),
+ "QPLIB: Unreponsive rcfw channel detected.!!")) {
+ dev_info(&pdev->dev, "rcfw timedout: cookie = %#x,"
+ " latency_msec = %ld free_slots = %d\n", cookie,
+ (long)jiffies_to_msecs(rcfw->cmdq.last_seen -
+ crsqe->send_timestamp),
+ crsqe->free_slots);
+ spin_unlock_irqrestore(&cmdq_hwq->lock, flags);
+ return rc;
+ }
+
+ if (crsqe->is_internal_cmd && !event->status)
+ atomic_dec(&rcfw->timeout_send);
+
+ if (crsqe->is_waiter_alive) {
+ if (crsqe->resp)
+ memcpy(crsqe->resp, event, sizeof(*event));
+ if (!blocked)
+ wait_cmds++;
+ }
+
+ req_size = crsqe->req_size;
+ is_waiter_alive = crsqe->is_waiter_alive;
+
+ crsqe->req_size = 0;
+ if (!crsqe->is_waiter_alive)
+ crsqe->resp = NULL;
+ crsqe->is_in_used = false;
+ /* Consumer is updated so that __send_message_no_waiter
+ * can never see queue full.
+ * It is safe since we are still holding cmdq_hwq->lock.
+ */
+ cmdq_hwq->cons += req_size;
+
+ /* This is a case to handle below scenario -
+ * Create AH is completed successfully by firmware,
+ * but completion took more time and driver already lost
+ * the context of create_ah from caller.
+ * We have already return failure for create_ah verbs,
+ * so let's destroy the same address vector since it is
+ * no more used in stack. We don't care about completion
+ * in __send_message_no_waiter.
+ * If destroy_ah is failued by firmware, there will be AH
+ * resource leak and relatively not critical + unlikely
+ * scenario. Current design is not to handle such case.
+ */
+ if (!is_waiter_alive && !event->status &&
+ event->event == CREQ_QP_EVENT_EVENT_CREATE_AH)
+ __destroy_timedout_ah(rcfw,
+ (struct creq_create_ah_resp *)
+ event);
+
+ spin_unlock_irqrestore(&cmdq_hwq->lock, flags);
+ }
+ *num_wait += wait_cmds;
+ return rc;
+}
+
+/* SP - CREQ Completion handlers */
+static void bnxt_qplib_service_creq(unsigned long data)
+{
+ struct bnxt_qplib_rcfw *rcfw = (struct bnxt_qplib_rcfw *)data;
+ struct bnxt_qplib_creq_ctx *creq = &rcfw->creq;
+ struct bnxt_qplib_res *res;
+ u32 type, budget = CREQ_ENTRY_POLL_BUDGET;
+ struct bnxt_qplib_hwq *creq_hwq = &creq->hwq;
+ struct creq_base *creqe;
+ struct pci_dev *pdev;
+ unsigned long flags;
+ u32 num_wakeup = 0;
+ int rc;
+
+ pdev = rcfw->pdev;
+ res = rcfw->res;
+ /* Service the CREQ until empty */
+ spin_lock_irqsave(&creq_hwq->lock, flags);
+ while (budget > 0) {
+ if (RCFW_NO_FW_ACCESS(rcfw)) {
+ spin_unlock_irqrestore(&creq_hwq->lock, flags);
+ return;
+ }
+ creqe = bnxt_qplib_get_qe(creq_hwq, creq_hwq->cons, NULL);
+ if (!CREQ_CMP_VALID(creqe, creq->creq_db.dbinfo.flags))
+ break;
+ /* The valid test of the entry must be done first before
+ * reading any further.
+ */
+ dma_rmb();
+ type = creqe->type & CREQ_BASE_TYPE_MASK;
+ rcfw->cmdq.last_seen = jiffies;
+
+ switch (type) {
+ case CREQ_BASE_TYPE_QP_EVENT:
+ bnxt_qplib_process_qp_event
+ (rcfw,(struct creq_qp_event *)creqe,
+ &num_wakeup);
+ creq->stats.creq_qp_event_processed++;
+ break;
+ case CREQ_BASE_TYPE_FUNC_EVENT:
+ rc = rcfw->creq.aeq_handler(rcfw, creqe, NULL);
+ if (rc)
+ dev_warn(&pdev->dev,
+ "QPLIB: async event type = 0x%x not handled",
+ type);
+ creq->stats.creq_func_event_processed++;
+ break;
+ default:
+ if (type != HWRM_ASYNC_EVENT_CMPL_TYPE_HWRM_ASYNC_EVENT) {
+ dev_warn(&pdev->dev,
+ "QPLIB: op_event = 0x%x not handled\n",
+ type);
+ }
+ break;
+ }
+ budget--;
+ bnxt_qplib_hwq_incr_cons(creq_hwq->max_elements, &creq_hwq->cons,
+ 1, &creq->creq_db.dbinfo.flags);
+ }
+ if (budget == CREQ_ENTRY_POLL_BUDGET &&
+ !CREQ_CMP_VALID(creqe, creq->creq_db.dbinfo.flags)) {
+ /* No completions received during this poll. Enable interrupt now */
+ bnxt_qplib_ring_nq_db(&creq->creq_db.dbinfo, res->cctx, true);
+ creq->stats.creq_arm_count++;
+ dev_dbg(&pdev->dev, "QPLIB: Num of Func (0x%llx) \n",
+ creq->stats.creq_func_event_processed);
+ dev_dbg(&pdev->dev, "QPLIB: QP (0x%llx) events processed\n",
+ creq->stats.creq_qp_event_processed);
+ dev_dbg(&pdev->dev, "QPLIB: Armed:%#llx resched:%#llx \n",
+ creq->stats.creq_arm_count,
+ creq->stats.creq_tasklet_schedule_count);
+ } else if (creq->requested) {
+ /*
+ * Currently there is no bottom half implementation to process
+ * completions, all completions are processed in interrupt context
+ * only. So enable interrupts.
+ */
+ bnxt_qplib_ring_nq_db(&creq->creq_db.dbinfo, res->cctx, true);
+ creq->stats.creq_tasklet_schedule_count++;
+ }
+ spin_unlock_irqrestore(&creq_hwq->lock, flags);
+ if (num_wakeup)
+ wake_up_all(&rcfw->cmdq.waitq);
+}
+
+static irqreturn_t bnxt_qplib_creq_irq(int irq, void *dev_instance)
+{
+ struct bnxt_qplib_rcfw *rcfw = dev_instance;
+
+ bnxt_qplib_service_creq((unsigned long)rcfw);
+ return IRQ_HANDLED;
+}
+
+/* RCFW */
+int bnxt_qplib_deinit_rcfw(struct bnxt_qplib_rcfw *rcfw)
+{
+ struct creq_deinitialize_fw_resp resp = {};
+ struct cmdq_deinitialize_fw req = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ int rc;
+
+ bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DEINITIALIZE_FW,
+ sizeof(req));
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL,
+ sizeof(req), sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+ if (rc)
+ return rc;
+ clear_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->cmdq.flags);
+ return 0;
+}
+
+int bnxt_qplib_init_rcfw(struct bnxt_qplib_rcfw *rcfw, int is_virtfn)
+{
+ struct creq_initialize_fw_resp resp = {};
+ struct cmdq_initialize_fw req = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct bnxt_qplib_chip_ctx *cctx;
+ struct bnxt_qplib_ctx *hctx;
+ struct bnxt_qplib_res *res;
+ struct bnxt_qplib_hwq *hwq;
+ int rc;
+
+ res = rcfw->res;
+ cctx = res->cctx;
+ hctx = res->hctx;
+
+ bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_INITIALIZE_FW,
+ sizeof(req));
+ /* Supply (log-base-2-of-host-page-size - base-page-shift)
+ * to bono to adjust the doorbell page sizes.
+ */
+ req.log2_dbr_pg_size = cpu_to_le16(PAGE_SHIFT -
+ RCFW_DBR_BASE_PAGE_SHIFT);
+ /*
+ * VFs need not setup the HW context area, PF
+ * shall setup this area for VF. Skipping the
+ * HW programming
+ */
+ if (is_virtfn || _is_chip_gen_p5_p7(cctx))
+ goto skip_ctx_setup;
+
+ hwq = &hctx->qp_ctx.hwq;
+ req.qpc_page_dir = cpu_to_le64(_get_base_addr(hwq));
+ req.number_of_qp = cpu_to_le32(hwq->max_elements);
+ req.qpc_pg_size_qpc_lvl = (_get_pte_pg_size(hwq) <<
+ CMDQ_INITIALIZE_FW_QPC_PG_SIZE_SFT) |
+ (u8)hwq->level;
+
+ hwq = &hctx->mrw_ctx.hwq;
+ req.mrw_page_dir = cpu_to_le64(_get_base_addr(hwq));
+ req.number_of_mrw = cpu_to_le32(hwq->max_elements);
+ req.mrw_pg_size_mrw_lvl = (_get_pte_pg_size(hwq) <<
+ CMDQ_INITIALIZE_FW_MRW_PG_SIZE_SFT) |
+ (u8)hwq->level;
+
+ hwq = &hctx->srq_ctx.hwq;
+ req.srq_page_dir = cpu_to_le64(_get_base_addr(hwq));
+ req.number_of_srq = cpu_to_le32(hwq->max_elements);
+ req.srq_pg_size_srq_lvl = (_get_pte_pg_size(hwq) <<
+ CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_SFT) |
+ (u8)hwq->level;
+
+ hwq = &hctx->cq_ctx.hwq;
+ req.cq_page_dir = cpu_to_le64(_get_base_addr(hwq));
+ req.number_of_cq = cpu_to_le32(hwq->max_elements);
+ req.cq_pg_size_cq_lvl = (_get_pte_pg_size(hwq) <<
+ CMDQ_INITIALIZE_FW_CQ_PG_SIZE_SFT) |
+ (u8)hwq->level;
+
+ hwq = &hctx->tim_ctx.hwq;
+ req.tim_page_dir = cpu_to_le64(_get_base_addr(hwq));
+ req.tim_pg_size_tim_lvl = (_get_pte_pg_size(hwq) <<
+ CMDQ_INITIALIZE_FW_TIM_PG_SIZE_SFT) |
+ (u8)hwq->level;
+ hwq = &hctx->tqm_ctx.pde;
+ req.tqm_page_dir = cpu_to_le64(_get_base_addr(hwq));
+ req.tqm_pg_size_tqm_lvl = (_get_pte_pg_size(hwq) <<
+ CMDQ_INITIALIZE_FW_TQM_PG_SIZE_SFT) |
+ (u8)hwq->level;
+skip_ctx_setup:
+ if (BNXT_RE_HW_RETX(res->dattr->dev_cap_flags))
+ req.flags |= CMDQ_INITIALIZE_FW_FLAGS_HW_REQUESTER_RETX_SUPPORTED;
+ req.stat_ctx_id = cpu_to_le32(hctx->stats.fw_id);
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL,
+ sizeof(req), sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+ if (rc)
+ return rc;
+ set_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->cmdq.flags);
+
+ return 0;
+}
+
+void bnxt_qplib_free_rcfw_channel(struct bnxt_qplib_res *res)
+{
+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+
+ vfree(rcfw->rcfw_lat_slab_msec);
+ rcfw->rcfw_lat_slab_msec = NULL;
+ vfree(rcfw->qp_create_stats);
+ rcfw->qp_create_stats = NULL;
+ vfree(rcfw->qp_destroy_stats);
+ rcfw->qp_destroy_stats = NULL;
+ vfree(rcfw->mr_create_stats);
+ rcfw->mr_create_stats = NULL;
+ vfree(rcfw->mr_destroy_stats);
+ rcfw->mr_destroy_stats = NULL;
+ vfree(rcfw->qp_modify_stats);
+ rcfw->qp_modify_stats = NULL;
+ rcfw->sp_perf_stats_enabled = false;
+
+ kfree(rcfw->crsqe_tbl);
+ rcfw->crsqe_tbl = NULL;
+
+ bnxt_qplib_free_hwq(res, &rcfw->cmdq.hwq);
+ bnxt_qplib_free_hwq(res, &rcfw->creq.hwq);
+ rcfw->pdev = NULL;
+}
+
+int bnxt_qplib_alloc_rcfw_channel(struct bnxt_qplib_res *res)
+{
+ struct bnxt_qplib_hwq_attr hwq_attr = {};
+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+ struct bnxt_qplib_sg_info sginfo = {};
+ struct bnxt_qplib_cmdq_ctx *cmdq;
+ struct bnxt_qplib_creq_ctx *creq;
+
+ rcfw->pdev = res->pdev;
+ rcfw->res = res;
+ cmdq = &rcfw->cmdq;
+ creq = &rcfw->creq;
+
+ sginfo.pgsize = PAGE_SIZE;
+ sginfo.pgshft = PAGE_SHIFT;
+
+ hwq_attr.sginfo = &sginfo;
+ hwq_attr.res = rcfw->res;
+ hwq_attr.depth = BNXT_QPLIB_CREQE_MAX_CNT;
+ hwq_attr.stride = BNXT_QPLIB_CREQE_UNITS;
+ hwq_attr.type = _get_hwq_type(res);
+
+ if (bnxt_qplib_alloc_init_hwq(&creq->hwq, &hwq_attr)) {
+ dev_err(&rcfw->pdev->dev,
+ "QPLIB: HW channel CREQ allocation failed\n");
+ return -ENOMEM;
+ }
+
+ sginfo.pgsize = BNXT_QPLIB_CMDQE_PAGE_SIZE;
+ hwq_attr.depth = BNXT_QPLIB_CMDQE_MAX_CNT & 0x7FFFFFFF;
+ hwq_attr.stride = BNXT_QPLIB_CMDQE_UNITS;
+ hwq_attr.type = HWQ_TYPE_CTX;
+ if (bnxt_qplib_alloc_init_hwq(&cmdq->hwq, &hwq_attr)) {
+ dev_err(&rcfw->pdev->dev,
+ "QPLIB: HW channel CMDQ allocation failed\n");
+ goto fail_free_creq_hwq;
+ }
+
+ rcfw->crsqe_tbl = kcalloc(cmdq->hwq.max_elements,
+ sizeof(*rcfw->crsqe_tbl), GFP_KERNEL);
+ if (!rcfw->crsqe_tbl) {
+ dev_err(&rcfw->pdev->dev,
+ "QPLIB: HW channel CRSQ allocation failed\n");
+ goto fail_free_cmdq_hwq;
+ }
+
+ rcfw->max_timeout = res->cctx->hwrm_cmd_max_timeout;
+
+ rcfw->sp_perf_stats_enabled = false;
+ rcfw->rcfw_lat_slab_msec = vzalloc(sizeof(u32) *
+ RCFW_MAX_LATENCY_MSEC_SLAB_INDEX);
+ rcfw->qp_create_stats = vzalloc(sizeof(u64) * RCFW_MAX_STAT_INDEX);
+ rcfw->qp_destroy_stats = vzalloc(sizeof(u64) * RCFW_MAX_STAT_INDEX);
+ rcfw->mr_create_stats = vzalloc(sizeof(u64) * RCFW_MAX_STAT_INDEX);
+ rcfw->mr_destroy_stats = vzalloc(sizeof(u64) * RCFW_MAX_STAT_INDEX);
+ rcfw->qp_modify_stats = vzalloc(sizeof(u64) * RCFW_MAX_STAT_INDEX);
+
+ if (rcfw->rcfw_lat_slab_msec &&
+ rcfw->qp_create_stats &&
+ rcfw->qp_destroy_stats &&
+ rcfw->mr_create_stats &&
+ rcfw->mr_destroy_stats &&
+ rcfw->qp_modify_stats)
+ rcfw->sp_perf_stats_enabled = true;
+
+ return 0;
+fail_free_cmdq_hwq:
+ bnxt_qplib_free_hwq(res, &rcfw->cmdq.hwq);
+fail_free_creq_hwq:
+ bnxt_qplib_free_hwq(res, &rcfw->creq.hwq);
+ return -ENOMEM;
+}
+
+void bnxt_qplib_rcfw_stop_irq(struct bnxt_qplib_rcfw *rcfw, bool kill)
+{
+ struct bnxt_qplib_creq_ctx *creq;
+ struct bnxt_qplib_res *res;
+
+ creq = &rcfw->creq;
+ res = rcfw->res;
+
+ if (!creq->requested)
+ return;
+
+ creq->requested = false;
+ /* Mask h/w interrupts */
+ bnxt_qplib_ring_nq_db(&creq->creq_db.dbinfo, res->cctx, false);
+ /* Sync with last running IRQ-handler */
+ synchronize_irq(creq->msix_vec);
+ free_irq(creq->msix_vec, rcfw);
+ kfree(creq->irq_name);
+ creq->irq_name = NULL;
+ /* rcfw_intr_enabled should not be greater than 1. Debug
+ * print to check if that is the case
+ */
+ if (atomic_read(&rcfw->rcfw_intr_enabled) > 1) {
+ dev_err(&rcfw->pdev->dev,
+ "%s: rcfw->rcfw_intr_enabled = 0x%x\n", __func__,
+ atomic_read(&rcfw->rcfw_intr_enabled));
+ }
+ atomic_set(&rcfw->rcfw_intr_enabled, 0);
+ rcfw->num_irq_stopped++;
+}
+
+void bnxt_qplib_disable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw)
+{
+ struct bnxt_qplib_creq_ctx *creq;
+ struct bnxt_qplib_cmdq_ctx *cmdq;
+
+ creq = &rcfw->creq;
+ cmdq = &rcfw->cmdq;
+ /* Make sure the HW channel is stopped! */
+ bnxt_qplib_rcfw_stop_irq(rcfw, true);
+
+ creq->creq_db.reg.bar_reg = NULL;
+ creq->creq_db.db = NULL;
+
+ if (cmdq->cmdq_mbox.reg.bar_reg) {
+ iounmap(cmdq->cmdq_mbox.reg.bar_reg);
+ cmdq->cmdq_mbox.reg.bar_reg = NULL;
+ cmdq->cmdq_mbox.prod = NULL;
+ cmdq->cmdq_mbox.db = NULL;
+ }
+
+ creq->aeq_handler = NULL;
+ creq->msix_vec = 0;
+}
+
+int bnxt_qplib_rcfw_start_irq(struct bnxt_qplib_rcfw *rcfw, int msix_vector,
+ bool need_init)
+{
+ struct bnxt_qplib_creq_ctx *creq;
+ struct bnxt_qplib_res *res;
+ int rc;
+
+ creq = &rcfw->creq;
+ res = rcfw->res;
+
+ if (creq->requested)
+ return -EFAULT;
+
+ creq->msix_vec = msix_vector;
+
+ creq->irq_name = kasprintf(GFP_KERNEL, "bnxt_re-creq@pci:%s\n",
+ pci_name(res->pdev));
+ if (!creq->irq_name)
+ return -ENOMEM;
+
+ rc = request_irq(creq->msix_vec, bnxt_qplib_creq_irq, 0,
+ creq->irq_name, rcfw);
+ if (rc) {
+ kfree(creq->irq_name);
+ creq->irq_name = NULL;
+ return rc;
+ }
+ creq->requested = true;
+
+ bnxt_qplib_ring_nq_db(&creq->creq_db.dbinfo, res->cctx, true);
+
+ rcfw->num_irq_started++;
+ /* Debug print to check rcfw interrupt enable/disable is invoked
+ * out of sequence
+ */
+ if (atomic_read(&rcfw->rcfw_intr_enabled) > 0) {
+ dev_err(&rcfw->pdev->dev,
+ "%s: rcfw->rcfw_intr_enabled = 0x%x\n", __func__,
+ atomic_read(&rcfw->rcfw_intr_enabled));
+ }
+ atomic_inc(&rcfw->rcfw_intr_enabled);
+ return 0;
+}
+
+static int bnxt_qplib_map_cmdq_mbox(struct bnxt_qplib_rcfw *rcfw)
+{
+ struct bnxt_qplib_cmdq_mbox *mbox;
+ resource_size_t bar_reg;
+ struct pci_dev *pdev;
+
+ pdev = rcfw->pdev;
+ mbox = &rcfw->cmdq.cmdq_mbox;
+
+ mbox->reg.bar_id = RCFW_COMM_PCI_BAR_REGION;
+ mbox->reg.len = RCFW_COMM_SIZE;
+ mbox->reg.bar_base = pci_resource_start(pdev, mbox->reg.bar_id);
+ if (!mbox->reg.bar_base) {
+ dev_err(&pdev->dev,
+ "QPLIB: CMDQ BAR region %d resc start is 0!\n",
+ mbox->reg.bar_id);
+ return -ENOMEM;
+ }
+
+ bar_reg = mbox->reg.bar_base + RCFW_COMM_BASE_OFFSET;
+ mbox->reg.len = RCFW_COMM_SIZE;
+ mbox->reg.bar_reg = ioremap(bar_reg, mbox->reg.len);
+ if (!mbox->reg.bar_reg) {
+ dev_err(&pdev->dev,
+ "QPLIB: CMDQ BAR region %d mapping failed\n",
+ mbox->reg.bar_id);
+ return -ENOMEM;
+ }
+
+ mbox->prod = (void __iomem *)((char *)mbox->reg.bar_reg +
+ RCFW_PF_VF_COMM_PROD_OFFSET);
+ mbox->db = (void __iomem *)((char *)mbox->reg.bar_reg +
+ RCFW_COMM_TRIG_OFFSET);
+ return 0;
+}
+
+static int bnxt_qplib_map_creq_db(struct bnxt_qplib_rcfw *rcfw, u32 reg_offt)
+{
+ struct bnxt_qplib_creq_db *creq_db;
+ struct bnxt_qplib_reg_desc *dbreg;
+ struct bnxt_qplib_res *res;
+
+ res = rcfw->res;
+ creq_db = &rcfw->creq.creq_db;
+ dbreg = &res->dpi_tbl.ucreg;
+
+ creq_db->reg.bar_id = dbreg->bar_id;
+ creq_db->reg.bar_base = dbreg->bar_base;
+ creq_db->reg.bar_reg = dbreg->bar_reg + reg_offt;
+ creq_db->reg.len = _is_chip_gen_p5_p7(res->cctx) ? sizeof(u64) :
+ sizeof(u32);
+
+ creq_db->dbinfo.db = creq_db->reg.bar_reg;
+ creq_db->dbinfo.hwq = &rcfw->creq.hwq;
+ creq_db->dbinfo.xid = rcfw->creq.ring_id;
+ creq_db->dbinfo.seed = rcfw->creq.ring_id;
+ creq_db->dbinfo.flags = 0;
+ spin_lock_init(&creq_db->dbinfo.lock);
+ creq_db->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID;
+ creq_db->dbinfo.res = rcfw->res;
+
+ return 0;
+}
+
+static void bnxt_qplib_start_rcfw(struct bnxt_qplib_rcfw *rcfw)
+{
+ struct bnxt_qplib_cmdq_ctx *cmdq;
+ struct bnxt_qplib_creq_ctx *creq;
+ struct bnxt_qplib_cmdq_mbox *mbox;
+ struct cmdq_init init = {0};
+
+ cmdq = &rcfw->cmdq;
+ creq = &rcfw->creq;
+ mbox = &cmdq->cmdq_mbox;
+
+ init.cmdq_pbl = cpu_to_le64(cmdq->hwq.pbl[PBL_LVL_0].pg_map_arr[0]);
+ init.cmdq_size_cmdq_lvl = cpu_to_le16(
+ ((BNXT_QPLIB_CMDQE_MAX_CNT << CMDQ_INIT_CMDQ_SIZE_SFT) &
+ CMDQ_INIT_CMDQ_SIZE_MASK) |
+ ((cmdq->hwq.level << CMDQ_INIT_CMDQ_LVL_SFT) &
+ CMDQ_INIT_CMDQ_LVL_MASK));
+ init.creq_ring_id = cpu_to_le16(creq->ring_id);
+ /* Write to the Bono mailbox register */
+ __iowrite32_copy(mbox->reg.bar_reg, &init, sizeof(init) / 4);
+}
+
+int bnxt_qplib_enable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw,
+ int msix_vector,
+ int cp_bar_reg_off,
+ aeq_handler_t aeq_handler)
+{
+ struct bnxt_qplib_cmdq_ctx *cmdq;
+ struct bnxt_qplib_creq_ctx *creq;
+ int rc;
+
+ cmdq = &rcfw->cmdq;
+ creq = &rcfw->creq;
+
+ /* Clear to defaults */
+ cmdq->seq_num = 0;
+ set_bit(FIRMWARE_FIRST_FLAG, &cmdq->flags);
+ init_waitqueue_head(&cmdq->waitq);
+
+ creq->stats.creq_qp_event_processed = 0;
+ creq->stats.creq_func_event_processed = 0;
+ creq->aeq_handler = aeq_handler;
+
+ rc = bnxt_qplib_map_cmdq_mbox(rcfw);
+ if (rc)
+ return rc;
+
+ rc = bnxt_qplib_map_creq_db(rcfw, cp_bar_reg_off);
+ if (rc)
+ return rc;
+
+ rc = bnxt_qplib_rcfw_start_irq(rcfw, msix_vector, true);
+ if (rc) {
+ dev_err(&rcfw->pdev->dev,
+ "QPLIB: Failed to request IRQ for CREQ rc = 0x%x\n", rc);
+ bnxt_qplib_disable_rcfw_channel(rcfw);
+ return rc;
+ }
+
+ rcfw->curr_shadow_qd = min_not_zero(cmdq_shadow_qd,
+ (unsigned int)RCFW_CMD_NON_BLOCKING_SHADOW_QD);
+ sema_init(&rcfw->rcfw_inflight, rcfw->curr_shadow_qd);
+ dev_dbg(&rcfw->pdev->dev,
+ "Perf Debug: shadow qd %d\n", rcfw->curr_shadow_qd);
+ bnxt_qplib_start_rcfw(rcfw);
+
+ return 0;
+}
diff --git a/sys/dev/bnxt/bnxt_re/qplib_rcfw.h b/sys/dev/bnxt/bnxt_re/qplib_rcfw.h
new file mode 100644
index 000000000000..f117525daacb
--- /dev/null
+++ b/sys/dev/bnxt/bnxt_re/qplib_rcfw.h
@@ -0,0 +1,354 @@
+/*
+ * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term
+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Description: RDMA Controller HW interface (header)
+ */
+
+#ifndef __BNXT_QPLIB_RCFW_H__
+#define __BNXT_QPLIB_RCFW_H__
+
+#include <linux/semaphore.h>
+#include <linux/module.h>
+#include <linux/netdevice.h>
+#include <linux/mutex.h>
+#include <linux/list.h>
+#include <linux/rculist.h>
+#include <linux/spinlock.h>
+#include <linux/pci.h>
+#include <net/ipv6.h>
+#include <linux/if_ether.h>
+#include <linux/debugfs.h>
+#include <linux/seq_file.h>
+#include <linux/interrupt.h>
+#include <linux/vmalloc.h>
+#include <linux/delay.h>
+
+#include "qplib_tlv.h"
+
+#define RCFW_CMDQ_TRIG_VAL 1
+#define RCFW_COMM_PCI_BAR_REGION 0
+#define RCFW_COMM_CONS_PCI_BAR_REGION 2
+#define RCFW_COMM_BASE_OFFSET 0x600
+#define RCFW_PF_VF_COMM_PROD_OFFSET 0xc
+#define RCFW_COMM_TRIG_OFFSET 0x100
+#define RCFW_COMM_SIZE 0x104
+
+#define RCFW_DBR_PCI_BAR_REGION 2
+#define RCFW_DBR_BASE_PAGE_SHIFT 12
+#define RCFW_MAX_LATENCY_SEC_SLAB_INDEX 128
+#define RCFW_MAX_LATENCY_MSEC_SLAB_INDEX 3000
+#define RCFW_MAX_STAT_INDEX 0xFFFF
+#define RCFW_FW_STALL_MAX_TIMEOUT 40
+
+#define GET_OPCODE_TYPE(x) \
+ ((x) == 0x1 ? "CREATE_QP": \
+ ((x) == 0x2 ? "DESTROY_QP": \
+ ((x) == 0x3 ? "MODIFY_QP": \
+ ((x) == 0x4 ? "QUERY_QP": \
+ ((x) == 0x5 ? "CREATE_SRQ": \
+ ((x) == 0x6 ? "DESTROY_SRQ": \
+ ((x) == 0x8 ? "QUERY_SRQ": \
+ ((x) == 0x9 ? "CREATE_CQ": \
+ ((x) == 0xa ? "DESTROY_CQ": \
+ ((x) == 0xc ? "RESIZE_CQ": \
+ ((x) == 0xd ? "ALLOCATE_MRW": \
+ ((x) == 0xe ? "DEALLOCATE_KEY": \
+ ((x) == 0xf ? "REGISTER_MR": \
+ ((x) == 0x10 ? "DEREGISTER_MR": \
+ ((x) == 0x11 ? "ADD_GID": \
+ ((x) == 0x12 ? "DELETE_GID": \
+ ((x) == 0x17 ? "MODIFY_GID": \
+ ((x) == 0x18 ? "QUERY_GID": \
+ ((x) == 0x13 ? "CREATE_QP1": \
+ ((x) == 0x14 ? "DESTROY_QP1": \
+ ((x) == 0x15 ? "CREATE_AH": \
+ ((x) == 0x16 ? "DESTROY_AH": \
+ ((x) == 0x80 ? "INITIALIZE_FW": \
+ ((x) == 0x81 ? "DEINITIALIZE_FW": \
+ ((x) == 0x82 ? "STOP_FUNC": \
+ ((x) == 0x83 ? "QUERY_FUNC": \
+ ((x) == 0x84 ? "SET_FUNC_RESOURCES": \
+ ((x) == 0x85 ? "READ_CONTEXT": \
+ ((x) == 0x86 ? "VF_BACKCHANNEL_REQUEST": \
+ ((x) == 0x87 ? "READ_VF_MEMORY": \
+ ((x) == 0x88 ? "COMPLETE_VF_REQUEST": \
+ ((x) == 0x89 ? "EXTEND_CONTEXT_ARRRAY": \
+ ((x) == 0x8a ? "MAP_TC_TO_COS": \
+ ((x) == 0x8b ? "QUERY_VERSION": \
+ ((x) == 0x8c ? "MODIFY_ROCE_CC": \
+ ((x) == 0x8d ? "QUERY_ROCE_CC": \
+ ((x) == 0x8e ? "QUERY_ROCE_STATS": \
+ ((x) == 0x8f ? "SET_LINK_AGGR_MODE": \
+ ((x) == 0x90 ? "MODIFY_CQ": \
+ ((x) == 0x91 ? "QUERY_QP_EXTEND": \
+ ((x) == 0x92 ? "QUERY_ROCE_STATS_EXT": \
+ "Unknown OPCODE" \
+ )))))))))))))))))))))))))))))))))))))))))
+
+extern unsigned int cmdq_shadow_qd;
+/* Cmdq contains a fix number of a 16-Byte slots */
+struct bnxt_qplib_cmdqe {
+ u8 data[16];
+};
+#define BNXT_QPLIB_CMDQE_UNITS sizeof(struct bnxt_qplib_cmdqe)
+
+static inline void bnxt_qplib_rcfw_cmd_prep(void *r, u8 opcode, u8 cmd_size)
+{
+ struct cmdq_base *req = r;
+
+ req->opcode = opcode;
+ req->cmd_size = cmd_size;
+}
+
+/* Shadow queue depth for non blocking command */
+#define RCFW_CMD_NON_BLOCKING_SHADOW_QD 64
+#define RCFW_CMD_DEV_ERR_CHECK_TIME_MS 1000 /* 1 Second time out*/
+#define RCFW_ERR_RETRY_COUNT (RCFW_CMD_WAIT_TIME_MS / RCFW_CMD_DEV_ERR_CHECK_TIME_MS)
+
+/* CMDQ elements */
+#define BNXT_QPLIB_CMDQE_MAX_CNT 8192
+#define BNXT_QPLIB_CMDQE_BYTES (BNXT_QPLIB_CMDQE_MAX_CNT * \
+ BNXT_QPLIB_CMDQE_UNITS)
+#define BNXT_QPLIB_CMDQE_NPAGES ((BNXT_QPLIB_CMDQE_BYTES % \
+ PAGE_SIZE) ? \
+ ((BNXT_QPLIB_CMDQE_BYTES / \
+ PAGE_SIZE) + 1) : \
+ (BNXT_QPLIB_CMDQE_BYTES / \
+ PAGE_SIZE))
+#define BNXT_QPLIB_CMDQE_PAGE_SIZE (BNXT_QPLIB_CMDQE_NPAGES * \
+ PAGE_SIZE)
+
+#define RCFW_MAX_OUTSTANDING_CMD BNXT_QPLIB_CMDQE_MAX_CNT
+#define RCFW_MAX_COOKIE_VALUE (BNXT_QPLIB_CMDQE_MAX_CNT - 1)
+#define RCFW_CMD_IS_BLOCKING 0x8000
+#define RCFW_NO_FW_ACCESS(rcfw) \
+ (test_bit(ERR_DEVICE_DETACHED, &(rcfw)->cmdq.flags) || \
+ pci_channel_offline((rcfw)->pdev))
+
+/* Crsq buf is 1024-Byte */
+struct bnxt_qplib_crsbe {
+ u8 data[1024];
+};
+
+/* Get the number of command units required for the req. The
+ * function returns correct value only if called before
+ * setting using bnxt_qplib_set_cmd_slots
+ */
+static inline u32 bnxt_qplib_get_cmd_slots(struct cmdq_base *req)
+{
+ u32 cmd_units = 0;
+
+ if (HAS_TLV_HEADER(req)) {
+ struct roce_tlv *tlv_req = (struct roce_tlv *)req;
+ cmd_units = tlv_req->total_size;
+ } else {
+ cmd_units = (req->cmd_size + BNXT_QPLIB_CMDQE_UNITS - 1) /
+ BNXT_QPLIB_CMDQE_UNITS;
+ }
+ return cmd_units;
+}
+
+/* Set the cmd_size to a factor of CMDQE unit */
+static inline u32 bnxt_qplib_set_cmd_slots(struct cmdq_base *req)
+{
+ u32 cmd_byte = 0;
+
+ if (HAS_TLV_HEADER(req)) {
+ struct roce_tlv *tlv_req = (struct roce_tlv *)req;
+ cmd_byte = tlv_req->total_size * BNXT_QPLIB_CMDQE_UNITS;
+ } else {
+ cmd_byte = req->cmd_size;
+ req->cmd_size = (req->cmd_size + BNXT_QPLIB_CMDQE_UNITS - 1) /
+ BNXT_QPLIB_CMDQE_UNITS;
+ }
+
+ return cmd_byte;
+}
+
+/* CREQ */
+/* Allocate 1 per QP for async error notification for now */
+#define BNXT_QPLIB_CREQE_MAX_CNT (64 * 1024)
+#define BNXT_QPLIB_CREQE_UNITS 16 /* 16-Bytes per prod unit */
+
+#define CREQ_CMP_VALID(hdr, pass) \
+ (!!((hdr)->v & CREQ_BASE_V) == \
+ !(pass & BNXT_QPLIB_FLAG_EPOCH_CONS_MASK))
+
+#define CREQ_ENTRY_POLL_BUDGET 8
+
+typedef int (*aeq_handler_t)(struct bnxt_qplib_rcfw *, void *, void *);
+
+struct bnxt_qplib_crsqe {
+ struct creq_qp_event *resp;
+ u32 req_size;
+ bool is_waiter_alive;
+ bool is_internal_cmd;
+ bool is_in_used;
+
+ /* Free slots at the time of submission */
+ u32 free_slots;
+ unsigned long send_timestamp;
+ u8 opcode;
+ u8 requested_qp_state;
+};
+
+struct bnxt_qplib_rcfw_sbuf {
+ void *sb;
+ dma_addr_t dma_addr;
+ u32 size;
+};
+
+#define BNXT_QPLIB_OOS_COUNT_MASK 0xFFFFFFFF
+
+#define FIRMWARE_INITIALIZED_FLAG (0)
+#define FIRMWARE_FIRST_FLAG (31)
+#define FIRMWARE_STALL_DETECTED (3)
+#define ERR_DEVICE_DETACHED (4)
+struct bnxt_qplib_cmdq_mbox {
+ struct bnxt_qplib_reg_desc reg;
+ void __iomem *prod;
+ void __iomem *db;
+};
+
+struct bnxt_qplib_cmdq_ctx {
+ struct bnxt_qplib_hwq hwq;
+ struct bnxt_qplib_cmdq_mbox cmdq_mbox;
+ wait_queue_head_t waitq;
+ unsigned long flags;
+ unsigned long last_seen;
+ u32 seq_num;
+};
+
+struct bnxt_qplib_creq_db {
+ struct bnxt_qplib_reg_desc reg;
+ void __iomem *db;
+ struct bnxt_qplib_db_info dbinfo;
+};
+
+struct bnxt_qplib_creq_stat {
+ u64 creq_arm_count;
+ u64 creq_tasklet_schedule_count;
+ u64 creq_qp_event_processed;
+ u64 creq_func_event_processed;
+};
+
+struct bnxt_qplib_creq_ctx {
+ struct bnxt_qplib_hwq hwq;
+ struct bnxt_qplib_creq_db creq_db;
+ struct bnxt_qplib_creq_stat stats;
+ aeq_handler_t aeq_handler;
+ u16 ring_id;
+ int msix_vec;
+ bool requested;
+ char *irq_name;
+};
+
+/* RCFW Communication Channels */
+#define BNXT_QPLIB_RCFW_SEND_RETRY_COUNT 4000
+struct bnxt_qplib_rcfw {
+ struct pci_dev *pdev;
+ struct bnxt_qplib_res *res;
+ struct bnxt_qplib_cmdq_ctx cmdq;
+ struct bnxt_qplib_creq_ctx creq;
+ struct bnxt_qplib_crsqe *crsqe_tbl;
+ u32 rcfw_lat_slab_sec[RCFW_MAX_LATENCY_SEC_SLAB_INDEX];
+
+ /* Slow path Perf Stats */
+ bool sp_perf_stats_enabled;
+ u32 *rcfw_lat_slab_msec;
+ u64 *qp_create_stats;
+ u64 *qp_destroy_stats;
+ u64 *qp_modify_stats;
+ u64 *mr_create_stats;
+ u64 *mr_destroy_stats;
+ u32 qp_create_stats_id;
+ u32 qp_destroy_stats_id;
+ u32 qp_modify_stats_id;
+ u32 mr_create_stats_id;
+ u32 mr_destroy_stats_id;
+ bool init_oos_stats;
+ u64 oos_prev;
+ u32 num_irq_stopped;
+ u32 num_irq_started;
+ u32 poll_in_intr_en;
+ u32 poll_in_intr_dis;
+ atomic_t rcfw_intr_enabled;
+ u32 cmdq_full_dbg;
+ struct semaphore rcfw_inflight;
+ unsigned int curr_shadow_qd;
+ atomic_t timeout_send;
+ /* cached from chip cctx for quick reference in slow path */
+ u16 max_timeout;
+};
+
+struct bnxt_qplib_cmdqmsg {
+ struct cmdq_base *req;
+ struct creq_base *resp;
+ void *sb;
+ u32 req_sz;
+ u32 res_sz;
+ u8 block;
+ u8 qp_state;
+};
+
+static inline void bnxt_qplib_fill_cmdqmsg(struct bnxt_qplib_cmdqmsg *msg,
+ void *req, void *resp, void *sb,
+ u32 req_sz, u32 res_sz, u8 block)
+{
+ msg->req = req;
+ msg->resp = resp;
+ msg->sb = sb;
+ msg->req_sz = req_sz;
+ msg->res_sz = res_sz;
+ msg->block = block;
+}
+
+void bnxt_qplib_free_rcfw_channel(struct bnxt_qplib_res *res);
+int bnxt_qplib_alloc_rcfw_channel(struct bnxt_qplib_res *res);
+void bnxt_qplib_rcfw_stop_irq(struct bnxt_qplib_rcfw *rcfw, bool kill);
+void bnxt_qplib_disable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw);
+int bnxt_qplib_rcfw_start_irq(struct bnxt_qplib_rcfw *rcfw, int msix_vector,
+ bool need_init);
+int bnxt_qplib_enable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw,
+ int msix_vector,
+ int cp_bar_reg_off,
+ aeq_handler_t aeq_handler);
+
+struct bnxt_qplib_rcfw_sbuf *bnxt_qplib_rcfw_alloc_sbuf(
+ struct bnxt_qplib_rcfw *rcfw,
+ u32 size);
+void bnxt_qplib_rcfw_free_sbuf(struct bnxt_qplib_rcfw *rcfw,
+ struct bnxt_qplib_rcfw_sbuf *sbuf);
+int bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw,
+ struct bnxt_qplib_cmdqmsg *msg);
+
+int bnxt_qplib_deinit_rcfw(struct bnxt_qplib_rcfw *rcfw);
+int bnxt_qplib_init_rcfw(struct bnxt_qplib_rcfw *rcfw, int is_virtfn);
+void bnxt_qplib_mark_qp_error(void *qp_handle);
+int __check_cmdq_stall(struct bnxt_qplib_rcfw *rcfw,
+ u32 *cur_prod, u32 *cur_cons);
+#endif
diff --git a/sys/dev/bnxt/bnxt_re/qplib_res.c b/sys/dev/bnxt/bnxt_re/qplib_res.c
new file mode 100644
index 000000000000..69661c67708c
--- /dev/null
+++ b/sys/dev/bnxt/bnxt_re/qplib_res.c
@@ -0,0 +1,1226 @@
+/*
+ * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term
+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Description: QPLib resource manager
+ */
+
+#include <linux/spinlock.h>
+#include <linux/pci.h>
+#include <linux/interrupt.h>
+#include <linux/inetdevice.h>
+#include <linux/dma-mapping.h>
+#include <linux/if_vlan.h>
+#include <linux/vmalloc.h>
+
+#include <net/ipv6.h>
+#include <rdma/ib_verbs.h>
+
+#include "hsi_struct_def.h"
+#include "qplib_res.h"
+#include "qplib_sp.h"
+#include "qplib_rcfw.h"
+#include "bnxt.h"
+#include "bnxt_ulp.h"
+
+uint8_t _get_chip_gen_p5_type(struct bnxt_qplib_chip_ctx *cctx)
+{
+ /* Extend this for granular type */
+ return(BNXT_RE_DEFAULT);
+}
+
+inline bool _is_alloc_mr_unified(struct bnxt_qplib_dev_attr *dattr)
+{
+ return dattr->dev_cap_flags &
+ CREQ_QUERY_FUNC_RESP_SB_MR_REGISTER_ALLOC;
+}
+
+/* PBL */
+static void __free_pbl(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_pbl *pbl, bool is_umem)
+{
+ struct pci_dev *pdev;
+ int i;
+
+ pdev = res->pdev;
+ if (is_umem == false) {
+ for (i = 0; i < pbl->pg_count; i++) {
+ if (pbl->pg_arr[i]) {
+ dma_free_coherent(&pdev->dev, pbl->pg_size,
+ (void *)((u64)pbl->pg_arr[i] &
+ PAGE_MASK),
+ pbl->pg_map_arr[i]);
+ }
+ else
+ dev_warn(&pdev->dev,
+ "QPLIB: PBL free pg_arr[%d] empty?!\n",
+ i);
+ pbl->pg_arr[i] = NULL;
+ }
+ }
+
+ if (pbl->pg_arr) {
+ vfree(pbl->pg_arr);
+ pbl->pg_arr = NULL;
+ }
+ if (pbl->pg_map_arr) {
+ vfree(pbl->pg_map_arr);
+ pbl->pg_map_arr = NULL;
+ }
+ pbl->pg_count = 0;
+ pbl->pg_size = 0;
+}
+
+struct qplib_sg {
+ dma_addr_t pg_map_arr;
+ u32 size;
+};
+
+static int __fill_user_dma_pages(struct bnxt_qplib_pbl *pbl,
+ struct bnxt_qplib_sg_info *sginfo)
+{
+ int sg_indx, pg_indx, tmp_size, offset;
+ struct qplib_sg *tmp_sg = NULL;
+ struct scatterlist *sg;
+ u64 pmask, addr;
+
+ tmp_sg = vzalloc(sginfo->nmap * sizeof(struct qplib_sg));
+ if (!tmp_sg)
+ return -ENOMEM;
+
+ pmask = BIT_ULL(sginfo->pgshft) - 1;
+ sg_indx = 0;
+ for_each_sg(sginfo->sghead, sg, sginfo->nmap, sg_indx) {
+ tmp_sg[sg_indx].pg_map_arr = sg_dma_address(sg);
+ tmp_sg[sg_indx].size = sg_dma_len(sg);
+ }
+ pg_indx = 0;
+ for (sg_indx = 0; sg_indx < sginfo->nmap; sg_indx++) {
+ tmp_size = tmp_sg[sg_indx].size;
+ offset = 0;
+ while (tmp_size > 0) {
+ addr = tmp_sg[sg_indx].pg_map_arr + offset;
+ if ((!sg_indx && !pg_indx) || !(addr & pmask)) {
+ pbl->pg_map_arr[pg_indx] = addr &(~pmask);
+ pbl->pg_count++;
+ pg_indx++;
+ }
+ offset += sginfo->pgsize;
+ tmp_size -= sginfo->pgsize;
+ }
+ }
+
+ vfree(tmp_sg);
+ return 0;
+}
+
+static int bnxt_qplib_fill_user_dma_pages(struct bnxt_qplib_pbl *pbl,
+ struct bnxt_qplib_sg_info *sginfo)
+{
+ int rc = 0;
+
+ rc = __fill_user_dma_pages(pbl, sginfo);
+
+ return rc;
+}
+
+static int __alloc_pbl(struct bnxt_qplib_res *res, struct bnxt_qplib_pbl *pbl,
+ struct bnxt_qplib_sg_info *sginfo)
+{
+ struct pci_dev *pdev;
+ bool is_umem = false;
+ int i;
+
+ if (sginfo->nopte)
+ return 0;
+
+ pdev = res->pdev;
+ /* page ptr arrays */
+ pbl->pg_arr = vmalloc(sginfo->npages * sizeof(void *));
+ if (!pbl->pg_arr)
+ return -ENOMEM;
+
+ pbl->pg_map_arr = vmalloc(sginfo->npages * sizeof(dma_addr_t));
+ if (!pbl->pg_map_arr) {
+ vfree(pbl->pg_arr);
+ return -ENOMEM;
+ }
+ pbl->pg_count = 0;
+ pbl->pg_size = sginfo->pgsize;
+ if (!sginfo->sghead) {
+ for (i = 0; i < sginfo->npages; i++) {
+ pbl->pg_arr[i] = dma_zalloc_coherent(&pdev->dev,
+ pbl->pg_size,
+ &pbl->pg_map_arr[i],
+ GFP_KERNEL);
+ if (!pbl->pg_arr[i])
+ goto fail;
+ pbl->pg_count++;
+ }
+ } else {
+ is_umem = true;
+ if (bnxt_qplib_fill_user_dma_pages(pbl, sginfo))
+ goto fail;
+ }
+
+ return 0;
+fail:
+ __free_pbl(res, pbl, is_umem);
+ return -ENOMEM;
+}
+
+/* HWQ */
+void bnxt_qplib_free_hwq(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_hwq *hwq)
+{
+ int i;
+
+ if (!hwq->max_elements)
+ return;
+ if (hwq->level >= PBL_LVL_MAX)
+ return;
+
+ for (i = 0; i < hwq->level + 1; i++) {
+ if (i == hwq->level)
+ __free_pbl(res, &hwq->pbl[i], hwq->is_user);
+ else
+ __free_pbl(res, &hwq->pbl[i], false);
+ }
+
+ hwq->level = PBL_LVL_MAX;
+ hwq->max_elements = 0;
+ hwq->element_size = 0;
+ hwq->prod = hwq->cons = 0;
+ hwq->cp_bit = 0;
+}
+
+/* All HWQs are power of 2 in size */
+int bnxt_qplib_alloc_init_hwq(struct bnxt_qplib_hwq *hwq,
+ struct bnxt_qplib_hwq_attr *hwq_attr)
+{
+ u32 npages = 0, depth, stride, aux_pages = 0;
+ dma_addr_t *src_phys_ptr, **dst_virt_ptr;
+ struct bnxt_qplib_sg_info sginfo = {};
+ u32 aux_size = 0, npbl, npde;
+ void *umem;
+ struct bnxt_qplib_res *res;
+ u32 aux_slots, pg_size;
+ struct pci_dev *pdev;
+ int i, rc, lvl;
+
+ res = hwq_attr->res;
+ pdev = res->pdev;
+ umem = hwq_attr->sginfo->sghead;
+ pg_size = hwq_attr->sginfo->pgsize;
+ hwq->level = PBL_LVL_MAX;
+
+ depth = roundup_pow_of_two(hwq_attr->depth);
+ stride = roundup_pow_of_two(hwq_attr->stride);
+ if (hwq_attr->aux_depth) {
+ aux_slots = hwq_attr->aux_depth;
+ aux_size = roundup_pow_of_two(hwq_attr->aux_stride);
+ aux_pages = (aux_slots * aux_size) / pg_size;
+ if ((aux_slots * aux_size) % pg_size)
+ aux_pages++;
+ }
+
+ if (!umem) {
+ hwq->is_user = false;
+ npages = (depth * stride) / pg_size + aux_pages;
+ if ((depth * stride) % pg_size)
+ npages++;
+ if (!npages)
+ return -EINVAL;
+ hwq_attr->sginfo->npages = npages;
+ } else {
+ hwq->is_user = true;
+ npages = hwq_attr->sginfo->npages;
+ npages = (npages * (u64)pg_size) /
+ BIT_ULL(hwq_attr->sginfo->pgshft);
+ if ((hwq_attr->sginfo->npages * (u64)pg_size) %
+ BIT_ULL(hwq_attr->sginfo->pgshft))
+ npages++;
+ }
+ if (npages == MAX_PBL_LVL_0_PGS && !hwq_attr->sginfo->nopte) {
+ /* This request is Level 0, map PTE */
+ rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_0], hwq_attr->sginfo);
+ if (rc)
+ goto fail;
+ hwq->level = PBL_LVL_0;
+ goto done;
+ }
+
+ if (npages >= MAX_PBL_LVL_0_PGS) {
+ if (npages > MAX_PBL_LVL_1_PGS) {
+ u32 flag = (hwq_attr->type == HWQ_TYPE_L2_CMPL) ?
+ 0 : PTU_PTE_VALID;
+ /* 2 levels of indirection */
+ npbl = npages >> MAX_PBL_LVL_1_PGS_SHIFT;
+ if (npages % BIT(MAX_PBL_LVL_1_PGS_SHIFT))
+ npbl++;
+ npde = npbl >> MAX_PDL_LVL_SHIFT;
+ if(npbl % BIT(MAX_PDL_LVL_SHIFT))
+ npde++;
+ /* Alloc PDE pages */
+ sginfo.pgsize = npde * PAGE_SIZE;
+ sginfo.npages = 1;
+ rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_0], &sginfo);
+
+ /* Alloc PBL pages */
+ sginfo.npages = npbl;
+ sginfo.pgsize = PAGE_SIZE;
+ rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_1], &sginfo);
+ if (rc)
+ goto fail;
+ /* Fill PDL with PBL page pointers */
+ dst_virt_ptr =
+ (dma_addr_t **)hwq->pbl[PBL_LVL_0].pg_arr;
+ src_phys_ptr = hwq->pbl[PBL_LVL_1].pg_map_arr;
+ if (hwq_attr->type == HWQ_TYPE_MR) {
+ /* For MR it is expected that we supply only 1 contigous
+ * page i.e only 1 entry in the PDL that will contain
+ * all the PBLs for the user supplied memory region
+ */
+ for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count; i++)
+ dst_virt_ptr[0][i] = src_phys_ptr[i] |
+ flag;
+ } else {
+ for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count; i++)
+ dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] =
+ src_phys_ptr[i] | PTU_PDE_VALID;
+ }
+ /* Alloc or init PTEs */
+ rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_2],
+ hwq_attr->sginfo);
+ if (rc)
+ goto fail;
+ hwq->level = PBL_LVL_2;
+ if (hwq_attr->sginfo->nopte)
+ goto done;
+ /* Fill PBLs with PTE pointers */
+ dst_virt_ptr =
+ (dma_addr_t **)hwq->pbl[PBL_LVL_1].pg_arr;
+ src_phys_ptr = hwq->pbl[PBL_LVL_2].pg_map_arr;
+ for (i = 0; i < hwq->pbl[PBL_LVL_2].pg_count; i++) {
+ dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] =
+ src_phys_ptr[i] | PTU_PTE_VALID;
+ }
+ if (hwq_attr->type == HWQ_TYPE_QUEUE) {
+ /* Find the last pg of the size */
+ i = hwq->pbl[PBL_LVL_2].pg_count;
+ dst_virt_ptr[PTR_PG(i - 1)][PTR_IDX(i - 1)] |=
+ PTU_PTE_LAST;
+ if (i > 1)
+ dst_virt_ptr[PTR_PG(i - 2)]
+ [PTR_IDX(i - 2)] |=
+ PTU_PTE_NEXT_TO_LAST;
+ }
+ } else { /* pages < 512 npbl = 1, npde = 0 */
+ u32 flag = (hwq_attr->type == HWQ_TYPE_L2_CMPL) ?
+ 0 : PTU_PTE_VALID;
+
+ /* 1 level of indirection */
+ npbl = npages >> MAX_PBL_LVL_1_PGS_SHIFT;
+ if (npages % BIT(MAX_PBL_LVL_1_PGS_SHIFT))
+ npbl++;
+ sginfo.npages = npbl;
+ sginfo.pgsize = PAGE_SIZE;
+ /* Alloc PBL page */
+ rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_0], &sginfo);
+ if (rc)
+ goto fail;
+ /* Alloc or init PTEs */
+ rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_1],
+ hwq_attr->sginfo);
+ if (rc)
+ goto fail;
+ hwq->level = PBL_LVL_1;
+ if (hwq_attr->sginfo->nopte)
+ goto done;
+ /* Fill PBL with PTE pointers */
+ dst_virt_ptr =
+ (dma_addr_t **)hwq->pbl[PBL_LVL_0].pg_arr;
+ src_phys_ptr = hwq->pbl[PBL_LVL_1].pg_map_arr;
+ for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count; i++)
+ dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] =
+ src_phys_ptr[i] | flag;
+ if (hwq_attr->type == HWQ_TYPE_QUEUE) {
+ /* Find the last pg of the size */
+ i = hwq->pbl[PBL_LVL_1].pg_count;
+ dst_virt_ptr[PTR_PG(i - 1)][PTR_IDX(i - 1)] |=
+ PTU_PTE_LAST;
+ if (i > 1)
+ dst_virt_ptr[PTR_PG(i - 2)]
+ [PTR_IDX(i - 2)] |=
+ PTU_PTE_NEXT_TO_LAST;
+ }
+ }
+ }
+done:
+ hwq->prod = 0;
+ hwq->cons = 0;
+ hwq->pdev = pdev;
+ hwq->depth = hwq_attr->depth;
+ hwq->max_elements = depth;
+ hwq->element_size = stride;
+ hwq->qe_ppg = (pg_size/stride);
+
+ if (hwq->level >= PBL_LVL_MAX)
+ goto fail;
+ /* For direct access to the elements */
+ lvl = hwq->level;
+ if (hwq_attr->sginfo->nopte && hwq->level)
+ lvl = hwq->level - 1;
+ hwq->pbl_ptr = hwq->pbl[lvl].pg_arr;
+ hwq->pbl_dma_ptr = hwq->pbl[lvl].pg_map_arr;
+ spin_lock_init(&hwq->lock);
+
+ return 0;
+fail:
+ bnxt_qplib_free_hwq(res, hwq);
+ return -ENOMEM;
+}
+
+/* Context Tables */
+void bnxt_qplib_free_hwctx(struct bnxt_qplib_res *res)
+{
+ struct bnxt_qplib_ctx *hctx;
+ int i;
+
+ hctx = res->hctx;
+ bnxt_qplib_free_hwq(res, &hctx->qp_ctx.hwq);
+ bnxt_qplib_free_hwq(res, &hctx->mrw_ctx.hwq);
+ bnxt_qplib_free_hwq(res, &hctx->srq_ctx.hwq);
+ bnxt_qplib_free_hwq(res, &hctx->cq_ctx.hwq);
+ bnxt_qplib_free_hwq(res, &hctx->tim_ctx.hwq);
+ for (i = 0; i < MAX_TQM_ALLOC_REQ; i++)
+ bnxt_qplib_free_hwq(res, &hctx->tqm_ctx.qtbl[i]);
+ /* restor original pde level before destroy */
+ hctx->tqm_ctx.pde.level = hctx->tqm_ctx.pde_level;
+ bnxt_qplib_free_hwq(res, &hctx->tqm_ctx.pde);
+}
+
+static int bnxt_qplib_alloc_tqm_rings(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_ctx *hctx)
+{
+ struct bnxt_qplib_hwq_attr hwq_attr = {};
+ struct bnxt_qplib_sg_info sginfo = {};
+ struct bnxt_qplib_tqm_ctx *tqmctx;
+ int rc = 0;
+ int i;
+
+ tqmctx = &hctx->tqm_ctx;
+
+ sginfo.pgsize = PAGE_SIZE;
+ sginfo.pgshft = PAGE_SHIFT;
+ hwq_attr.sginfo = &sginfo;
+ hwq_attr.res = res;
+ hwq_attr.type = HWQ_TYPE_CTX;
+ hwq_attr.depth = 512;
+ hwq_attr.stride = sizeof(u64);
+ /* Alloc pdl buffer */
+ rc = bnxt_qplib_alloc_init_hwq(&tqmctx->pde, &hwq_attr);
+ if (rc)
+ goto out;
+ /* Save original pdl level */
+ tqmctx->pde_level = tqmctx->pde.level;
+
+ hwq_attr.stride = 1;
+ for (i = 0; i < MAX_TQM_ALLOC_REQ; i++) {
+ if (!tqmctx->qcount[i])
+ continue;
+ hwq_attr.depth = hctx->qp_ctx.max * tqmctx->qcount[i];
+ rc = bnxt_qplib_alloc_init_hwq(&tqmctx->qtbl[i], &hwq_attr);
+ if (rc)
+ goto out;
+ }
+out:
+ return rc;
+}
+
+static void bnxt_qplib_map_tqm_pgtbl(struct bnxt_qplib_tqm_ctx *ctx)
+{
+ struct bnxt_qplib_hwq *qtbl_hwq;
+ dma_addr_t *dma_ptr;
+ __le64 **pbl_ptr, *ptr;
+ int i, j, k;
+ int fnz_idx = -1;
+ int pg_count;
+
+ pbl_ptr = (__le64 **)ctx->pde.pbl_ptr;
+
+ for (i = 0, j = 0; i < MAX_TQM_ALLOC_REQ;
+ i++, j += MAX_TQM_ALLOC_BLK_SIZE) {
+ qtbl_hwq = &ctx->qtbl[i];
+ if (!qtbl_hwq->max_elements)
+ continue;
+ if (fnz_idx == -1)
+ fnz_idx = i; /* first non-zero index */
+ switch (qtbl_hwq->level) {
+ case PBL_LVL_2:
+ pg_count = qtbl_hwq->pbl[PBL_LVL_1].pg_count;
+ for (k = 0; k < pg_count; k++) {
+ ptr = &pbl_ptr[PTR_PG(j + k)][PTR_IDX(j + k)];
+ dma_ptr = &qtbl_hwq->pbl[PBL_LVL_1].pg_map_arr[k];
+ *ptr = cpu_to_le64(*dma_ptr | PTU_PTE_VALID);
+ }
+ break;
+ case PBL_LVL_1:
+ case PBL_LVL_0:
+ default:
+ ptr = &pbl_ptr[PTR_PG(j)][PTR_IDX(j)];
+ *ptr = cpu_to_le64(qtbl_hwq->pbl[PBL_LVL_0].pg_map_arr[0] |
+ PTU_PTE_VALID);
+ break;
+ }
+ }
+ if (fnz_idx == -1)
+ fnz_idx = 0;
+ /* update pde level as per page table programming */
+ ctx->pde.level = (ctx->qtbl[fnz_idx].level == PBL_LVL_2) ? PBL_LVL_2 :
+ ctx->qtbl[fnz_idx].level + 1;
+}
+
+static int bnxt_qplib_setup_tqm_rings(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_ctx *hctx)
+{
+ int rc = 0;
+
+ rc = bnxt_qplib_alloc_tqm_rings(res, hctx);
+ if (rc)
+ goto fail;
+
+ bnxt_qplib_map_tqm_pgtbl(&hctx->tqm_ctx);
+fail:
+ return rc;
+}
+
+/*
+ * Routine: bnxt_qplib_alloc_hwctx
+ * Description:
+ * Context tables are memories which are used by the chip.
+ * The 6 tables defined are:
+ * QPC ctx - holds QP states
+ * MRW ctx - holds memory region and window
+ * SRQ ctx - holds shared RQ states
+ * CQ ctx - holds completion queue states
+ * TQM ctx - holds Tx Queue Manager context
+ * TIM ctx - holds timer context
+ * Depending on the size of the tbl requested, either a 1 Page Buffer List
+ * or a 1-to-2-stage indirection Page Directory List + 1 PBL is used
+ * instead.
+ * Table might be employed as follows:
+ * For 0 < ctx size <= 1 PAGE, 0 level of ind is used
+ * For 1 PAGE < ctx size <= 512 entries size, 1 level of ind is used
+ * For 512 < ctx size <= MAX, 2 levels of ind is used
+ * Returns:
+ * 0 if success, else -ERRORS
+ */
+int bnxt_qplib_alloc_hwctx(struct bnxt_qplib_res *res)
+{
+ struct bnxt_qplib_hwq_attr hwq_attr = {};
+ struct bnxt_qplib_sg_info sginfo = {};
+ struct bnxt_qplib_ctx *hctx;
+ struct bnxt_qplib_hwq *hwq;
+ int rc = 0;
+
+ hctx = res->hctx;
+ /* QPC Tables */
+ sginfo.pgsize = PAGE_SIZE;
+ sginfo.pgshft = PAGE_SHIFT;
+ hwq_attr.sginfo = &sginfo;
+
+ hwq_attr.res = res;
+ hwq_attr.depth = hctx->qp_ctx.max;
+ hwq_attr.stride = BNXT_QPLIB_MAX_QP_CTX_ENTRY_SIZE;
+ hwq_attr.type = HWQ_TYPE_CTX;
+ hwq = &hctx->qp_ctx.hwq;
+ rc = bnxt_qplib_alloc_init_hwq(hwq, &hwq_attr);
+ if (rc)
+ goto fail;
+
+ /* MRW Tables */
+ hwq_attr.depth = hctx->mrw_ctx.max;
+ hwq_attr.stride = BNXT_QPLIB_MAX_MRW_CTX_ENTRY_SIZE;
+ hwq = &hctx->mrw_ctx.hwq;
+ rc = bnxt_qplib_alloc_init_hwq(hwq, &hwq_attr);
+ if (rc)
+ goto fail;
+
+ /* SRQ Tables */
+ hwq_attr.depth = hctx->srq_ctx.max;
+ hwq_attr.stride = BNXT_QPLIB_MAX_SRQ_CTX_ENTRY_SIZE;
+ hwq = &hctx->srq_ctx.hwq;
+ rc = bnxt_qplib_alloc_init_hwq(hwq, &hwq_attr);
+ if (rc)
+ goto fail;
+
+ /* CQ Tables */
+ hwq_attr.depth = hctx->cq_ctx.max;
+ hwq_attr.stride = BNXT_QPLIB_MAX_CQ_CTX_ENTRY_SIZE;
+ hwq = &hctx->cq_ctx.hwq;
+ rc = bnxt_qplib_alloc_init_hwq(hwq, &hwq_attr);
+ if (rc)
+ goto fail;
+
+ /* TQM Buffer */
+ rc = bnxt_qplib_setup_tqm_rings(res, hctx);
+ if (rc)
+ goto fail;
+ /* TIM Buffer */
+ hwq_attr.depth = hctx->qp_ctx.max * 16;
+ hwq_attr.stride = 1;
+ hwq = &hctx->tim_ctx.hwq;
+ rc = bnxt_qplib_alloc_init_hwq(hwq, &hwq_attr);
+ if (rc)
+ goto fail;
+
+ return 0;
+fail:
+ bnxt_qplib_free_hwctx(res);
+ return rc;
+}
+
+/* GUID */
+void bnxt_qplib_get_guid(const u8 *dev_addr, u8 *guid)
+{
+ u8 mac[ETH_ALEN];
+
+ /* MAC-48 to EUI-64 mapping */
+ memcpy(mac, dev_addr, ETH_ALEN);
+ guid[0] = mac[0] ^ 2;
+ guid[1] = mac[1];
+ guid[2] = mac[2];
+ guid[3] = 0xff;
+ guid[4] = 0xfe;
+ guid[5] = mac[3];
+ guid[6] = mac[4];
+ guid[7] = mac[5];
+}
+
+static void bnxt_qplib_free_sgid_tbl(struct bnxt_qplib_res *res)
+{
+ struct bnxt_qplib_sgid_tbl *sgid_tbl;
+
+ sgid_tbl = &res->sgid_tbl;
+
+ if (sgid_tbl->tbl) {
+ kfree(sgid_tbl->tbl);
+ sgid_tbl->tbl = NULL;
+ kfree(sgid_tbl->hw_id);
+ sgid_tbl->hw_id = NULL;
+ kfree(sgid_tbl->ctx);
+ sgid_tbl->ctx = NULL;
+ kfree(sgid_tbl->vlan);
+ sgid_tbl->vlan = NULL;
+ } else {
+ dev_dbg(&res->pdev->dev, "QPLIB: SGID tbl not present");
+ }
+ sgid_tbl->max = 0;
+ sgid_tbl->active = 0;
+}
+
+static void bnxt_qplib_free_reftbls(struct bnxt_qplib_res *res)
+{
+ struct bnxt_qplib_reftbl *tbl;
+
+ tbl = &res->reftbl.srqref;
+ vfree(tbl->rec);
+
+ tbl = &res->reftbl.cqref;
+ vfree(tbl->rec);
+
+ tbl = &res->reftbl.qpref;
+ vfree(tbl->rec);
+}
+
+static int bnxt_qplib_alloc_reftbl(struct bnxt_qplib_reftbl *tbl, u32 max)
+{
+ tbl->max = max;
+ tbl->rec = vzalloc(sizeof(*tbl->rec) * max);
+ if (!tbl->rec)
+ return -ENOMEM;
+ spin_lock_init(&tbl->lock);
+ return 0;
+}
+
+static int bnxt_qplib_alloc_reftbls(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_dev_attr *dattr)
+{
+ u32 max_cq = BNXT_QPLIB_MAX_CQ_COUNT;
+ struct bnxt_qplib_reftbl *tbl;
+ u32 res_cnt;
+ int rc;
+
+ /*
+ * Allocating one extra entry to hold QP1 info.
+ * Store QP1 info at the last entry of the table.
+ * Decrement the tbl->max by one so that modulo
+ * operation to get the qp table index from qp id
+ * returns any value between 0 and max_qp-1
+ */
+ res_cnt = max_t(u32, BNXT_QPLIB_MAX_QPC_COUNT + 1, dattr->max_qp);
+ tbl = &res->reftbl.qpref;
+ rc = bnxt_qplib_alloc_reftbl(tbl, res_cnt);
+ if (rc)
+ goto fail;
+ tbl->max--;
+
+ if (_is_chip_gen_p5_p7(res->cctx))
+ max_cq = BNXT_QPLIB_MAX_CQ_COUNT_P5;
+ res_cnt = max_t(u32, max_cq, dattr->max_cq);
+ tbl = &res->reftbl.cqref;
+ rc = bnxt_qplib_alloc_reftbl(tbl, res_cnt);
+ if (rc)
+ goto fail;
+
+ res_cnt = max_t(u32, BNXT_QPLIB_MAX_SRQC_COUNT, dattr->max_cq);
+ tbl = &res->reftbl.srqref;
+ rc = bnxt_qplib_alloc_reftbl(tbl, BNXT_QPLIB_MAX_SRQC_COUNT);
+ if (rc)
+ goto fail;
+
+ return 0;
+fail:
+ return rc;
+}
+
+static int bnxt_qplib_alloc_sgid_tbl(struct bnxt_qplib_res *res, u16 max)
+{
+ struct bnxt_qplib_sgid_tbl *sgid_tbl;
+
+ sgid_tbl = &res->sgid_tbl;
+
+ sgid_tbl->tbl = kcalloc(max, sizeof(*sgid_tbl->tbl), GFP_KERNEL);
+ if (!sgid_tbl->tbl)
+ return -ENOMEM;
+
+ sgid_tbl->hw_id = kcalloc(max, sizeof(u32), GFP_KERNEL);
+ if (!sgid_tbl->hw_id)
+ goto free_tbl;
+
+ sgid_tbl->ctx = kcalloc(max, sizeof(void *), GFP_KERNEL);
+ if (!sgid_tbl->ctx)
+ goto free_hw_id;
+
+ sgid_tbl->vlan = kcalloc(max, sizeof(u8), GFP_KERNEL);
+ if (!sgid_tbl->vlan)
+ goto free_ctx;
+
+ sgid_tbl->max = max;
+ return 0;
+free_ctx:
+ kfree(sgid_tbl->ctx);
+free_hw_id:
+ kfree(sgid_tbl->hw_id);
+free_tbl:
+ kfree(sgid_tbl->tbl);
+ return -ENOMEM;
+};
+
+static void bnxt_qplib_cleanup_sgid_tbl(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_sgid_tbl *sgid_tbl)
+{
+ int i;
+
+ for (i = 0; i < sgid_tbl->max; i++) {
+ if (memcmp(&sgid_tbl->tbl[i], &bnxt_qplib_gid_zero,
+ sizeof(bnxt_qplib_gid_zero)))
+ bnxt_qplib_del_sgid(sgid_tbl, &sgid_tbl->tbl[i].gid,
+ sgid_tbl->tbl[i].vlan_id, true);
+ }
+ memset(sgid_tbl->tbl, 0, sizeof(*sgid_tbl->tbl) * sgid_tbl->max);
+ memset(sgid_tbl->hw_id, -1, sizeof(u16) * sgid_tbl->max);
+ memset(sgid_tbl->vlan, 0, sizeof(u8) * sgid_tbl->max);
+ sgid_tbl->active = 0;
+}
+
+static void bnxt_qplib_init_sgid_tbl(struct bnxt_qplib_sgid_tbl *sgid_tbl,
+ struct ifnet *netdev)
+{
+ u32 i;
+
+ for (i = 0; i < sgid_tbl->max; i++)
+ sgid_tbl->tbl[i].vlan_id = 0xffff;
+ memset(sgid_tbl->hw_id, -1, sizeof(u16) * sgid_tbl->max);
+}
+
+/* PDs */
+int bnxt_qplib_alloc_pd(struct bnxt_qplib_res *res, struct bnxt_qplib_pd *pd)
+{
+ u32 bit_num;
+ int rc = 0;
+ struct bnxt_qplib_pd_tbl *pdt = &res->pd_tbl;
+
+ mutex_lock(&res->pd_tbl_lock);
+ bit_num = find_first_bit(pdt->tbl, pdt->max);
+ if (bit_num == pdt->max - 1) {/* Last bit is reserved */
+ rc = -ENOMEM;
+ goto fail;
+ }
+
+ /* Found unused PD */
+ clear_bit(bit_num, pdt->tbl);
+ pd->id = bit_num;
+fail:
+ mutex_unlock(&res->pd_tbl_lock);
+ return rc;
+}
+
+int bnxt_qplib_dealloc_pd(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_pd_tbl *pdt,
+ struct bnxt_qplib_pd *pd)
+{
+ mutex_lock(&res->pd_tbl_lock);
+ if (test_and_set_bit(pd->id, pdt->tbl)) {
+ dev_warn(&res->pdev->dev, "Freeing an unused PD? pdn = %d\n",
+ pd->id);
+ mutex_unlock(&res->pd_tbl_lock);
+ return -EINVAL;
+ }
+ /* Reset to reserved pdid. */
+ pd->id = pdt->max - 1;
+
+ mutex_unlock(&res->pd_tbl_lock);
+ return 0;
+}
+
+static void bnxt_qplib_free_pd_tbl(struct bnxt_qplib_pd_tbl *pdt)
+{
+ if (pdt->tbl) {
+ kfree(pdt->tbl);
+ pdt->tbl = NULL;
+ }
+ pdt->max = 0;
+}
+
+static int bnxt_qplib_alloc_pd_tbl(struct bnxt_qplib_res *res, u32 max)
+{
+ struct bnxt_qplib_pd_tbl *pdt;
+ u32 bytes;
+
+ pdt = &res->pd_tbl;
+
+ max++; /* One extra for reserved pdid. */
+ bytes = DIV_ROUND_UP(max, 8);
+
+ if (!bytes)
+ bytes = 1;
+ pdt->tbl = kmalloc(bytes, GFP_KERNEL);
+ if (!pdt->tbl) {
+ dev_err(&res->pdev->dev,
+ "QPLIB: PD tbl allocation failed for size = %d\n", bytes);
+ return -ENOMEM;
+ }
+ pdt->max = max;
+ memset((u8 *)pdt->tbl, 0xFF, bytes);
+ mutex_init(&res->pd_tbl_lock);
+
+ return 0;
+}
+
+/* DPIs */
+int bnxt_qplib_alloc_dpi(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_dpi *dpi,
+ void *app, u8 type)
+{
+ struct bnxt_qplib_dpi_tbl *dpit = &res->dpi_tbl;
+ struct bnxt_qplib_reg_desc *reg;
+ u32 bit_num;
+ u64 umaddr;
+ int rc = 0;
+
+ reg = &dpit->wcreg;
+ mutex_lock(&res->dpi_tbl_lock);
+ if (type == BNXT_QPLIB_DPI_TYPE_WC && _is_chip_p7(res->cctx) &&
+ !dpit->avail_ppp) {
+ rc = -ENOMEM;
+ goto fail;
+ }
+ bit_num = find_first_bit(dpit->tbl, dpit->max);
+ if (bit_num == dpit->max) {
+ rc = -ENOMEM;
+ goto fail;
+ }
+ /* Found unused DPI */
+ clear_bit(bit_num, dpit->tbl);
+ dpit->app_tbl[bit_num] = app;
+ dpi->bit = bit_num;
+ dpi->dpi = bit_num + (reg->offset - dpit->ucreg.offset) / PAGE_SIZE;
+
+ umaddr = reg->bar_base + reg->offset + bit_num * PAGE_SIZE;
+ dpi->umdbr = umaddr;
+ switch (type) {
+ case BNXT_QPLIB_DPI_TYPE_KERNEL:
+ /* priviledged dbr was already mapped just initialize it. */
+ dpi->umdbr = dpit->ucreg.bar_base +
+ dpit->ucreg.offset + bit_num * PAGE_SIZE;
+ dpi->dbr = dpit->priv_db;
+ dpi->dpi = dpi->bit;
+ break;
+ case BNXT_QPLIB_DPI_TYPE_WC:
+ dpi->dbr = ioremap_wc(umaddr, PAGE_SIZE);
+ if (_is_chip_p7(res->cctx) && dpi->dbr)
+ dpit->avail_ppp--;
+ break;
+ default:
+ dpi->dbr = ioremap(umaddr, PAGE_SIZE);
+ }
+ if (!dpi->dbr) {
+ dev_err(&res->pdev->dev, "QPLIB: DB remap failed, type = %d\n",
+ type);
+ rc = -ENOMEM;
+ }
+ dpi->type = type;
+fail:
+ mutex_unlock(&res->dpi_tbl_lock);
+ return rc;
+}
+
+int bnxt_qplib_dealloc_dpi(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_dpi *dpi)
+{
+ struct bnxt_qplib_dpi_tbl *dpit = &res->dpi_tbl;
+ int rc = 0;
+
+ mutex_lock(&res->dpi_tbl_lock);
+ if (dpi->bit >= dpit->max) {
+ dev_warn(&res->pdev->dev,
+ "Invalid DPI? dpi = %d, bit = %d\n",
+ dpi->dpi, dpi->bit);
+ rc = -EINVAL;
+ goto fail;
+ }
+
+ if (dpi->dpi && dpi->type != BNXT_QPLIB_DPI_TYPE_KERNEL) {
+ if (dpi->type == BNXT_QPLIB_DPI_TYPE_WC &&
+ _is_chip_p7(res->cctx) && dpi->dbr)
+ dpit->avail_ppp++;
+ pci_iounmap(res->pdev, dpi->dbr);
+ }
+
+ if (test_and_set_bit(dpi->bit, dpit->tbl)) {
+ dev_warn(&res->pdev->dev,
+ "Freeing an unused DPI? dpi = %d, bit = %d\n",
+ dpi->dpi, dpi->bit);
+ rc = -EINVAL;
+ goto fail;
+ }
+ if (dpit->app_tbl)
+ dpit->app_tbl[dpi->bit] = NULL;
+ memset(dpi, 0, sizeof(*dpi));
+fail:
+ mutex_unlock(&res->dpi_tbl_lock);
+ return rc;
+}
+
+static void bnxt_qplib_free_dpi_tbl(struct bnxt_qplib_dpi_tbl *dpit)
+{
+ kfree(dpit->tbl);
+ kfree(dpit->app_tbl);
+ dpit->tbl = NULL;
+ dpit->app_tbl = NULL;
+ dpit->max = 0;
+}
+
+static int bnxt_qplib_alloc_dpi_tbl(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_dev_attr *dev_attr,
+ u8 pppp_factor)
+{
+ struct bnxt_qplib_dpi_tbl *dpit;
+ struct bnxt_qplib_reg_desc *reg;
+ unsigned long bar_len;
+ u32 dbr_offset;
+ u32 bytes;
+
+ dpit = &res->dpi_tbl;
+ reg = &dpit->wcreg;
+
+ if (!_is_chip_gen_p5_p7(res->cctx)) {
+ /* Offest should come from L2 driver */
+ dbr_offset = dev_attr->l2_db_size;
+ dpit->ucreg.offset = dbr_offset;
+ dpit->wcreg.offset = dbr_offset;
+ }
+
+ bar_len = pci_resource_len(res->pdev, reg->bar_id);
+ dpit->max = (bar_len - reg->offset) / PAGE_SIZE;
+ if (dev_attr->max_dpi)
+ dpit->max = min_t(u32, dpit->max, dev_attr->max_dpi);
+
+ dpit->app_tbl = kzalloc(dpit->max * sizeof(void*), GFP_KERNEL);
+ if (!dpit->app_tbl) {
+ dev_err(&res->pdev->dev,
+ "QPLIB: DPI app tbl allocation failed");
+ return -ENOMEM;
+ }
+
+ bytes = dpit->max >> 3;
+ if (!bytes)
+ bytes = 1;
+
+ dpit->tbl = kmalloc(bytes, GFP_KERNEL);
+ if (!dpit->tbl) {
+ kfree(dpit->app_tbl);
+ dev_err(&res->pdev->dev,
+ "QPLIB: DPI tbl allocation failed for size = %d\n",
+ bytes);
+ return -ENOMEM;
+ }
+
+ memset((u8 *)dpit->tbl, 0xFF, bytes);
+ /*
+ * On SR2, 2nd doorbell page of each function
+ * is reserved for L2 PPP. Now that the tbl is
+ * initialized, mark it as unavailable. By default
+ * RoCE can make use of the 512 extended pages for
+ * PPP.
+ */
+ if (_is_chip_p7(res->cctx)) {
+ clear_bit(1, dpit->tbl);
+ if (pppp_factor)
+ dpit->avail_ppp =
+ BNXT_QPLIB_MAX_EXTENDED_PPP_PAGES / pppp_factor;
+ }
+ mutex_init(&res->dpi_tbl_lock);
+ dpit->priv_db = dpit->ucreg.bar_reg + dpit->ucreg.offset;
+
+ return 0;
+}
+
+/* Stats */
+void bnxt_qplib_free_stat_mem(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_stats *stats)
+{
+ struct pci_dev *pdev;
+
+ pdev = res->pdev;
+ if (stats->dma)
+ dma_free_coherent(&pdev->dev, stats->size,
+ stats->dma, stats->dma_map);
+
+ memset(stats, 0, sizeof(*stats));
+ stats->fw_id = -1;
+}
+
+int bnxt_qplib_alloc_stat_mem(struct pci_dev *pdev,
+ struct bnxt_qplib_chip_ctx *cctx,
+ struct bnxt_qplib_stats *stats)
+{
+ cctx->hw_stats_size = 168;
+
+ memset(stats, 0, sizeof(*stats));
+ stats->fw_id = -1;
+ stats->size = cctx->hw_stats_size;
+ stats->dma = dma_alloc_coherent(&pdev->dev, stats->size,
+ &stats->dma_map, GFP_KERNEL);
+ if (!stats->dma) {
+ dev_err(&pdev->dev, "QPLIB: Stats DMA allocation failed");
+ return -ENOMEM;
+ }
+ return 0;
+}
+
+/* Resource */
+int bnxt_qplib_stop_res(struct bnxt_qplib_res *res)
+{
+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+ struct creq_stop_func_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct cmdq_stop_func req = {};
+ int rc;
+
+ bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_STOP_FUNC,
+ sizeof(req));
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+ return rc;
+}
+
+void bnxt_qplib_clear_tbls(struct bnxt_qplib_res *res)
+{
+ bnxt_qplib_cleanup_sgid_tbl(res, &res->sgid_tbl);
+}
+
+int bnxt_qplib_init_tbls(struct bnxt_qplib_res *res)
+{
+ bnxt_qplib_init_sgid_tbl(&res->sgid_tbl, res->netdev);
+
+ return 0;
+}
+
+void bnxt_qplib_free_tbls(struct bnxt_qplib_res *res)
+{
+ bnxt_qplib_free_sgid_tbl(res);
+ bnxt_qplib_free_pd_tbl(&res->pd_tbl);
+ bnxt_qplib_free_dpi_tbl(&res->dpi_tbl);
+ bnxt_qplib_free_reftbls(res);
+}
+
+int bnxt_qplib_alloc_tbls(struct bnxt_qplib_res *res, u8 pppp_factor)
+{
+ struct bnxt_qplib_dev_attr *dev_attr;
+ int rc = 0;
+
+ dev_attr = res->dattr;
+
+ rc = bnxt_qplib_alloc_reftbls(res, dev_attr);
+ if (rc)
+ goto fail;
+
+ rc = bnxt_qplib_alloc_sgid_tbl(res, dev_attr->max_sgid);
+ if (rc)
+ goto fail;
+
+ rc = bnxt_qplib_alloc_pd_tbl(res, dev_attr->max_pd);
+ if (rc)
+ goto fail;
+
+ rc = bnxt_qplib_alloc_dpi_tbl(res, dev_attr, pppp_factor);
+ if (rc)
+ goto fail;
+
+ return 0;
+fail:
+ bnxt_qplib_free_tbls(res);
+ return rc;
+}
+
+void bnxt_qplib_unmap_db_bar(struct bnxt_qplib_res *res)
+{
+ struct bnxt_qplib_reg_desc *reg;
+
+ reg = &res->dpi_tbl.ucreg;
+ if (reg->bar_reg)
+ pci_iounmap(res->pdev, reg->bar_reg);
+ reg->bar_reg = NULL;
+ reg->bar_base = 0;
+ reg->len = 0;
+ reg->bar_id = 0; /* Zero? or ff */
+}
+
+int bnxt_qplib_map_db_bar(struct bnxt_qplib_res *res)
+{
+ struct bnxt_qplib_reg_desc *ucreg;
+ struct bnxt_qplib_reg_desc *wcreg;
+
+ wcreg = &res->dpi_tbl.wcreg;
+ wcreg->bar_id = RCFW_DBR_PCI_BAR_REGION;
+ if (!res || !res->pdev || !wcreg)
+ return -1;
+ wcreg->bar_base = pci_resource_start(res->pdev, wcreg->bar_id);
+ /* No need to set the wcreg->len here */
+
+ ucreg = &res->dpi_tbl.ucreg;
+ ucreg->bar_id = RCFW_DBR_PCI_BAR_REGION;
+ ucreg->bar_base = pci_resource_start(res->pdev, ucreg->bar_id);
+
+ ucreg->offset = 65536;
+
+ ucreg->len = ucreg->offset + PAGE_SIZE;
+
+ if (!ucreg->len || ((ucreg->len & (PAGE_SIZE - 1)) != 0)) {
+ dev_err(&res->pdev->dev, "QPLIB: invalid dbr length %d\n",
+ (int)ucreg->len);
+ return -EINVAL;
+ }
+ ucreg->bar_reg = ioremap(ucreg->bar_base, ucreg->len);
+ if (!ucreg->bar_reg) {
+ dev_err(&res->pdev->dev, "priviledged dpi map failed!\n");
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+/**
+ * pci_enable_atomic_ops_to_root - enable AtomicOp requests to root port
+ * @dev: the PCI device
+ * @cap_mask: mask of desired AtomicOp sizes, including one or more of:
+ * PCI_EXP_DEVCAP2_ATOMIC_COMP32
+ * PCI_EXP_DEVCAP2_ATOMIC_COMP64
+ * PCI_EXP_DEVCAP2_ATOMIC_COMP128
+ *
+ * Return 0 if all upstream bridges support AtomicOp routing, egress
+ * blocking is disabled on all upstream ports, and the root port supports
+ * the requested completion capabilities (32-bit, 64-bit and/or 128-bit
+ * AtomicOp completion), or negative otherwise.
+ */
+int pci_enable_atomic_ops_to_root(struct pci_dev *dev, u32 cap_mask)
+{
+ struct pci_bus *bus = dev->bus;
+ struct pci_dev *bridge;
+ u32 cap;
+
+ if (!pci_is_pcie(dev))
+ return -EINVAL;
+
+ /*
+ * Per PCIe r4.0, sec 6.15, endpoints and root ports may be
+ * AtomicOp requesters. For now, we only support endpoints as
+ * requesters and root ports as completers. No endpoints as
+ * completers, and no peer-to-peer.
+ */
+
+ switch (pci_pcie_type(dev)) {
+ case PCI_EXP_TYPE_ENDPOINT:
+ case PCI_EXP_TYPE_LEG_END:
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ bridge = bus->self;
+
+ pcie_capability_read_dword(bridge, PCI_EXP_DEVCAP2, &cap);
+
+ switch (pci_pcie_type(bridge)) {
+ case PCI_EXP_TYPE_DOWNSTREAM:
+ if (!(cap & PCI_EXP_DEVCAP2_ATOMIC_ROUTE))
+ return -EINVAL;
+ break;
+
+ /* Ensure root port supports all the sizes we care about */
+ case PCI_EXP_TYPE_ROOT_PORT:
+ if ((cap & cap_mask) != cap_mask)
+ return -EINVAL;
+ break;
+ }
+ return 0;
+}
+
+int bnxt_qplib_enable_atomic_ops_to_root(struct pci_dev *dev)
+{
+ u16 ctl2;
+
+ if(pci_enable_atomic_ops_to_root(dev, PCI_EXP_DEVCAP2_ATOMIC_COMP32) &&
+ pci_enable_atomic_ops_to_root(dev, PCI_EXP_DEVCAP2_ATOMIC_COMP64))
+ return -EOPNOTSUPP;
+
+ pcie_capability_read_word(dev, PCI_EXP_DEVCTL2, &ctl2);
+ return !(ctl2 & PCI_EXP_DEVCTL2_ATOMIC_REQ);
+}
diff --git a/sys/dev/bnxt/bnxt_re/qplib_res.h b/sys/dev/bnxt/bnxt_re/qplib_res.h
new file mode 100644
index 000000000000..6468207a49aa
--- /dev/null
+++ b/sys/dev/bnxt/bnxt_re/qplib_res.h
@@ -0,0 +1,840 @@
+/*
+ * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term
+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Description: QPLib resource manager (header)
+ */
+
+#ifndef __BNXT_QPLIB_RES_H__
+#define __BNXT_QPLIB_RES_H__
+
+#include "hsi_struct_def.h"
+
+extern const struct bnxt_qplib_gid bnxt_qplib_gid_zero;
+
+#define CHIP_NUM_57508 0x1750
+#define CHIP_NUM_57504 0x1751
+#define CHIP_NUM_57502 0x1752
+#define CHIP_NUM_58818 0xd818
+#define CHIP_NUM_57608 0x1760
+
+#define BNXT_QPLIB_MAX_QPC_COUNT (64 * 1024)
+#define BNXT_QPLIB_MAX_SRQC_COUNT (64 * 1024)
+#define BNXT_QPLIB_MAX_CQ_COUNT (64 * 1024)
+#define BNXT_QPLIB_MAX_CQ_COUNT_P5 (128 * 1024)
+
+#define BNXT_QPLIB_DBR_VALID (0x1UL << 26)
+#define BNXT_QPLIB_DBR_EPOCH_SHIFT 24
+#define BNXT_QPLIB_DBR_TOGGLE_SHIFT 25
+
+#define BNXT_QPLIB_DBR_PF_DB_OFFSET 0x10000
+#define BNXT_QPLIB_DBR_VF_DB_OFFSET 0x4000
+
+#define BNXT_QPLIB_DBR_KEY_INVALID -1
+
+/* chip gen type */
+#define BNXT_RE_DEFAULT 0xf
+
+enum bnxt_qplib_wqe_mode {
+ BNXT_QPLIB_WQE_MODE_STATIC = 0x00,
+ BNXT_QPLIB_WQE_MODE_VARIABLE = 0x01,
+ BNXT_QPLIB_WQE_MODE_INVALID = 0x02
+};
+
+#define BNXT_RE_PUSH_MODE_NONE 0
+#define BNXT_RE_PUSH_MODE_WCB 1
+#define BNXT_RE_PUSH_MODE_PPP 2
+#define BNXT_RE_PUSH_ENABLED(mode) ((mode) == BNXT_RE_PUSH_MODE_WCB ||\
+ (mode) == BNXT_RE_PUSH_MODE_PPP)
+#define BNXT_RE_PPP_ENABLED(cctx) ((cctx)->modes.db_push_mode ==\
+ BNXT_RE_PUSH_MODE_PPP)
+#define PCI_EXP_DEVCAP2_ATOMIC_ROUTE 0x00000040 /* Atomic Op routing */
+#define PCI_EXP_DEVCAP2_ATOMIC_COMP32 0x00000080 /* 32b AtomicOp completion */
+#define PCI_EXP_DEVCAP2_ATOMIC_COMP64 0x00000100 /* 64b AtomicOp completion */
+#define PCI_EXP_DEVCTL2_ATOMIC_EGRESS_BLOCK 0x0080 /* Block atomic egress */
+#define PCI_EXP_DEVCTL2_ATOMIC_REQ 0x0040 /* Set Atomic requests */
+
+int pci_enable_atomic_ops_to_root(struct pci_dev *dev, u32 cap_mask);
+
+struct bnxt_qplib_drv_modes {
+ u8 wqe_mode;
+ u8 te_bypass;
+ u8 db_push;
+ /* To control advanced cc params display in configfs */
+ u8 cc_pr_mode;
+ /* Other modes to follow here e.g. GSI QP mode */
+ u8 dbr_pacing;
+ u8 dbr_pacing_ext;
+ u8 dbr_drop_recov;
+ u8 dbr_primary_pf;
+ u8 dbr_pacing_v0;
+};
+
+struct bnxt_qplib_chip_ctx {
+ u16 chip_num;
+ u8 chip_rev;
+ u8 chip_metal;
+ u64 hwrm_intf_ver;
+ struct bnxt_qplib_drv_modes modes;
+ u32 dbr_stat_db_fifo;
+ u32 dbr_aeq_arm_reg;
+ u32 dbr_throttling_reg;
+ u16 hw_stats_size;
+ u16 hwrm_cmd_max_timeout;
+};
+
+static inline bool _is_chip_num_p7(u16 chip_num)
+{
+ return (chip_num == CHIP_NUM_58818 ||
+ chip_num == CHIP_NUM_57608);
+}
+
+static inline bool _is_chip_p7(struct bnxt_qplib_chip_ctx *cctx)
+{
+ return _is_chip_num_p7(cctx->chip_num);
+}
+
+/* SR2 is Gen P5 */
+static inline bool _is_chip_gen_p5(struct bnxt_qplib_chip_ctx *cctx)
+{
+ return (cctx->chip_num == CHIP_NUM_57508 ||
+ cctx->chip_num == CHIP_NUM_57504 ||
+ cctx->chip_num == CHIP_NUM_57502);
+}
+
+static inline bool _is_chip_gen_p5_p7(struct bnxt_qplib_chip_ctx *cctx)
+{
+ return (_is_chip_gen_p5(cctx) || _is_chip_p7(cctx));
+}
+
+static inline bool _is_wqe_mode_variable(struct bnxt_qplib_chip_ctx *cctx)
+{
+ return cctx->modes.wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE;
+}
+
+struct bnxt_qplib_db_pacing_data {
+ u32 do_pacing;
+ u32 pacing_th;
+ u32 dev_err_state;
+ u32 alarm_th;
+ u32 grc_reg_offset;
+ u32 fifo_max_depth;
+ u32 fifo_room_mask;
+ u8 fifo_room_shift;
+};
+
+static inline u8 bnxt_qplib_dbr_pacing_en(struct bnxt_qplib_chip_ctx *cctx)
+{
+ return cctx->modes.dbr_pacing;
+}
+
+static inline u8 bnxt_qplib_dbr_pacing_ext_en(struct bnxt_qplib_chip_ctx *cctx)
+{
+ return cctx->modes.dbr_pacing_ext;
+}
+
+static inline u8 bnxt_qplib_dbr_pacing_is_primary_pf(struct bnxt_qplib_chip_ctx *cctx)
+{
+ return cctx->modes.dbr_primary_pf;
+}
+
+static inline void bnxt_qplib_dbr_pacing_set_primary_pf
+ (struct bnxt_qplib_chip_ctx *cctx, u8 val)
+{
+ cctx->modes.dbr_primary_pf = val;
+}
+
+/* Defines for handling the HWRM version check */
+#define HWRM_VERSION_DEV_ATTR_MAX_DPI 0x1000A0000000D
+#define HWRM_VERSION_ROCE_STATS_FN_ID 0x1000A00000045
+
+#define PTR_CNT_PER_PG (PAGE_SIZE / sizeof(void *))
+#define PTR_MAX_IDX_PER_PG (PTR_CNT_PER_PG - 1)
+#define PTR_PG(x) (((x) & ~PTR_MAX_IDX_PER_PG) / PTR_CNT_PER_PG)
+#define PTR_IDX(x) ((x) & PTR_MAX_IDX_PER_PG)
+
+#define HWQ_CMP(idx, hwq) ((idx) & ((hwq)->max_elements - 1))
+#define HWQ_FREE_SLOTS(hwq) (hwq->max_elements - \
+ ((HWQ_CMP(hwq->prod, hwq)\
+ - HWQ_CMP(hwq->cons, hwq))\
+ & (hwq->max_elements - 1)))
+enum bnxt_qplib_hwq_type {
+ HWQ_TYPE_CTX,
+ HWQ_TYPE_QUEUE,
+ HWQ_TYPE_L2_CMPL,
+ HWQ_TYPE_MR
+};
+
+#define MAX_PBL_LVL_0_PGS 1
+#define MAX_PBL_LVL_1_PGS 512
+#define MAX_PBL_LVL_1_PGS_SHIFT 9
+#define MAX_PDL_LVL_SHIFT 9
+
+enum bnxt_qplib_pbl_lvl {
+ PBL_LVL_0,
+ PBL_LVL_1,
+ PBL_LVL_2,
+ PBL_LVL_MAX
+};
+
+#define ROCE_PG_SIZE_4K (4 * 1024)
+#define ROCE_PG_SIZE_8K (8 * 1024)
+#define ROCE_PG_SIZE_64K (64 * 1024)
+#define ROCE_PG_SIZE_2M (2 * 1024 * 1024)
+#define ROCE_PG_SIZE_8M (8 * 1024 * 1024)
+#define ROCE_PG_SIZE_1G (1024 * 1024 * 1024)
+enum bnxt_qplib_hwrm_pg_size {
+ BNXT_QPLIB_HWRM_PG_SIZE_4K = 0,
+ BNXT_QPLIB_HWRM_PG_SIZE_8K = 1,
+ BNXT_QPLIB_HWRM_PG_SIZE_64K = 2,
+ BNXT_QPLIB_HWRM_PG_SIZE_2M = 3,
+ BNXT_QPLIB_HWRM_PG_SIZE_8M = 4,
+ BNXT_QPLIB_HWRM_PG_SIZE_1G = 5,
+};
+
+struct bnxt_qplib_reg_desc {
+ u8 bar_id;
+ resource_size_t bar_base;
+ unsigned long offset;
+ void __iomem *bar_reg;
+ size_t len;
+};
+
+struct bnxt_qplib_pbl {
+ u32 pg_count;
+ u32 pg_size;
+ void **pg_arr;
+ dma_addr_t *pg_map_arr;
+};
+
+struct bnxt_qplib_sg_info {
+ struct scatterlist *sghead;
+ u32 nmap;
+ u32 npages;
+ u32 pgshft;
+ u32 pgsize;
+ bool nopte;
+};
+
+struct bnxt_qplib_hwq_attr {
+ struct bnxt_qplib_res *res;
+ struct bnxt_qplib_sg_info *sginfo;
+ enum bnxt_qplib_hwq_type type;
+ u32 depth;
+ u32 stride;
+ u32 aux_stride;
+ u32 aux_depth;
+};
+
+struct bnxt_qplib_hwq {
+ struct pci_dev *pdev;
+ spinlock_t lock;
+ struct bnxt_qplib_pbl pbl[PBL_LVL_MAX];
+ enum bnxt_qplib_pbl_lvl level; /* 0, 1, or 2 */
+ void **pbl_ptr; /* ptr for easy access
+ to the PBL entries */
+ dma_addr_t *pbl_dma_ptr; /* ptr for easy access
+ to the dma_addr */
+ u32 max_elements;
+ u32 depth; /* original requested depth */
+ u16 element_size; /* Size of each entry */
+ u16 qe_ppg; /* queue entry per page */
+
+ u32 prod; /* raw */
+ u32 cons; /* raw */
+ u8 cp_bit;
+ u8 is_user;
+ u64 *pad_pg;
+ u32 pad_stride;
+ u32 pad_pgofft;
+};
+
+struct bnxt_qplib_db_info {
+ void __iomem *db;
+ void __iomem *priv_db;
+ struct bnxt_qplib_hwq *hwq;
+ struct bnxt_qplib_res *res;
+ u32 xid;
+ u32 max_slot;
+ u32 flags;
+ u8 toggle;
+ spinlock_t lock;
+ u64 shadow_key;
+ u64 shadow_key_arm_ena;
+ u32 seed; /* For DB pacing */
+};
+
+enum bnxt_qplib_db_info_flags_mask {
+ BNXT_QPLIB_FLAG_EPOCH_CONS_SHIFT = 0x0UL,
+ BNXT_QPLIB_FLAG_EPOCH_PROD_SHIFT = 0x1UL,
+ BNXT_QPLIB_FLAG_EPOCH_CONS_MASK = 0x1UL,
+ BNXT_QPLIB_FLAG_EPOCH_PROD_MASK = 0x2UL,
+};
+
+enum bnxt_qplib_db_epoch_flag_shift {
+ BNXT_QPLIB_DB_EPOCH_CONS_SHIFT = BNXT_QPLIB_DBR_EPOCH_SHIFT,
+ BNXT_QPLIB_DB_EPOCH_PROD_SHIFT = (BNXT_QPLIB_DBR_EPOCH_SHIFT - 1)
+};
+
+/* Tables */
+struct bnxt_qplib_pd_tbl {
+ unsigned long *tbl;
+ u32 max;
+};
+
+struct bnxt_qplib_sgid_tbl {
+ struct bnxt_qplib_gid_info *tbl;
+ u16 *hw_id;
+ u16 max;
+ u16 active;
+ void *ctx;
+ bool *vlan;
+};
+
+enum {
+ BNXT_QPLIB_DPI_TYPE_KERNEL = 0,
+ BNXT_QPLIB_DPI_TYPE_UC = 1,
+ BNXT_QPLIB_DPI_TYPE_WC = 2
+};
+
+struct bnxt_qplib_dpi {
+ u32 dpi;
+ u32 bit;
+ void __iomem *dbr;
+ u64 umdbr;
+ u8 type;
+};
+
+#define BNXT_QPLIB_MAX_EXTENDED_PPP_PAGES 512
+struct bnxt_qplib_dpi_tbl {
+ void **app_tbl;
+ unsigned long *tbl;
+ u16 max;
+ u16 avail_ppp;
+ struct bnxt_qplib_reg_desc ucreg; /* Hold entire DB bar. */
+ struct bnxt_qplib_reg_desc wcreg;
+ void __iomem *priv_db;
+};
+
+struct bnxt_qplib_stats {
+ dma_addr_t dma_map;
+ void *dma;
+ u32 size;
+ u32 fw_id;
+};
+
+struct bnxt_qplib_vf_res {
+ u32 max_qp;
+ u32 max_mrw;
+ u32 max_srq;
+ u32 max_cq;
+ u32 max_gid;
+};
+
+#define BNXT_QPLIB_MAX_QP_CTX_ENTRY_SIZE 448
+#define BNXT_QPLIB_MAX_SRQ_CTX_ENTRY_SIZE 64
+#define BNXT_QPLIB_MAX_CQ_CTX_ENTRY_SIZE 64
+#define BNXT_QPLIB_MAX_MRW_CTX_ENTRY_SIZE 128
+
+#define MAX_TQM_ALLOC_REQ 48
+#define MAX_TQM_ALLOC_BLK_SIZE 8
+struct bnxt_qplib_tqm_ctx {
+ struct bnxt_qplib_hwq pde;
+ enum bnxt_qplib_pbl_lvl pde_level; /* Original level */
+ struct bnxt_qplib_hwq qtbl[MAX_TQM_ALLOC_REQ];
+ u8 qcount[MAX_TQM_ALLOC_REQ];
+};
+
+struct bnxt_qplib_hctx {
+ struct bnxt_qplib_hwq hwq;
+ u32 max;
+};
+
+struct bnxt_qplib_refrec {
+ void *handle;
+ u32 xid;
+};
+
+struct bnxt_qplib_reftbl {
+ struct bnxt_qplib_refrec *rec;
+ u32 max;
+ spinlock_t lock; /* reftbl lock */
+};
+
+struct bnxt_qplib_reftbls {
+ struct bnxt_qplib_reftbl qpref;
+ struct bnxt_qplib_reftbl cqref;
+ struct bnxt_qplib_reftbl srqref;
+};
+
+#define GET_TBL_INDEX(id, tbl) ((id) % (((tbl)->max) - 1))
+static inline u32 map_qp_id_to_tbl_indx(u32 qid, struct bnxt_qplib_reftbl *tbl)
+{
+ return (qid == 1) ? tbl->max : GET_TBL_INDEX(qid, tbl);
+}
+
+/*
+ * This structure includes the number of various roce resource table sizes
+ * actually allocated by the driver. May be less than the maximums the firmware
+ * allows if the driver imposes lower limits than the firmware.
+ */
+struct bnxt_qplib_ctx {
+ struct bnxt_qplib_hctx qp_ctx;
+ struct bnxt_qplib_hctx mrw_ctx;
+ struct bnxt_qplib_hctx srq_ctx;
+ struct bnxt_qplib_hctx cq_ctx;
+ struct bnxt_qplib_hctx tim_ctx;
+ struct bnxt_qplib_tqm_ctx tqm_ctx;
+
+ struct bnxt_qplib_stats stats;
+ struct bnxt_qplib_stats stats2;
+ struct bnxt_qplib_vf_res vf_res;
+};
+
+struct bnxt_qplib_res {
+ struct pci_dev *pdev;
+ struct bnxt_qplib_chip_ctx *cctx;
+ struct bnxt_qplib_dev_attr *dattr;
+ struct bnxt_qplib_ctx *hctx;
+ struct ifnet *netdev;
+ struct bnxt_en_dev *en_dev;
+
+ struct bnxt_qplib_rcfw *rcfw;
+
+ struct bnxt_qplib_pd_tbl pd_tbl;
+ struct mutex pd_tbl_lock;
+ struct bnxt_qplib_sgid_tbl sgid_tbl;
+ struct bnxt_qplib_dpi_tbl dpi_tbl;
+ struct mutex dpi_tbl_lock;
+ struct bnxt_qplib_reftbls reftbl;
+ bool prio;
+ bool is_vf;
+ struct bnxt_qplib_db_pacing_data *pacing_data;
+};
+
+struct bnxt_qplib_query_stats_info {
+ u32 function_id;
+ u8 collection_id;
+ bool vf_valid;
+};
+
+struct bnxt_qplib_query_qp_info {
+ u32 function_id;
+ u32 num_qps;
+ u32 start_index;
+ bool vf_valid;
+};
+
+struct bnxt_qplib_query_fn_info {
+ bool vf_valid;
+ u32 host;
+ u32 filter;
+};
+
+
+#define to_bnxt_qplib(ptr, type, member) \
+ container_of(ptr, type, member)
+
+struct bnxt_qplib_pd;
+struct bnxt_qplib_dev_attr;
+
+bool _is_chip_gen_p5(struct bnxt_qplib_chip_ctx *cctx);
+bool _is_chip_gen_p5_p7(struct bnxt_qplib_chip_ctx *cctx);
+bool _is_chip_a0(struct bnxt_qplib_chip_ctx *cctx);
+bool _is_chip_p7(struct bnxt_qplib_chip_ctx *cctx);
+bool _is_alloc_mr_unified(struct bnxt_qplib_dev_attr *dattr);
+void bnxt_qplib_free_hwq(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_hwq *hwq);
+int bnxt_qplib_alloc_init_hwq(struct bnxt_qplib_hwq *hwq,
+ struct bnxt_qplib_hwq_attr *hwq_attr);
+void bnxt_qplib_get_guid(const u8 *dev_addr, u8 *guid);
+int bnxt_qplib_alloc_pd(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_pd *pd);
+int bnxt_qplib_dealloc_pd(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_pd_tbl *pd_tbl,
+ struct bnxt_qplib_pd *pd);
+int bnxt_qplib_alloc_dpi(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_dpi *dpi,
+ void *app, u8 type);
+int bnxt_qplib_dealloc_dpi(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_dpi *dpi);
+int bnxt_qplib_stop_res(struct bnxt_qplib_res *res);
+void bnxt_qplib_clear_tbls(struct bnxt_qplib_res *res);
+int bnxt_qplib_init_tbls(struct bnxt_qplib_res *res);
+void bnxt_qplib_free_tbls(struct bnxt_qplib_res *res);
+int bnxt_qplib_alloc_tbls(struct bnxt_qplib_res *res, u8 pppp_factor);
+void bnxt_qplib_free_hwctx(struct bnxt_qplib_res *res);
+int bnxt_qplib_alloc_hwctx(struct bnxt_qplib_res *res);
+int bnxt_qplib_alloc_stat_mem(struct pci_dev *pdev,
+ struct bnxt_qplib_chip_ctx *cctx,
+ struct bnxt_qplib_stats *stats);
+void bnxt_qplib_free_stat_mem(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_stats *stats);
+
+int bnxt_qplib_map_db_bar(struct bnxt_qplib_res *res);
+void bnxt_qplib_unmap_db_bar(struct bnxt_qplib_res *res);
+int bnxt_qplib_enable_atomic_ops_to_root(struct pci_dev *dev);
+u8 _get_chip_gen_p5_type(struct bnxt_qplib_chip_ctx *cctx);
+
+static inline void *bnxt_qplib_get_qe(struct bnxt_qplib_hwq *hwq,
+ u32 indx, u64 *pg)
+{
+ u32 pg_num, pg_idx;
+
+ pg_num = (indx / hwq->qe_ppg);
+ pg_idx = (indx % hwq->qe_ppg);
+ if (pg)
+ *pg = (u64)&hwq->pbl_ptr[pg_num];
+ return (void *)((u8 *)hwq->pbl_ptr[pg_num] + hwq->element_size * pg_idx);
+}
+
+static inline void bnxt_qplib_hwq_incr_prod(struct bnxt_qplib_db_info *dbinfo,
+ struct bnxt_qplib_hwq *hwq, u32 cnt)
+{
+ /* move prod and update toggle/epoch if wrap around */
+ hwq->prod += cnt;
+ if (hwq->prod >= hwq->depth) {
+ hwq->prod %= hwq->depth;
+ dbinfo->flags ^= 1UL << BNXT_QPLIB_FLAG_EPOCH_PROD_SHIFT;
+ }
+}
+
+static inline void bnxt_qplib_hwq_incr_cons(u32 max_elements, u32 *cons,
+ u32 cnt, u32 *dbinfo_flags)
+{
+ /* move cons and update toggle/epoch if wrap around */
+ *cons += cnt;
+ if (*cons >= max_elements) {
+ *cons %= max_elements;
+ *dbinfo_flags ^= 1UL << BNXT_QPLIB_FLAG_EPOCH_CONS_SHIFT;
+ }
+}
+
+static inline u8 _get_pte_pg_size(struct bnxt_qplib_hwq *hwq)
+{
+ u8 pg_size = BNXT_QPLIB_HWRM_PG_SIZE_4K;
+ struct bnxt_qplib_pbl *pbl;
+
+ pbl = &hwq->pbl[hwq->level];
+ switch (pbl->pg_size) {
+ case ROCE_PG_SIZE_4K: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_4K;
+ break;
+ case ROCE_PG_SIZE_8K: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_8K;
+ break;
+ case ROCE_PG_SIZE_64K: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_64K;
+ break;
+ case ROCE_PG_SIZE_2M: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_2M;
+ break;
+ case ROCE_PG_SIZE_8M: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_8M;
+ break;
+ case ROCE_PG_SIZE_1G: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_1G;
+ break;
+ default:
+ break;
+ }
+ return pg_size;
+}
+
+static inline u64 _get_base_addr(struct bnxt_qplib_hwq *hwq)
+{
+ return hwq->pbl[PBL_LVL_0].pg_map_arr[0];
+}
+
+static inline u8 _get_base_pg_size(struct bnxt_qplib_hwq *hwq)
+{
+ u8 pg_size = BNXT_QPLIB_HWRM_PG_SIZE_4K;
+ struct bnxt_qplib_pbl *pbl;
+
+ pbl = &hwq->pbl[PBL_LVL_0];
+ switch (pbl->pg_size) {
+ case ROCE_PG_SIZE_4K: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_4K;
+ break;
+ case ROCE_PG_SIZE_8K: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_8K;
+ break;
+ case ROCE_PG_SIZE_64K: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_64K;
+ break;
+ case ROCE_PG_SIZE_2M: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_2M;
+ break;
+ case ROCE_PG_SIZE_8M: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_8M;
+ break;
+ case ROCE_PG_SIZE_1G: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_1G;
+ break;
+ default:
+ break;
+ }
+ return pg_size;
+}
+
+static inline enum bnxt_qplib_hwq_type _get_hwq_type(struct bnxt_qplib_res *res)
+{
+ return _is_chip_gen_p5_p7(res->cctx) ? HWQ_TYPE_QUEUE : HWQ_TYPE_L2_CMPL;
+}
+
+static inline bool _is_ext_stats_supported(u16 dev_cap_flags)
+{
+ return dev_cap_flags &
+ CREQ_QUERY_FUNC_RESP_SB_EXT_STATS;
+}
+
+static inline int bnxt_ext_stats_supported(struct bnxt_qplib_chip_ctx *ctx,
+ u16 flags, bool virtfn)
+{
+ return (_is_ext_stats_supported(flags) &&
+ ((virtfn && _is_chip_p7(ctx)) || (!virtfn)));
+}
+
+static inline bool _is_hw_retx_supported(u16 dev_cap_flags)
+{
+ return dev_cap_flags &
+ (CREQ_QUERY_FUNC_RESP_SB_HW_REQUESTER_RETX_ENABLED |
+ CREQ_QUERY_FUNC_RESP_SB_HW_RESPONDER_RETX_ENABLED);
+}
+
+/* Disable HW_RETX */
+#define BNXT_RE_HW_RETX(a) _is_hw_retx_supported((a))
+
+static inline bool _is_cqe_v2_supported(u16 dev_cap_flags)
+{
+ return dev_cap_flags &
+ CREQ_QUERY_FUNC_RESP_SB_CQE_V2;
+}
+
+#define BNXT_DB_FIFO_ROOM_MASK 0x1fff8000
+#define BNXT_DB_FIFO_ROOM_SHIFT 15
+#define BNXT_MAX_FIFO_DEPTH 0x2c00
+
+#define BNXT_DB_PACING_ALGO_THRESHOLD 250
+#define BNXT_DEFAULT_PACING_PROBABILITY 0xFFFF
+
+#define BNXT_DBR_PACING_WIN_BASE 0x2000
+#define BNXT_DBR_PACING_WIN_MAP_OFF 4
+#define BNXT_DBR_PACING_WIN_OFF(reg) (BNXT_DBR_PACING_WIN_BASE + \
+
+static inline void bnxt_qplib_ring_db32(struct bnxt_qplib_db_info *info,
+ bool arm)
+{
+ u32 key = 0;
+
+ key = info->hwq->cons | (CMPL_DOORBELL_IDX_VALID |
+ (CMPL_DOORBELL_KEY_CMPL & CMPL_DOORBELL_KEY_MASK));
+ if (!arm)
+ key |= CMPL_DOORBELL_MASK;
+ /* memory barrier */
+ wmb();
+ writel(key, info->db);
+}
+
+#define BNXT_QPLIB_INIT_DBHDR(xid, type, indx, toggle) \
+ (((u64)(((xid) & DBC_DBC_XID_MASK) | DBC_DBC_PATH_ROCE | \
+ (type) | BNXT_QPLIB_DBR_VALID) << 32) | (indx) | \
+ ((toggle) << (BNXT_QPLIB_DBR_TOGGLE_SHIFT)))
+
+static inline void bnxt_qplib_write_db(struct bnxt_qplib_db_info *info,
+ u64 key, void __iomem *db,
+ u64 *shadow_key)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&info->lock, flags);
+ *shadow_key = key;
+ writeq(key, db);
+ spin_unlock_irqrestore(&info->lock, flags);
+}
+
+static inline void __replay_writeq(u64 key, void __iomem *db)
+{
+ /* No need to replay uninitialised shadow_keys */
+ if (key != BNXT_QPLIB_DBR_KEY_INVALID)
+ writeq(key, db);
+}
+
+static inline void bnxt_qplib_replay_db(struct bnxt_qplib_db_info *info,
+ bool is_arm_ena)
+
+{
+ if (!spin_trylock_irq(&info->lock))
+ return;
+
+ if (is_arm_ena)
+ __replay_writeq(info->shadow_key_arm_ena, info->priv_db);
+ else
+ __replay_writeq(info->shadow_key, info->db);
+
+ spin_unlock_irq(&info->lock);
+}
+
+static inline void bnxt_qplib_ring_db(struct bnxt_qplib_db_info *info,
+ u32 type)
+{
+ u64 key = 0;
+ u32 indx;
+ u8 toggle = 0;
+
+ if (type == DBC_DBC_TYPE_CQ_ARMALL ||
+ type == DBC_DBC_TYPE_CQ_ARMSE)
+ toggle = info->toggle;
+
+ indx = ((info->hwq->cons & DBC_DBC_INDEX_MASK) |
+ ((info->flags & BNXT_QPLIB_FLAG_EPOCH_CONS_MASK) <<
+ BNXT_QPLIB_DB_EPOCH_CONS_SHIFT));
+
+ key = BNXT_QPLIB_INIT_DBHDR(info->xid, type, indx, toggle);
+ bnxt_qplib_write_db(info, key, info->db, &info->shadow_key);
+}
+
+static inline void bnxt_qplib_ring_prod_db(struct bnxt_qplib_db_info *info,
+ u32 type)
+{
+ u64 key = 0;
+ u32 indx;
+
+ indx = (((info->hwq->prod / info->max_slot) & DBC_DBC_INDEX_MASK) |
+ ((info->flags & BNXT_QPLIB_FLAG_EPOCH_PROD_MASK) <<
+ BNXT_QPLIB_DB_EPOCH_PROD_SHIFT));
+ key = BNXT_QPLIB_INIT_DBHDR(info->xid, type, indx, 0);
+ bnxt_qplib_write_db(info, key, info->db, &info->shadow_key);
+}
+
+static inline void bnxt_qplib_armen_db(struct bnxt_qplib_db_info *info,
+ u32 type)
+{
+ u64 key = 0;
+ u8 toggle = 0;
+
+ if (type == DBC_DBC_TYPE_CQ_ARMENA)
+ toggle = info->toggle;
+ /* Index always at 0 */
+ key = BNXT_QPLIB_INIT_DBHDR(info->xid, type, 0, toggle);
+ bnxt_qplib_write_db(info, key, info->priv_db,
+ &info->shadow_key_arm_ena);
+}
+
+static inline void bnxt_qplib_cq_coffack_db(struct bnxt_qplib_db_info *info)
+{
+ u64 key = 0;
+
+ /* Index always at 0 */
+ key = BNXT_QPLIB_INIT_DBHDR(info->xid, DBC_DBC_TYPE_CQ_CUTOFF_ACK, 0, 0);
+ bnxt_qplib_write_db(info, key, info->priv_db, &info->shadow_key);
+}
+
+static inline void bnxt_qplib_srq_arm_db(struct bnxt_qplib_db_info *info)
+{
+ u64 key = 0;
+
+ /* Index always at 0 */
+ key = BNXT_QPLIB_INIT_DBHDR(info->xid, DBC_DBC_TYPE_SRQ_ARM, 0, 0);
+ bnxt_qplib_write_db(info, key, info->priv_db, &info->shadow_key);
+}
+
+static inline void bnxt_qplib_ring_nq_db(struct bnxt_qplib_db_info *info,
+ struct bnxt_qplib_chip_ctx *cctx,
+ bool arm)
+{
+ u32 type;
+
+ type = arm ? DBC_DBC_TYPE_NQ_ARM : DBC_DBC_TYPE_NQ;
+ if (_is_chip_gen_p5_p7(cctx))
+ bnxt_qplib_ring_db(info, type);
+ else
+ bnxt_qplib_ring_db32(info, arm);
+}
+
+struct bnxt_qplib_max_res {
+ u32 max_qp;
+ u32 max_mr;
+ u32 max_cq;
+ u32 max_srq;
+ u32 max_ah;
+ u32 max_pd;
+};
+
+/*
+ * Defines for maximum resources supported for chip revisions
+ * Maximum PDs supported are restricted to Max QPs
+ * GENP4 - Wh+
+ * DEFAULT - Thor
+ */
+#define BNXT_QPLIB_GENP4_PF_MAX_QP (16 * 1024)
+#define BNXT_QPLIB_GENP4_PF_MAX_MRW (16 * 1024)
+#define BNXT_QPLIB_GENP4_PF_MAX_CQ (16 * 1024)
+#define BNXT_QPLIB_GENP4_PF_MAX_SRQ (1 * 1024)
+#define BNXT_QPLIB_GENP4_PF_MAX_AH (16 * 1024)
+#define BNXT_QPLIB_GENP4_PF_MAX_PD BNXT_QPLIB_GENP4_PF_MAX_QP
+
+#define BNXT_QPLIB_DEFAULT_PF_MAX_QP (64 * 1024)
+#define BNXT_QPLIB_DEFAULT_PF_MAX_MRW (256 * 1024)
+#define BNXT_QPLIB_DEFAULT_PF_MAX_CQ (64 * 1024)
+#define BNXT_QPLIB_DEFAULT_PF_MAX_SRQ (4 * 1024)
+#define BNXT_QPLIB_DEFAULT_PF_MAX_AH (64 * 1024)
+#define BNXT_QPLIB_DEFAULT_PF_MAX_PD BNXT_QPLIB_DEFAULT_PF_MAX_QP
+
+#define BNXT_QPLIB_DEFAULT_VF_MAX_QP (6 * 1024)
+#define BNXT_QPLIB_DEFAULT_VF_MAX_MRW (6 * 1024)
+#define BNXT_QPLIB_DEFAULT_VF_MAX_CQ (6 * 1024)
+#define BNXT_QPLIB_DEFAULT_VF_MAX_SRQ (4 * 1024)
+#define BNXT_QPLIB_DEFAULT_VF_MAX_AH (6 * 1024)
+#define BNXT_QPLIB_DEFAULT_VF_MAX_PD BNXT_QPLIB_DEFAULT_VF_MAX_QP
+
+static inline void bnxt_qplib_max_res_supported(struct bnxt_qplib_chip_ctx *cctx,
+ struct bnxt_qplib_res *qpl_res,
+ struct bnxt_qplib_max_res *max_res,
+ bool vf_res_limit)
+{
+ switch (cctx->chip_num) {
+ case CHIP_NUM_57608:
+ case CHIP_NUM_58818:
+ case CHIP_NUM_57504:
+ case CHIP_NUM_57502:
+ case CHIP_NUM_57508:
+ if (!qpl_res->is_vf) {
+ max_res->max_qp = BNXT_QPLIB_DEFAULT_PF_MAX_QP;
+ max_res->max_mr = BNXT_QPLIB_DEFAULT_PF_MAX_MRW;
+ max_res->max_cq = BNXT_QPLIB_DEFAULT_PF_MAX_CQ;
+ max_res->max_srq = BNXT_QPLIB_DEFAULT_PF_MAX_SRQ;
+ max_res->max_ah = BNXT_QPLIB_DEFAULT_PF_MAX_AH;
+ max_res->max_pd = BNXT_QPLIB_DEFAULT_PF_MAX_PD;
+ } else {
+ max_res->max_qp = BNXT_QPLIB_DEFAULT_VF_MAX_QP;
+ max_res->max_mr = BNXT_QPLIB_DEFAULT_VF_MAX_MRW;
+ max_res->max_cq = BNXT_QPLIB_DEFAULT_VF_MAX_CQ;
+ max_res->max_srq = BNXT_QPLIB_DEFAULT_VF_MAX_SRQ;
+ max_res->max_ah = BNXT_QPLIB_DEFAULT_VF_MAX_AH;
+ max_res->max_pd = BNXT_QPLIB_DEFAULT_VF_MAX_PD;
+ }
+ break;
+ default:
+ /* Wh+/Stratus max resources */
+ max_res->max_qp = BNXT_QPLIB_GENP4_PF_MAX_QP;
+ max_res->max_mr = BNXT_QPLIB_GENP4_PF_MAX_MRW;
+ max_res->max_cq = BNXT_QPLIB_GENP4_PF_MAX_CQ;
+ max_res->max_srq = BNXT_QPLIB_GENP4_PF_MAX_SRQ;
+ max_res->max_ah = BNXT_QPLIB_GENP4_PF_MAX_AH;
+ max_res->max_pd = BNXT_QPLIB_GENP4_PF_MAX_PD;
+ break;
+ }
+}
+#endif
diff --git a/sys/dev/bnxt/bnxt_re/qplib_sp.c b/sys/dev/bnxt/bnxt_re/qplib_sp.c
new file mode 100644
index 000000000000..8faa3cd9390c
--- /dev/null
+++ b/sys/dev/bnxt/bnxt_re/qplib_sp.c
@@ -0,0 +1,1234 @@
+/*
+ * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term
+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Description: Slow Path Operators
+ */
+
+#include <linux/interrupt.h>
+#include <linux/spinlock.h>
+#include <linux/sched.h>
+#include <linux/pci.h>
+#include <linux/if_ether.h>
+#include <linux/printk.h>
+
+#include "hsi_struct_def.h"
+#include "qplib_tlv.h"
+#include "qplib_res.h"
+#include "qplib_rcfw.h"
+#include "qplib_sp.h"
+
+const struct bnxt_qplib_gid bnxt_qplib_gid_zero = {{ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0 }};
+
+/* Device */
+static u8 bnxt_qplib_is_atomic_cap(struct bnxt_qplib_rcfw *rcfw)
+{
+ u16 pcie_ctl2 = 0;
+
+ if (!_is_chip_gen_p5_p7(rcfw->res->cctx))
+ return false;
+ pcie_capability_read_word(rcfw->pdev, PCI_EXP_DEVCTL2, &pcie_ctl2);
+ return (pcie_ctl2 & PCI_EXP_DEVCTL2_ATOMIC_REQ);
+}
+
+static void bnxt_qplib_query_version(struct bnxt_qplib_rcfw *rcfw, char *fw_ver)
+{
+ struct creq_query_version_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct cmdq_query_version req = {};
+ int rc = 0;
+
+ bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_QUERY_VERSION,
+ sizeof(req));
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+ if (rc) {
+ dev_err(&rcfw->pdev->dev, "QPLIB: Failed to query version\n");
+ return;
+ }
+ fw_ver[0] = resp.fw_maj;
+ fw_ver[1] = resp.fw_minor;
+ fw_ver[2] = resp.fw_bld;
+ fw_ver[3] = resp.fw_rsvd;
+}
+
+int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw)
+{
+ struct creq_query_func_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct creq_query_func_resp_sb *sb;
+ struct bnxt_qplib_rcfw_sbuf sbuf;
+ struct bnxt_qplib_dev_attr *attr;
+ struct bnxt_qplib_chip_ctx *cctx;
+ struct cmdq_query_func req = {};
+ u8 *tqm_alloc;
+ int i, rc = 0;
+ u32 temp;
+ u8 chip_gen = BNXT_RE_DEFAULT;
+
+ cctx = rcfw->res->cctx;
+ attr = rcfw->res->dattr;
+
+ bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_QUERY_FUNC,
+ sizeof(req));
+
+ sbuf.size = sizeof(*sb);
+ sbuf.sb = dma_zalloc_coherent(&rcfw->pdev->dev, sbuf.size,
+ &sbuf.dma_addr, GFP_KERNEL);
+ if (!sbuf.sb)
+ return -ENOMEM;
+
+ sb = sbuf.sb;
+ req.resp_size = sbuf.size / BNXT_QPLIB_CMDQE_UNITS;
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req),
+ sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+ if (rc)
+ goto bail;
+ /* Extract the context from the side buffer */
+ chip_gen = _get_chip_gen_p5_type(cctx);
+ attr->max_qp = le32_to_cpu(sb->max_qp);
+ attr->max_qp = min_t(u32, attr->max_qp, BNXT_RE_MAX_QP_SUPPORTED(chip_gen));
+ /* max_qp value reported by FW does not include the QP1 */
+ attr->max_qp += 1;
+ attr->max_qp_rd_atom =
+ sb->max_qp_rd_atom > BNXT_QPLIB_MAX_OUT_RD_ATOM ?
+ BNXT_QPLIB_MAX_OUT_RD_ATOM : sb->max_qp_rd_atom;
+ attr->max_qp_init_rd_atom =
+ sb->max_qp_init_rd_atom > BNXT_QPLIB_MAX_OUT_RD_ATOM ?
+ BNXT_QPLIB_MAX_OUT_RD_ATOM : sb->max_qp_init_rd_atom;
+ /* Report 1 less than the max_qp_wqes reported by FW as driver adds
+ * one extra entry while creating the qp
+ */
+ attr->max_qp_wqes = le16_to_cpu(sb->max_qp_wr) - 1;
+ /* Adjust for max_qp_wqes for variable wqe */
+ if (cctx->modes.wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) {
+ attr->max_qp_wqes = (BNXT_MAX_SQ_SIZE) /
+ (BNXT_MAX_VAR_WQE_SIZE / BNXT_SGE_SIZE) - 1;
+ }
+ if (!_is_chip_gen_p5_p7(cctx)) {
+ /*
+ * 128 WQEs needs to be reserved for the HW (8916). Prevent
+ * reporting the max number for gen-p4 only.
+ */
+ attr->max_qp_wqes -= BNXT_QPLIB_RESERVED_QP_WRS;
+ }
+ attr->max_qp_sges = sb->max_sge;
+ if (_is_chip_gen_p5_p7(cctx) &&
+ cctx->modes.wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE)
+ attr->max_qp_sges = sb->max_sge_var_wqe;
+ attr->max_cq = le32_to_cpu(sb->max_cq);
+ attr->max_cq = min_t(u32, attr->max_cq, BNXT_RE_MAX_CQ_SUPPORTED(chip_gen));
+
+ attr->max_cq_wqes = le32_to_cpu(sb->max_cqe);
+ attr->max_cq_wqes = min_t(u32, BNXT_QPLIB_MAX_CQ_WQES, attr->max_cq_wqes);
+
+ attr->max_cq_sges = attr->max_qp_sges;
+ attr->max_mr = le32_to_cpu(sb->max_mr);
+ attr->max_mr = min_t(u32, attr->max_mr, BNXT_RE_MAX_MRW_SUPPORTED(chip_gen));
+ attr->max_mw = le32_to_cpu(sb->max_mw);
+ attr->max_mw = min_t(u32, attr->max_mw, BNXT_RE_MAX_MRW_SUPPORTED(chip_gen));
+
+ attr->max_mr_size = le64_to_cpu(sb->max_mr_size);
+ attr->max_pd = BNXT_QPLIB_MAX_PD;
+ attr->max_raw_ethy_qp = le32_to_cpu(sb->max_raw_eth_qp);
+ attr->max_ah = le32_to_cpu(sb->max_ah);
+ attr->max_ah = min_t(u32, attr->max_ah, BNXT_RE_MAX_AH_SUPPORTED(chip_gen));
+
+ attr->max_fmr = le32_to_cpu(sb->max_fmr);
+ attr->max_map_per_fmr = sb->max_map_per_fmr;
+
+ attr->max_srq = le16_to_cpu(sb->max_srq);
+ attr->max_srq = min_t(u32, attr->max_srq, BNXT_RE_MAX_SRQ_SUPPORTED(chip_gen));
+ attr->max_srq_wqes = le32_to_cpu(sb->max_srq_wr) - 1;
+ attr->max_srq_sges = sb->max_srq_sge;
+ attr->max_pkey = 1;
+
+ attr->max_inline_data = !cctx->modes.wqe_mode ?
+ le32_to_cpu(sb->max_inline_data) :
+ le16_to_cpu(sb->max_inline_data_var_wqe);
+ if (!_is_chip_p7(cctx)) {
+ attr->l2_db_size = (sb->l2_db_space_size + 1) *
+ (0x01 << RCFW_DBR_BASE_PAGE_SHIFT);
+ }
+ attr->max_sgid = le32_to_cpu(sb->max_gid);
+
+ /* TODO: remove this hack for statically allocated gid_map */
+ bnxt_re_set_max_gid(&attr->max_sgid);
+
+ attr->dev_cap_flags = le16_to_cpu(sb->dev_cap_flags);
+ attr->page_size_cap = BIT_ULL(28) | BIT_ULL(21) | BIT_ULL(12);
+
+ bnxt_qplib_query_version(rcfw, attr->fw_ver);
+
+ for (i = 0; i < MAX_TQM_ALLOC_REQ / 4; i++) {
+ temp = le32_to_cpu(sb->tqm_alloc_reqs[i]);
+ tqm_alloc = (u8 *)&temp;
+ attr->tqm_alloc_reqs[i * 4] = *tqm_alloc;
+ attr->tqm_alloc_reqs[i * 4 + 1] = *(++tqm_alloc);
+ attr->tqm_alloc_reqs[i * 4 + 2] = *(++tqm_alloc);
+ attr->tqm_alloc_reqs[i * 4 + 3] = *(++tqm_alloc);
+ }
+
+ if (rcfw->res->cctx->hwrm_intf_ver >= HWRM_VERSION_DEV_ATTR_MAX_DPI)
+ attr->max_dpi = le32_to_cpu(sb->max_dpi);
+
+ attr->is_atomic = bnxt_qplib_is_atomic_cap(rcfw);
+bail:
+ dma_free_coherent(&rcfw->pdev->dev, sbuf.size,
+ sbuf.sb, sbuf.dma_addr);
+ return rc;
+}
+
+int bnxt_qplib_set_func_resources(struct bnxt_qplib_res *res)
+{
+ struct creq_set_func_resources_resp resp = {};
+ struct cmdq_set_func_resources req = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct bnxt_qplib_rcfw *rcfw;
+ struct bnxt_qplib_ctx *hctx;
+ int rc = 0;
+
+ rcfw = res->rcfw;
+ hctx = res->hctx;
+ bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_SET_FUNC_RESOURCES,
+ sizeof(req));
+
+ req.number_of_qp = cpu_to_le32(hctx->qp_ctx.max);
+ req.number_of_mrw = cpu_to_le32(hctx->mrw_ctx.max);
+ req.number_of_srq = cpu_to_le32(hctx->srq_ctx.max);
+ req.number_of_cq = cpu_to_le32(hctx->cq_ctx.max);
+
+ req.max_qp_per_vf = cpu_to_le32(hctx->vf_res.max_qp);
+ req.max_mrw_per_vf = cpu_to_le32(hctx->vf_res.max_mrw);
+ req.max_srq_per_vf = cpu_to_le32(hctx->vf_res.max_srq);
+ req.max_cq_per_vf = cpu_to_le32(hctx->vf_res.max_cq);
+ req.max_gid_per_vf = cpu_to_le32(hctx->vf_res.max_gid);
+
+ /* Keep the old stats context id of PF */
+ req.stat_ctx_id = cpu_to_le32(hctx->stats.fw_id);
+
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+ if (rc)
+ dev_err(&res->pdev->dev,
+ "QPLIB: Failed to set function resources\n");
+
+ return rc;
+}
+
+int bnxt_qplib_update_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,
+ struct bnxt_qplib_gid *gid, u16 gid_idx, const u8 *smac)
+{
+ struct bnxt_qplib_res *res = to_bnxt_qplib(sgid_tbl,
+ struct bnxt_qplib_res,
+ sgid_tbl);
+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+ struct creq_modify_gid_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct cmdq_modify_gid req = {};
+ int rc;
+
+ bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_MODIFY_GID,
+ sizeof(req));
+
+ req.gid[0] = cpu_to_be32(((u32 *)gid->data)[3]);
+ req.gid[1] = cpu_to_be32(((u32 *)gid->data)[2]);
+ req.gid[2] = cpu_to_be32(((u32 *)gid->data)[1]);
+ req.gid[3] = cpu_to_be32(((u32 *)gid->data)[0]);
+ if (res->prio) {
+ req.vlan |= cpu_to_le16(CMDQ_ADD_GID_VLAN_TPID_TPID_8100 |
+ CMDQ_ADD_GID_VLAN_VLAN_EN);
+ }
+
+ /* MAC in network format */
+ req.src_mac[0] = cpu_to_be16(((u16 *)smac)[0]);
+ req.src_mac[1] = cpu_to_be16(((u16 *)smac)[1]);
+ req.src_mac[2] = cpu_to_be16(((u16 *)smac)[2]);
+ req.gid_index = cpu_to_le16(gid_idx);
+
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+ if (rc) {
+ dev_err(&res->pdev->dev,
+ "QPLIB: update SGID table failed\n");
+ return rc;
+ }
+ return 0;
+}
+
+/* SGID */
+int bnxt_qplib_get_sgid(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_sgid_tbl *sgid_tbl, int index,
+ struct bnxt_qplib_gid *gid)
+{
+ if (index > sgid_tbl->max) {
+ dev_err(&res->pdev->dev,
+ "QPLIB: Index %d exceeded SGID table max (%d)\n",
+ index, sgid_tbl->max);
+ return -EINVAL;
+ }
+ memcpy(gid, &sgid_tbl->tbl[index].gid, sizeof(*gid));
+ return 0;
+}
+
+int bnxt_qplib_del_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,
+ struct bnxt_qplib_gid *gid,
+ u16 vlan_id, bool update)
+{
+ struct bnxt_qplib_res *res = to_bnxt_qplib(sgid_tbl,
+ struct bnxt_qplib_res,
+ sgid_tbl);
+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+ int index;
+
+ if (sgid_tbl == NULL) {
+ dev_err(&res->pdev->dev, "QPLIB: SGID table not allocated\n");
+ return -EINVAL;
+ }
+ /* Do we need a sgid_lock here? */
+ if (!sgid_tbl->active) {
+ dev_err(&res->pdev->dev,
+ "QPLIB: SGID table has no active entries\n");
+ return -ENOMEM;
+ }
+ for (index = 0; index < sgid_tbl->max; index++) {
+ if (!memcmp(&sgid_tbl->tbl[index].gid, gid, sizeof(*gid)) &&
+ vlan_id == sgid_tbl->tbl[index].vlan_id)
+ break;
+ }
+ if (index == sgid_tbl->max) {
+ dev_warn(&res->pdev->dev, "GID not found in the SGID table\n");
+ return 0;
+ }
+
+ if (update) {
+ struct creq_delete_gid_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct cmdq_delete_gid req = {};
+ int rc;
+
+ bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DELETE_GID,
+ sizeof(req));
+ if (sgid_tbl->hw_id[index] == 0xFFFF) {
+ dev_err(&res->pdev->dev,
+ "QPLIB: GID entry contains an invalid HW id");
+ return -EINVAL;
+ }
+ req.gid_index = cpu_to_le16(sgid_tbl->hw_id[index]);
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+ if (rc)
+ return rc;
+ }
+ memcpy(&sgid_tbl->tbl[index].gid, &bnxt_qplib_gid_zero,
+ sizeof(bnxt_qplib_gid_zero));
+ sgid_tbl->tbl[index].vlan_id = 0xFFFF;
+ sgid_tbl->vlan[index] = false;
+ sgid_tbl->active--;
+ dev_dbg(&res->pdev->dev,
+ "QPLIB: SGID deleted hw_id[0x%x] = 0x%x active = 0x%x\n",
+ index, sgid_tbl->hw_id[index], sgid_tbl->active);
+ sgid_tbl->hw_id[index] = (u16)-1;
+
+ return 0;
+}
+
+int bnxt_qplib_add_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,
+ const union ib_gid *gid, const u8 *smac, u16 vlan_id,
+ bool update, u32 *index)
+{
+ struct bnxt_qplib_res *res = to_bnxt_qplib(sgid_tbl,
+ struct bnxt_qplib_res,
+ sgid_tbl);
+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+ int i, free_idx;
+
+ if (sgid_tbl == NULL) {
+ dev_err(&res->pdev->dev, "QPLIB: SGID table not allocated\n");
+ return -EINVAL;
+ }
+ /* Do we need a sgid_lock here? */
+ if (sgid_tbl->active == sgid_tbl->max) {
+ dev_err(&res->pdev->dev, "QPLIB: SGID table is full\n");
+ return -ENOMEM;
+ }
+ free_idx = sgid_tbl->max;
+ for (i = 0; i < sgid_tbl->max; i++) {
+ if (!memcmp(&sgid_tbl->tbl[i], gid, sizeof(*gid)) &&
+ sgid_tbl->tbl[i].vlan_id == vlan_id) {
+ dev_dbg(&res->pdev->dev,
+ "QPLIB: SGID entry already exist in entry %d!\n",
+ i);
+ *index = i;
+ return -EALREADY;
+ } else if (!memcmp(&sgid_tbl->tbl[i], &bnxt_qplib_gid_zero,
+ sizeof(bnxt_qplib_gid_zero)) &&
+ free_idx == sgid_tbl->max) {
+ free_idx = i;
+ }
+ }
+ if (free_idx == sgid_tbl->max) {
+ dev_err(&res->pdev->dev,
+ "QPLIB: SGID table is FULL but count is not MAX??\n");
+ return -ENOMEM;
+ }
+ if (update) {
+ struct creq_add_gid_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct cmdq_add_gid req = {};
+ int rc;
+
+ bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_ADD_GID,
+ sizeof(req));
+
+ req.gid[0] = cpu_to_be32(((u32 *)gid->raw)[3]);
+ req.gid[1] = cpu_to_be32(((u32 *)gid->raw)[2]);
+ req.gid[2] = cpu_to_be32(((u32 *)gid->raw)[1]);
+ req.gid[3] = cpu_to_be32(((u32 *)gid->raw)[0]);
+ /* driver should ensure that all RoCE traffic is always VLAN tagged
+ * if RoCE traffic is running on non-zero VLAN ID or
+ * RoCE traffic is running on non-zero Priority.
+ */
+ if ((vlan_id != 0xFFFF) || res->prio) {
+ if (vlan_id != 0xFFFF)
+ req.vlan = cpu_to_le16(vlan_id &
+ CMDQ_ADD_GID_VLAN_VLAN_ID_MASK);
+ req.vlan |=
+ cpu_to_le16(CMDQ_ADD_GID_VLAN_TPID_TPID_8100 |
+ CMDQ_ADD_GID_VLAN_VLAN_EN);
+ }
+
+ /* MAC in network format */
+ req.src_mac[0] = cpu_to_be16(((u16 *)smac)[0]);
+ req.src_mac[1] = cpu_to_be16(((u16 *)smac)[1]);
+ req.src_mac[2] = cpu_to_be16(((u16 *)smac)[2]);
+
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+ if (rc)
+ return rc;
+ sgid_tbl->hw_id[free_idx] = le32_to_cpu(resp.xid);
+ }
+
+ if (vlan_id != 0xFFFF)
+ sgid_tbl->vlan[free_idx] = true;
+
+ memcpy(&sgid_tbl->tbl[free_idx], gid, sizeof(*gid));
+ sgid_tbl->tbl[free_idx].vlan_id = vlan_id;
+ sgid_tbl->active++;
+ dev_dbg(&res->pdev->dev,
+ "QPLIB: SGID added hw_id[0x%x] = 0x%x active = 0x%x\n",
+ free_idx, sgid_tbl->hw_id[free_idx], sgid_tbl->active);
+
+ *index = free_idx;
+ /* unlock */
+ return 0;
+}
+
+/* AH */
+int bnxt_qplib_create_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah,
+ bool block)
+{
+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+ struct creq_create_ah_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct cmdq_create_ah req = {};
+ u32 temp32[4];
+ u16 temp16[3];
+ int rc;
+
+ bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_CREATE_AH,
+ sizeof(req));
+
+ memcpy(temp32, ah->dgid.data, sizeof(struct bnxt_qplib_gid));
+ req.dgid[0] = cpu_to_le32(temp32[0]);
+ req.dgid[1] = cpu_to_le32(temp32[1]);
+ req.dgid[2] = cpu_to_le32(temp32[2]);
+ req.dgid[3] = cpu_to_le32(temp32[3]);
+
+ req.type = ah->nw_type;
+ req.hop_limit = ah->hop_limit;
+ req.sgid_index = cpu_to_le16(res->sgid_tbl.hw_id[ah->sgid_index]);
+ req.dest_vlan_id_flow_label = cpu_to_le32((ah->flow_label &
+ CMDQ_CREATE_AH_FLOW_LABEL_MASK) |
+ CMDQ_CREATE_AH_DEST_VLAN_ID_MASK);
+ req.pd_id = cpu_to_le32(ah->pd->id);
+ req.traffic_class = ah->traffic_class;
+
+ /* MAC in network format */
+ memcpy(temp16, ah->dmac, ETH_ALEN);
+ req.dest_mac[0] = cpu_to_le16(temp16[0]);
+ req.dest_mac[1] = cpu_to_le16(temp16[1]);
+ req.dest_mac[2] = cpu_to_le16(temp16[2]);
+
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), block);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+ if (rc)
+ return rc;
+
+ ah->id = le32_to_cpu(resp.xid);
+ /* for Cu/Wh AHID 0 is not valid */
+ if (!_is_chip_gen_p5_p7(res->cctx) && !ah->id)
+ rc = -EINVAL;
+
+ return rc;
+}
+
+int bnxt_qplib_destroy_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah,
+ bool block)
+{
+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+ struct creq_destroy_ah_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct cmdq_destroy_ah req = {};
+ int rc;
+
+ bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DESTROY_AH,
+ sizeof(req));
+
+ req.ah_cid = cpu_to_le32(ah->id);
+
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), block);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+ return rc;
+}
+
+/* MRW */
+int bnxt_qplib_free_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw)
+{
+ struct creq_deallocate_key_resp resp = {};
+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+ struct cmdq_deallocate_key req = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ int rc;
+
+ if (mrw->lkey == 0xFFFFFFFF) {
+ dev_info(&res->pdev->dev,
+ "QPLIB: SP: Free a reserved lkey MRW\n");
+ return 0;
+ }
+
+ bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DEALLOCATE_KEY,
+ sizeof(req));
+
+ req.mrw_flags = mrw->type;
+
+ if ((mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1) ||
+ (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A) ||
+ (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B))
+ req.key = cpu_to_le32(mrw->rkey);
+ else
+ req.key = cpu_to_le32(mrw->lkey);
+
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+ if (rc)
+ return rc;
+
+ if (mrw->hwq.max_elements)
+ bnxt_qplib_free_hwq(res, &mrw->hwq);
+
+ return 0;
+}
+
+int bnxt_qplib_alloc_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw)
+{
+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+ struct creq_allocate_mrw_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct cmdq_allocate_mrw req = {};
+ int rc;
+
+ bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_ALLOCATE_MRW,
+ sizeof(req));
+
+ req.pd_id = cpu_to_le32(mrw->pd->id);
+ req.mrw_flags = mrw->type;
+ if ((mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR &&
+ mrw->flags & BNXT_QPLIB_FR_PMR) ||
+ mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A ||
+ mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B)
+ req.access = CMDQ_ALLOCATE_MRW_ACCESS_CONSUMER_OWNED_KEY;
+ req.mrw_handle = cpu_to_le64(mrw);
+
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+ if (rc)
+ return rc;
+ if ((mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1) ||
+ (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A) ||
+ (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B))
+ mrw->rkey = le32_to_cpu(resp.xid);
+ else
+ mrw->lkey = le32_to_cpu(resp.xid);
+
+ return 0;
+}
+
+int bnxt_qplib_dereg_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw,
+ bool block)
+{
+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+ struct creq_deregister_mr_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct cmdq_deregister_mr req = {};
+ int rc;
+
+ bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DEREGISTER_MR,
+ sizeof(req));
+
+ req.lkey = cpu_to_le32(mrw->lkey);
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), block);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+ if (rc)
+ return rc;
+
+ if (mrw->hwq.max_elements) {
+ mrw->va = 0;
+ mrw->total_size = 0;
+ bnxt_qplib_free_hwq(res, &mrw->hwq);
+ }
+
+ return 0;
+}
+
+int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_mrinfo *mrinfo,
+ bool block)
+{
+ struct bnxt_qplib_hwq_attr hwq_attr = {};
+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+ struct creq_register_mr_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct cmdq_register_mr req = {};
+ struct bnxt_qplib_mrw *mr;
+ u32 buf_pg_size;
+ u32 pg_size;
+ u16 level;
+ u16 flags;
+ int rc;
+
+ mr = mrinfo->mrw;
+ buf_pg_size = 0x01ULL << mrinfo->sg.pgshft;
+ if (mrinfo->sg.npages) {
+ /* Free the hwq if it already exist, must be a rereg */
+ if (mr->hwq.max_elements)
+ bnxt_qplib_free_hwq(res, &mr->hwq);
+ /* Use system PAGE_SIZE */
+ hwq_attr.res = res;
+ hwq_attr.depth = mrinfo->sg.npages;
+ hwq_attr.stride = PAGE_SIZE;
+ hwq_attr.type = HWQ_TYPE_MR;
+ hwq_attr.sginfo = &mrinfo->sg;
+ rc = bnxt_qplib_alloc_init_hwq(&mr->hwq, &hwq_attr);
+ if (rc) {
+ dev_err(&res->pdev->dev,
+ "SP: Reg MR memory allocation failed\n");
+ return -ENOMEM;
+ }
+ }
+
+ bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_REGISTER_MR,
+ sizeof(req));
+ /* Configure the request */
+ if (mrinfo->is_dma) {
+ /* No PBL provided, just use system PAGE_SIZE */
+ level = 0;
+ req.pbl = 0;
+ pg_size = PAGE_SIZE;
+ } else {
+ level = mr->hwq.level;
+ req.pbl = cpu_to_le64(mr->hwq.pbl[PBL_LVL_0].pg_map_arr[0]);
+ }
+
+ pg_size = buf_pg_size ? buf_pg_size : PAGE_SIZE;
+ req.log2_pg_size_lvl = (level << CMDQ_REGISTER_MR_LVL_SFT) |
+ ((ilog2(pg_size) <<
+ CMDQ_REGISTER_MR_LOG2_PG_SIZE_SFT) &
+ CMDQ_REGISTER_MR_LOG2_PG_SIZE_MASK);
+ req.log2_pbl_pg_size = cpu_to_le16(((ilog2(PAGE_SIZE) <<
+ CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_SFT) &
+ CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_MASK));
+ req.access = (mr->flags & 0xFFFF);
+ req.va = cpu_to_le64(mr->va);
+ req.key = cpu_to_le32(mr->lkey);
+ if (_is_alloc_mr_unified(res->dattr)) {
+ flags = 0;
+ req.key = cpu_to_le32(mr->pd->id);
+ flags |= CMDQ_REGISTER_MR_FLAGS_ALLOC_MR;
+ req.flags = cpu_to_le16(flags);
+ }
+ req.mr_size = cpu_to_le64(mr->total_size);
+
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), block);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+ if (rc)
+ goto fail;
+
+ if (_is_alloc_mr_unified(res->dattr)) {
+ mr->lkey = le32_to_cpu(resp.xid);
+ mr->rkey = mr->lkey;
+ }
+
+ return 0;
+fail:
+ if (mr->hwq.max_elements)
+ bnxt_qplib_free_hwq(res, &mr->hwq);
+ return rc;
+}
+
+int bnxt_qplib_alloc_fast_reg_page_list(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_frpl *frpl,
+ int max_pg_ptrs)
+{
+ struct bnxt_qplib_hwq_attr hwq_attr = {};
+ struct bnxt_qplib_sg_info sginfo = {};
+ int pg_ptrs, rc;
+
+ /* Re-calculate the max to fit the HWQ allocation model */
+ pg_ptrs = roundup_pow_of_two(max_pg_ptrs);
+
+ sginfo.pgsize = PAGE_SIZE;
+ sginfo.nopte = true;
+
+ hwq_attr.res = res;
+ hwq_attr.depth = pg_ptrs;
+ hwq_attr.stride = PAGE_SIZE;
+ hwq_attr.sginfo = &sginfo;
+ hwq_attr.type = HWQ_TYPE_CTX;
+ rc = bnxt_qplib_alloc_init_hwq(&frpl->hwq, &hwq_attr);
+ if (!rc)
+ frpl->max_pg_ptrs = pg_ptrs;
+
+ return rc;
+}
+
+void bnxt_qplib_free_fast_reg_page_list(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_frpl *frpl)
+{
+ bnxt_qplib_free_hwq(res, &frpl->hwq);
+}
+
+int bnxt_qplib_map_tc2cos(struct bnxt_qplib_res *res, u16 *cids)
+{
+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+ struct creq_map_tc_to_cos_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct cmdq_map_tc_to_cos req = {};
+ int rc;
+
+ bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_MAP_TC_TO_COS,
+ sizeof(req));
+ req.cos0 = cpu_to_le16(cids[0]);
+ req.cos1 = cpu_to_le16(cids[1]);
+
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+ return rc;
+}
+
+static void bnxt_qplib_fill_cc_gen1(struct cmdq_modify_roce_cc_gen1_tlv *ext_req,
+ struct bnxt_qplib_cc_param_ext *cc_ext)
+{
+ ext_req->modify_mask = cpu_to_le64(cc_ext->ext_mask);
+ cc_ext->ext_mask = 0;
+ ext_req->inactivity_th_hi = cpu_to_le16(cc_ext->inact_th_hi);
+ ext_req->min_time_between_cnps = cpu_to_le16(cc_ext->min_delta_cnp);
+ ext_req->init_cp = cpu_to_le16(cc_ext->init_cp);
+ ext_req->tr_update_mode = cc_ext->tr_update_mode;
+ ext_req->tr_update_cycles = cc_ext->tr_update_cyls;
+ ext_req->fr_num_rtts = cc_ext->fr_rtt;
+ ext_req->ai_rate_increase = cc_ext->ai_rate_incr;
+ ext_req->reduction_relax_rtts_th = cpu_to_le16(cc_ext->rr_rtt_th);
+ ext_req->additional_relax_cr_th = cpu_to_le16(cc_ext->ar_cr_th);
+ ext_req->cr_min_th = cpu_to_le16(cc_ext->cr_min_th);
+ ext_req->bw_avg_weight = cc_ext->bw_avg_weight;
+ ext_req->actual_cr_factor = cc_ext->cr_factor;
+ ext_req->max_cp_cr_th = cpu_to_le16(cc_ext->cr_th_max_cp);
+ ext_req->cp_bias_en = cc_ext->cp_bias_en;
+ ext_req->cp_bias = cc_ext->cp_bias;
+ ext_req->cnp_ecn = cc_ext->cnp_ecn;
+ ext_req->rtt_jitter_en = cc_ext->rtt_jitter_en;
+ ext_req->link_bytes_per_usec = cpu_to_le16(cc_ext->bytes_per_usec);
+ ext_req->reset_cc_cr_th = cpu_to_le16(cc_ext->cc_cr_reset_th);
+ ext_req->cr_width = cc_ext->cr_width;
+ ext_req->quota_period_min = cc_ext->min_quota;
+ ext_req->quota_period_max = cc_ext->max_quota;
+ ext_req->quota_period_abs_max = cc_ext->abs_max_quota;
+ ext_req->tr_lower_bound = cpu_to_le16(cc_ext->tr_lb);
+ ext_req->cr_prob_factor = cc_ext->cr_prob_fac;
+ ext_req->tr_prob_factor = cc_ext->tr_prob_fac;
+ ext_req->fairness_cr_th = cpu_to_le16(cc_ext->fair_cr_th);
+ ext_req->red_div = cc_ext->red_div;
+ ext_req->cnp_ratio_th = cc_ext->cnp_ratio_th;
+ ext_req->exp_ai_rtts = cpu_to_le16(cc_ext->ai_ext_rtt);
+ ext_req->exp_ai_cr_cp_ratio = cc_ext->exp_crcp_ratio;
+ ext_req->use_rate_table = cc_ext->low_rate_en;
+ ext_req->cp_exp_update_th = cpu_to_le16(cc_ext->cpcr_update_th);
+ ext_req->high_exp_ai_rtts_th1 = cpu_to_le16(cc_ext->ai_rtt_th1);
+ ext_req->high_exp_ai_rtts_th2 = cpu_to_le16(cc_ext->ai_rtt_th2);
+ ext_req->actual_cr_cong_free_rtts_th = cpu_to_le16(cc_ext->cf_rtt_th);
+ ext_req->severe_cong_cr_th1 = cpu_to_le16(cc_ext->sc_cr_th1);
+ ext_req->severe_cong_cr_th2 = cpu_to_le16(cc_ext->sc_cr_th2);
+ ext_req->link64B_per_rtt = cpu_to_le32(cc_ext->l64B_per_rtt);
+ ext_req->cc_ack_bytes = cc_ext->cc_ack_bytes;
+ ext_req->reduce_init_cong_free_rtts_th = cpu_to_le16(cc_ext->reduce_cf_rtt_th);
+}
+
+int bnxt_qplib_modify_cc(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_cc_param *cc_param)
+{
+ struct bnxt_qplib_tlv_modify_cc_req tlv_req = {};
+ struct creq_modify_roce_cc_resp resp = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct cmdq_modify_roce_cc *req;
+ int req_size;
+ void *cmd;
+ int rc;
+
+ /* Prepare the older base command */
+ req = &tlv_req.base_req;
+ cmd = req;
+ req_size = sizeof(*req);
+ bnxt_qplib_rcfw_cmd_prep(req, CMDQ_BASE_OPCODE_MODIFY_ROCE_CC,
+ sizeof(*req));
+ req->modify_mask = cpu_to_le32(cc_param->mask);
+ req->enable_cc = cc_param->enable;
+ req->g = cc_param->g;
+ req->num_phases_per_state = cc_param->nph_per_state;
+ req->time_per_phase = cc_param->time_pph;
+ req->pkts_per_phase = cc_param->pkts_pph;
+ req->init_cr = cpu_to_le16(cc_param->init_cr);
+ req->init_tr = cpu_to_le16(cc_param->init_tr);
+ req->tos_dscp_tos_ecn = (cc_param->tos_dscp <<
+ CMDQ_MODIFY_ROCE_CC_TOS_DSCP_SFT) |
+ (cc_param->tos_ecn &
+ CMDQ_MODIFY_ROCE_CC_TOS_ECN_MASK);
+ req->alt_vlan_pcp = cc_param->alt_vlan_pcp;
+ req->alt_tos_dscp = cpu_to_le16(cc_param->alt_tos_dscp);
+ req->rtt = cpu_to_le16(cc_param->rtt);
+ req->tcp_cp = cpu_to_le16(cc_param->tcp_cp);
+ req->cc_mode = cc_param->cc_mode;
+ req->inactivity_th = cpu_to_le16(cc_param->inact_th);
+
+ /* For chip gen P5 onwards fill extended cmd and header */
+ if (_is_chip_gen_p5_p7(res->cctx)) {
+ struct roce_tlv *hdr;
+ u32 payload;
+ u32 chunks;
+
+ cmd = &tlv_req;
+ req_size = sizeof(tlv_req);
+ /* Prepare primary tlv header */
+ hdr = &tlv_req.tlv_hdr;
+ chunks = CHUNKS(sizeof(struct bnxt_qplib_tlv_modify_cc_req));
+ payload = sizeof(struct cmdq_modify_roce_cc);
+ ROCE_1ST_TLV_PREP(hdr, chunks, payload, true);
+ /* Prepare secondary tlv header */
+ hdr = (struct roce_tlv *)&tlv_req.ext_req;
+ payload = sizeof(struct cmdq_modify_roce_cc_gen1_tlv) -
+ sizeof(struct roce_tlv);
+ ROCE_EXT_TLV_PREP(hdr, TLV_TYPE_MODIFY_ROCE_CC_GEN1, payload,
+ false, true);
+ bnxt_qplib_fill_cc_gen1(&tlv_req.ext_req, &cc_param->cc_ext);
+ }
+
+ bnxt_qplib_fill_cmdqmsg(&msg, cmd, &resp, NULL, req_size,
+ sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(res->rcfw, &msg);
+ return rc;
+}
+
+static void bnxt_qplib_read_cc_gen1(struct bnxt_qplib_cc_param_ext *cc_ext,
+ struct creq_query_roce_cc_gen1_resp_sb_tlv *sb)
+{
+ cc_ext->inact_th_hi = le16_to_cpu(sb->inactivity_th_hi);
+ cc_ext->min_delta_cnp = le16_to_cpu(sb->min_time_between_cnps);
+ cc_ext->init_cp = le16_to_cpu(sb->init_cp);
+ cc_ext->tr_update_mode = sb->tr_update_mode;
+ cc_ext->tr_update_cyls = sb->tr_update_cycles;
+ cc_ext->fr_rtt = sb->fr_num_rtts;
+ cc_ext->ai_rate_incr = sb->ai_rate_increase;
+ cc_ext->rr_rtt_th = le16_to_cpu(sb->reduction_relax_rtts_th);
+ cc_ext->ar_cr_th = le16_to_cpu(sb->additional_relax_cr_th);
+ cc_ext->cr_min_th = le16_to_cpu(sb->cr_min_th);
+ cc_ext->bw_avg_weight = sb->bw_avg_weight;
+ cc_ext->cr_factor = sb->actual_cr_factor;
+ cc_ext->cr_th_max_cp = le16_to_cpu(sb->max_cp_cr_th);
+ cc_ext->cp_bias_en = sb->cp_bias_en;
+ cc_ext->cp_bias = sb->cp_bias;
+ cc_ext->cnp_ecn = sb->cnp_ecn;
+ cc_ext->rtt_jitter_en = sb->rtt_jitter_en;
+ cc_ext->bytes_per_usec = le16_to_cpu(sb->link_bytes_per_usec);
+ cc_ext->cc_cr_reset_th = le16_to_cpu(sb->reset_cc_cr_th);
+ cc_ext->cr_width = sb->cr_width;
+ cc_ext->min_quota = sb->quota_period_min;
+ cc_ext->max_quota = sb->quota_period_max;
+ cc_ext->abs_max_quota = sb->quota_period_abs_max;
+ cc_ext->tr_lb = le16_to_cpu(sb->tr_lower_bound);
+ cc_ext->cr_prob_fac = sb->cr_prob_factor;
+ cc_ext->tr_prob_fac = sb->tr_prob_factor;
+ cc_ext->fair_cr_th = le16_to_cpu(sb->fairness_cr_th);
+ cc_ext->red_div = sb->red_div;
+ cc_ext->cnp_ratio_th = sb->cnp_ratio_th;
+ cc_ext->ai_ext_rtt = le16_to_cpu(sb->exp_ai_rtts);
+ cc_ext->exp_crcp_ratio = sb->exp_ai_cr_cp_ratio;
+ cc_ext->low_rate_en = sb->use_rate_table;
+ cc_ext->cpcr_update_th = le16_to_cpu(sb->cp_exp_update_th);
+ cc_ext->ai_rtt_th1 = le16_to_cpu(sb->high_exp_ai_rtts_th1);
+ cc_ext->ai_rtt_th2 = le16_to_cpu(sb->high_exp_ai_rtts_th2);
+ cc_ext->cf_rtt_th = le16_to_cpu(sb->actual_cr_cong_free_rtts_th);
+ cc_ext->sc_cr_th1 = le16_to_cpu(sb->severe_cong_cr_th1);
+ cc_ext->sc_cr_th2 = le16_to_cpu(sb->severe_cong_cr_th2);
+ cc_ext->l64B_per_rtt = le32_to_cpu(sb->link64B_per_rtt);
+ cc_ext->cc_ack_bytes = sb->cc_ack_bytes;
+ cc_ext->reduce_cf_rtt_th = le16_to_cpu(sb->reduce_init_cong_free_rtts_th);
+}
+
+int bnxt_qplib_query_cc_param(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_cc_param *cc_param)
+{
+ struct creq_query_roce_cc_gen1_resp_sb_tlv *gen1_sb;
+ struct bnxt_qplib_tlv_query_rcc_sb *ext_sb;
+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+ struct creq_query_roce_cc_resp resp = {};
+ struct creq_query_roce_cc_resp_sb *sb;
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct cmdq_query_roce_cc req = {};
+ struct bnxt_qplib_rcfw_sbuf sbuf;
+ size_t resp_size;
+ int rc;
+
+ /* Query the parameters from chip */
+ bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_QUERY_ROCE_CC,
+ sizeof(req));
+ if (_is_chip_gen_p5_p7(res->cctx))
+ resp_size = sizeof(*ext_sb);
+ else
+ resp_size = sizeof(*sb);
+ sbuf.size = ALIGN(resp_size, BNXT_QPLIB_CMDQE_UNITS);
+ sbuf.sb = dma_zalloc_coherent(&rcfw->pdev->dev, sbuf.size,
+ &sbuf.dma_addr, GFP_KERNEL);
+ if (!sbuf.sb)
+ return -ENOMEM;
+
+ req.resp_size = sbuf.size / BNXT_QPLIB_CMDQE_UNITS;
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req),
+ sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(res->rcfw, &msg);
+ if (rc) {
+ dev_dbg(&res->pdev->dev, "%s:Query CC param failed:0x%x\n",
+ __func__, rc);
+ goto out;
+ }
+
+ ext_sb = sbuf.sb;
+ gen1_sb = &ext_sb->gen1_sb;
+ sb = _is_chip_gen_p5_p7(res->cctx) ? &ext_sb->base_sb :
+ (struct creq_query_roce_cc_resp_sb *)ext_sb;
+
+ cc_param->enable = sb->enable_cc & CREQ_QUERY_ROCE_CC_RESP_SB_ENABLE_CC;
+ cc_param->tos_ecn = (sb->tos_dscp_tos_ecn &
+ CREQ_QUERY_ROCE_CC_RESP_SB_TOS_ECN_MASK) >>
+ CREQ_QUERY_ROCE_CC_RESP_SB_TOS_ECN_SFT;
+ cc_param->tos_dscp = (sb->tos_dscp_tos_ecn &
+ CREQ_QUERY_ROCE_CC_RESP_SB_TOS_DSCP_MASK) >>
+ CREQ_QUERY_ROCE_CC_RESP_SB_TOS_DSCP_SFT;
+ cc_param->alt_tos_dscp = sb->alt_tos_dscp;
+ cc_param->alt_vlan_pcp = sb->alt_vlan_pcp;
+
+ cc_param->g = sb->g;
+ cc_param->nph_per_state = sb->num_phases_per_state;
+ cc_param->init_cr = le16_to_cpu(sb->init_cr);
+ cc_param->init_tr = le16_to_cpu(sb->init_tr);
+ cc_param->cc_mode = sb->cc_mode;
+ cc_param->inact_th = le16_to_cpu(sb->inactivity_th);
+ cc_param->rtt = le16_to_cpu(sb->rtt);
+ cc_param->tcp_cp = le16_to_cpu(sb->tcp_cp);
+ cc_param->time_pph = sb->time_per_phase;
+ cc_param->pkts_pph = sb->pkts_per_phase;
+ if (_is_chip_gen_p5_p7(res->cctx))
+ bnxt_qplib_read_cc_gen1(&cc_param->cc_ext, gen1_sb);
+out:
+ dma_free_coherent(&rcfw->pdev->dev, sbuf.size,
+ sbuf.sb, sbuf.dma_addr);
+ return rc;
+}
+
+
+int bnxt_qplib_get_roce_error_stats(struct bnxt_qplib_rcfw *rcfw,
+ struct bnxt_qplib_roce_stats *stats,
+ struct bnxt_qplib_query_stats_info *sinfo)
+{
+ struct creq_query_roce_stats_resp resp = {};
+ struct creq_query_roce_stats_resp_sb *sb;
+ struct cmdq_query_roce_stats req = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct bnxt_qplib_rcfw_sbuf sbuf;
+ u16 cmd_flags = 0;
+ u32 fn_id = 0;
+ int rc = 0;
+
+ bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_QUERY_ROCE_STATS,
+ sizeof(req));
+
+ sbuf.size = sizeof(*sb);
+ sbuf.sb = dma_zalloc_coherent(&rcfw->pdev->dev, sbuf.size,
+ &sbuf.dma_addr, GFP_KERNEL);
+ if (!sbuf.sb)
+ return -ENOMEM;
+ sb = sbuf.sb;
+
+ if (rcfw->res->cctx->hwrm_intf_ver >= HWRM_VERSION_ROCE_STATS_FN_ID) {
+ if (sinfo->function_id != 0xFFFFFFFF) {
+ cmd_flags = CMDQ_QUERY_ROCE_STATS_FLAGS_FUNCTION_ID;
+ if (sinfo->vf_valid) {
+ fn_id = CMDQ_QUERY_ROCE_STATS_VF_VALID;
+ fn_id |= (sinfo->function_id <<
+ CMDQ_QUERY_ROCE_STATS_VF_NUM_SFT) &
+ CMDQ_QUERY_ROCE_STATS_VF_NUM_MASK;
+ } else {
+ fn_id = sinfo->function_id &
+ CMDQ_QUERY_ROCE_STATS_PF_NUM_MASK;
+ }
+ }
+
+ req.flags = cpu_to_le16(cmd_flags);
+ req.function_id = cpu_to_le32(fn_id);
+
+ if (sinfo->collection_id != 0xFF) {
+ cmd_flags |= CMDQ_QUERY_ROCE_STATS_FLAGS_COLLECTION_ID;
+ req.collection_id = sinfo->collection_id;
+ }
+ } else {
+ /* For older HWRM version, the command length has to be
+ * adjusted. 8 bytes are more in the newer command.
+ * So subtract these 8 bytes for older HWRM version.
+ * command units are adjusted inside
+ * bnxt_qplib_rcfw_send_message.
+ */
+ req.cmd_size -= 8;
+ }
+
+ req.resp_size = sbuf.size / BNXT_QPLIB_CMDQE_UNITS;
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req),
+ sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+ if (rc)
+ goto bail;
+ /* Extract the context from the side buffer */
+ stats->to_retransmits = le64_to_cpu(sb->to_retransmits);
+ stats->seq_err_naks_rcvd = le64_to_cpu(sb->seq_err_naks_rcvd);
+ stats->max_retry_exceeded = le64_to_cpu(sb->max_retry_exceeded);
+ stats->rnr_naks_rcvd = le64_to_cpu(sb->rnr_naks_rcvd);
+ stats->missing_resp = le64_to_cpu(sb->missing_resp);
+ stats->unrecoverable_err = le64_to_cpu(sb->unrecoverable_err);
+ stats->bad_resp_err = le64_to_cpu(sb->bad_resp_err);
+ stats->local_qp_op_err = le64_to_cpu(sb->local_qp_op_err);
+ stats->local_protection_err = le64_to_cpu(sb->local_protection_err);
+ stats->mem_mgmt_op_err = le64_to_cpu(sb->mem_mgmt_op_err);
+ stats->remote_invalid_req_err = le64_to_cpu(sb->remote_invalid_req_err);
+ stats->remote_access_err = le64_to_cpu(sb->remote_access_err);
+ stats->remote_op_err = le64_to_cpu(sb->remote_op_err);
+ stats->dup_req = le64_to_cpu(sb->dup_req);
+ stats->res_exceed_max = le64_to_cpu(sb->res_exceed_max);
+ stats->res_length_mismatch = le64_to_cpu(sb->res_length_mismatch);
+ stats->res_exceeds_wqe = le64_to_cpu(sb->res_exceeds_wqe);
+ stats->res_opcode_err = le64_to_cpu(sb->res_opcode_err);
+ stats->res_rx_invalid_rkey = le64_to_cpu(sb->res_rx_invalid_rkey);
+ stats->res_rx_domain_err = le64_to_cpu(sb->res_rx_domain_err);
+ stats->res_rx_no_perm = le64_to_cpu(sb->res_rx_no_perm);
+ stats->res_rx_range_err = le64_to_cpu(sb->res_rx_range_err);
+ stats->res_tx_invalid_rkey = le64_to_cpu(sb->res_tx_invalid_rkey);
+ stats->res_tx_domain_err = le64_to_cpu(sb->res_tx_domain_err);
+ stats->res_tx_no_perm = le64_to_cpu(sb->res_tx_no_perm);
+ stats->res_tx_range_err = le64_to_cpu(sb->res_tx_range_err);
+ stats->res_irrq_oflow = le64_to_cpu(sb->res_irrq_oflow);
+ stats->res_unsup_opcode = le64_to_cpu(sb->res_unsup_opcode);
+ stats->res_unaligned_atomic = le64_to_cpu(sb->res_unaligned_atomic);
+ stats->res_rem_inv_err = le64_to_cpu(sb->res_rem_inv_err);
+ stats->res_mem_error = le64_to_cpu(sb->res_mem_error);
+ stats->res_srq_err = le64_to_cpu(sb->res_srq_err);
+ stats->res_cmp_err = le64_to_cpu(sb->res_cmp_err);
+ stats->res_invalid_dup_rkey = le64_to_cpu(sb->res_invalid_dup_rkey);
+ stats->res_wqe_format_err = le64_to_cpu(sb->res_wqe_format_err);
+ stats->res_cq_load_err = le64_to_cpu(sb->res_cq_load_err);
+ stats->res_srq_load_err = le64_to_cpu(sb->res_srq_load_err);
+ stats->res_tx_pci_err = le64_to_cpu(sb->res_tx_pci_err);
+ stats->res_rx_pci_err = le64_to_cpu(sb->res_rx_pci_err);
+
+ if (!rcfw->init_oos_stats) {
+ rcfw->oos_prev = le64_to_cpu(sb->res_oos_drop_count);
+ rcfw->init_oos_stats = true;
+ } else {
+ stats->res_oos_drop_count += (le64_to_cpu(sb->res_oos_drop_count) -
+ rcfw->oos_prev) &
+ BNXT_QPLIB_OOS_COUNT_MASK;
+ rcfw->oos_prev = le64_to_cpu(sb->res_oos_drop_count);
+ }
+
+ stats->active_qp_count_p0 = le64_to_cpu(sb->active_qp_count_p0);
+ stats->active_qp_count_p1 = le64_to_cpu(sb->active_qp_count_p1);
+ stats->active_qp_count_p2 = le64_to_cpu(sb->active_qp_count_p2);
+ stats->active_qp_count_p3 = le64_to_cpu(sb->active_qp_count_p3);
+bail:
+ dma_free_coherent(&rcfw->pdev->dev, sbuf.size,
+ sbuf.sb, sbuf.dma_addr);
+ return rc;
+}
+
+int bnxt_qplib_set_link_aggr_mode(struct bnxt_qplib_res *res,
+ u8 aggr_mode, u8 member_port_map,
+ u8 active_port_map, bool aggr_en,
+ u32 stats_fw_id)
+{
+ struct creq_set_link_aggr_mode_resources_resp resp = {};
+ struct cmdq_set_link_aggr_mode_cc req = {};
+ struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+ struct bnxt_qplib_cmdqmsg msg = {};
+ int rc = 0;
+
+ bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_SET_LINK_AGGR_MODE,
+ sizeof(req));
+
+ req.aggr_enable = aggr_en;
+ req.active_port_map = active_port_map;
+ req.member_port_map = member_port_map;
+ req.link_aggr_mode = aggr_mode;
+
+ /* need to specify only second port stats ctx id for now */
+ req.stat_ctx_id[1] = cpu_to_le16((u16)(stats_fw_id));
+
+ req.modify_mask =
+ cpu_to_le32(CMDQ_SET_LINK_AGGR_MODE_MODIFY_MASK_AGGR_EN |
+ CMDQ_SET_LINK_AGGR_MODE_MODIFY_MASK_ACTIVE_PORT_MAP |
+ CMDQ_SET_LINK_AGGR_MODE_MODIFY_MASK_MEMBER_PORT_MAP |
+ CMDQ_SET_LINK_AGGR_MODE_MODIFY_MASK_AGGR_MODE |
+ CMDQ_SET_LINK_AGGR_MODE_MODIFY_MASK_STAT_CTX_ID);
+
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+ sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+ if (rc)
+ dev_err(&res->pdev->dev,
+ "QPLIB: Failed to set link aggr mode, %#x\n", rc);
+
+ return rc;
+}
+
+int bnxt_qplib_qext_stat(struct bnxt_qplib_rcfw *rcfw, u32 fid,
+ struct bnxt_qplib_ext_stat *estat,
+ struct bnxt_qplib_query_stats_info *sinfo)
+{
+ struct creq_query_roce_stats_ext_resp resp = {};
+ struct creq_query_roce_stats_ext_resp_sb *sb;
+ struct cmdq_query_roce_stats_ext req = {};
+ struct bnxt_qplib_cmdqmsg msg = {};
+ struct bnxt_qplib_rcfw_sbuf sbuf;
+ int rc;
+
+ sbuf.size = sizeof(*sb);
+ sbuf.sb = dma_zalloc_coherent(&rcfw->pdev->dev, sbuf.size,
+ &sbuf.dma_addr, GFP_KERNEL);
+ if (!sbuf.sb) {
+ dev_err(&rcfw->pdev->dev,
+ "QPLIB: SP: QUERY_ROCE_STATS_EXT alloc sb failed\n");
+ return -ENOMEM;
+ }
+ sb = sbuf.sb;
+
+ bnxt_qplib_rcfw_cmd_prep(&req,
+ CMDQ_QUERY_ROCE_STATS_EXT_OPCODE_QUERY_ROCE_STATS,
+ sizeof(req));
+ req.resp_size = sbuf.size;
+ req.resp_addr = cpu_to_le64(sbuf.dma_addr);
+ req.flags = cpu_to_le16(CMDQ_QUERY_ROCE_STATS_EXT_FLAGS_FUNCTION_ID);
+ if (_is_chip_p7(rcfw->res->cctx) && rcfw->res->is_vf) {
+ if (sinfo->vf_valid)
+ req.function_id =
+ cpu_to_le32(CMDQ_QUERY_ROCE_STATS_EXT_VF_VALID |
+ (fid << CMDQ_QUERY_ROCE_STATS_EXT_VF_NUM_SFT));
+ else
+ req.flags = cpu_to_le16(0);
+ } else {
+ req.function_id = cpu_to_le32(fid);
+ }
+
+ bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req),
+ sizeof(resp), 0);
+ rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+ if (rc)
+ goto bail;
+
+ /* dump when dyndbg is enabled */
+ print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, sb, sizeof(*sb));
+ estat->tx_atomic_req = le64_to_cpu(sb->tx_atomic_req_pkts);
+ estat->tx_read_req = le64_to_cpu(sb->tx_read_req_pkts);
+ estat->tx_read_res = le64_to_cpu(sb->tx_read_res_pkts);
+ estat->tx_write_req = le64_to_cpu(sb->tx_write_req_pkts);
+ estat->tx_send_req = le64_to_cpu(sb->tx_send_req_pkts);
+ estat->tx_roce_pkts = le64_to_cpu(sb->tx_roce_pkts);
+ estat->tx_roce_bytes = le64_to_cpu(sb->tx_roce_bytes);
+ estat->rx_atomic_req = le64_to_cpu(sb->rx_atomic_req_pkts);
+ estat->rx_read_req = le64_to_cpu(sb->rx_read_req_pkts);
+ estat->rx_read_res = le64_to_cpu(sb->rx_read_res_pkts);
+ estat->rx_write_req = le64_to_cpu(sb->rx_write_req_pkts);
+ estat->rx_send_req = le64_to_cpu(sb->rx_send_req_pkts);
+ estat->rx_roce_pkts = le64_to_cpu(sb->rx_roce_pkts);
+ estat->rx_roce_bytes = le64_to_cpu(sb->rx_roce_bytes);
+ estat->rx_roce_good_pkts = le64_to_cpu(sb->rx_roce_good_pkts);
+ estat->rx_roce_good_bytes = le64_to_cpu(sb->rx_roce_good_bytes);
+ estat->rx_out_of_buffer = le64_to_cpu(sb->rx_out_of_buffer_pkts);
+ estat->rx_out_of_sequence = le64_to_cpu(sb->rx_out_of_sequence_pkts);
+ estat->tx_cnp = le64_to_cpu(sb->tx_cnp_pkts);
+ estat->rx_cnp = le64_to_cpu(sb->rx_cnp_pkts);
+ estat->rx_ecn_marked = le64_to_cpu(sb->rx_ecn_marked_pkts);
+ estat->seq_err_naks_rcvd = le64_to_cpu(sb->seq_err_naks_rcvd);
+ estat->rnr_naks_rcvd = le64_to_cpu(sb->rnr_naks_rcvd);
+ estat->missing_resp = le64_to_cpu(sb->missing_resp);
+ estat->to_retransmits = le64_to_cpu(sb->to_retransmit);
+ estat->dup_req = le64_to_cpu(sb->dup_req);
+ estat->rx_dcn_payload_cut = le64_to_cpu(sb->rx_dcn_payload_cut);
+ estat->te_bypassed = le64_to_cpu(sb->te_bypassed);
+bail:
+ dma_free_coherent(&rcfw->pdev->dev, sbuf.size,
+ sbuf.sb, sbuf.dma_addr);
+ return rc;
+}
diff --git a/sys/dev/bnxt/bnxt_re/qplib_sp.h b/sys/dev/bnxt/bnxt_re/qplib_sp.h
new file mode 100644
index 000000000000..e306db3b9d8e
--- /dev/null
+++ b/sys/dev/bnxt/bnxt_re/qplib_sp.h
@@ -0,0 +1,432 @@
+/*
+ * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term
+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Description: Slow Path Operators (header)
+ */
+
+#ifndef __BNXT_QPLIB_SP_H__
+#define __BNXT_QPLIB_SP_H__
+
+#include <rdma/ib_verbs.h>
+
+#define BNXT_QPLIB_RESERVED_QP_WRS 128
+
+/* Resource maximums reported by the firmware */
+struct bnxt_qplib_dev_attr {
+#define FW_VER_ARR_LEN 4
+ u8 fw_ver[FW_VER_ARR_LEN];
+ u16 max_sgid;
+ u16 max_mrw;
+ u32 max_qp;
+#define BNXT_QPLIB_MAX_OUT_RD_ATOM 126
+ u32 max_qp_rd_atom;
+ u32 max_qp_init_rd_atom;
+ u32 max_qp_wqes;
+ u32 max_qp_sges;
+ u32 max_cq;
+ /* HW supports only 8K entries in PBL.
+ * So max CQEs that can be supported per CQ is 1M.
+ */
+#define BNXT_QPLIB_MAX_CQ_WQES 0xfffff
+ u32 max_cq_wqes;
+ u32 max_cq_sges;
+ u32 max_mr;
+ u64 max_mr_size;
+#define BNXT_QPLIB_MAX_PD (64 * 1024)
+ u32 max_pd;
+ u32 max_mw;
+ u32 max_raw_ethy_qp;
+ u32 max_ah;
+ u32 max_fmr;
+ u32 max_map_per_fmr;
+ u32 max_srq;
+ u32 max_srq_wqes;
+ u32 max_srq_sges;
+ u32 max_pkey;
+ u32 max_inline_data;
+ u32 l2_db_size;
+ u8 tqm_alloc_reqs[MAX_TQM_ALLOC_REQ];
+ u8 is_atomic;
+ u16 dev_cap_flags;
+ u64 page_size_cap;
+ u32 max_dpi;
+};
+
+struct bnxt_qplib_pd {
+ u32 id;
+};
+
+struct bnxt_qplib_gid {
+ u8 data[16];
+};
+
+struct bnxt_qplib_gid_info {
+ struct bnxt_qplib_gid gid;
+ u16 vlan_id;
+};
+
+struct bnxt_qplib_ah {
+ struct bnxt_qplib_gid dgid;
+ struct bnxt_qplib_pd *pd;
+ u32 id;
+ u8 sgid_index;
+ u8 host_sgid_index; /* For Query AH if the hw table and SW table are differnt */
+ u8 traffic_class;
+ u32 flow_label;
+ u8 hop_limit;
+ u8 sl;
+ u8 dmac[6];
+ u16 vlan_id;
+ u8 nw_type;
+ u8 enable_cc;
+};
+
+struct bnxt_qplib_mrw {
+ struct bnxt_qplib_pd *pd;
+ int type;
+ u32 flags;
+#define BNXT_QPLIB_FR_PMR 0x80000000
+ u32 lkey;
+ u32 rkey;
+#define BNXT_QPLIB_RSVD_LKEY 0xFFFFFFFF
+ u64 va;
+ u64 total_size;
+ u32 npages;
+ u64 mr_handle;
+ struct bnxt_qplib_hwq hwq;
+};
+
+struct bnxt_qplib_mrinfo {
+ struct bnxt_qplib_mrw *mrw;
+ struct bnxt_qplib_sg_info sg;
+ u64 *ptes;
+ bool is_dma;
+};
+
+struct bnxt_qplib_frpl {
+ int max_pg_ptrs;
+ struct bnxt_qplib_hwq hwq;
+};
+
+struct bnxt_qplib_cc_param_ext {
+ u64 ext_mask;
+ u16 inact_th_hi;
+ u16 min_delta_cnp;
+ u16 init_cp;
+ u8 tr_update_mode;
+ u8 tr_update_cyls;
+ u8 fr_rtt;
+ u8 ai_rate_incr;
+ u16 rr_rtt_th;
+ u16 ar_cr_th;
+ u16 cr_min_th;
+ u8 bw_avg_weight;
+ u8 cr_factor;
+ u16 cr_th_max_cp;
+ u8 cp_bias_en;
+ u8 cp_bias;
+ u8 cnp_ecn;
+ u8 rtt_jitter_en;
+ u16 bytes_per_usec;
+ u16 cc_cr_reset_th;
+ u8 cr_width;
+ u8 min_quota;
+ u8 max_quota;
+ u8 abs_max_quota;
+ u16 tr_lb;
+ u8 cr_prob_fac;
+ u8 tr_prob_fac;
+ u16 fair_cr_th;
+ u8 red_div;
+ u8 cnp_ratio_th;
+ u16 ai_ext_rtt;
+ u8 exp_crcp_ratio;
+ u8 low_rate_en;
+ u16 cpcr_update_th;
+ u16 ai_rtt_th1;
+ u16 ai_rtt_th2;
+ u16 cf_rtt_th;
+ u16 sc_cr_th1; /* severe congestion cr threshold 1 */
+ u16 sc_cr_th2; /* severe congestion cr threshold 2 */
+ u32 l64B_per_rtt;
+ u8 cc_ack_bytes;
+ u16 reduce_cf_rtt_th;
+};
+
+struct bnxt_qplib_cc_param {
+ u8 alt_vlan_pcp;
+ u16 alt_tos_dscp;
+#define BNXT_QPLIB_USER_DSCP_VALID 0x80
+ u8 cnp_dscp_user;
+ u8 roce_dscp_user;
+ u8 cc_mode;
+ u8 enable;
+ u16 inact_th;
+ u16 init_cr;
+ u16 init_tr;
+ u16 rtt;
+ u8 g;
+ u8 nph_per_state;
+ u8 time_pph;
+ u8 pkts_pph;
+ u8 tos_ecn;
+ u8 tos_dscp;
+ u8 qp1_tos_dscp;
+ u16 tcp_cp;
+ struct bnxt_qplib_cc_param_ext cc_ext;
+ u8 disable_prio_vlan_tx;
+ /* Mask used while programming the configfs values */
+ u32 mask;
+ /* Mask used while displaying the configfs values */
+ u32 cur_mask;
+ u8 roce_pri;
+#define BNXT_QPLIB_CC_PARAM_MASK_VLAN_TX_DISABLE 0x40000
+#define BNXT_QPLIB_CC_PARAM_MASK_ROCE_PRI 0x80000
+ /* prev value to clear dscp table */
+ u8 prev_roce_pri;
+ u8 prev_alt_vlan_pcp;
+ u8 prev_tos_dscp;
+ u16 prev_alt_tos_dscp;
+ /* To track if admin has enabled ECN explicitly */
+ u8 admin_enable;
+};
+
+struct bnxt_qplib_roce_stats {
+ u64 to_retransmits;
+ u64 seq_err_naks_rcvd;
+ /* seq_err_naks_rcvd is 64 b */
+ u64 max_retry_exceeded;
+ /* max_retry_exceeded is 64 b */
+ u64 rnr_naks_rcvd;
+ /* rnr_naks_rcvd is 64 b */
+ u64 missing_resp;
+ u64 unrecoverable_err;
+ /* unrecoverable_err is 64 b */
+ u64 bad_resp_err;
+ /* bad_resp_err is 64 b */
+ u64 local_qp_op_err;
+ /* local_qp_op_err is 64 b */
+ u64 local_protection_err;
+ /* local_protection_err is 64 b */
+ u64 mem_mgmt_op_err;
+ /* mem_mgmt_op_err is 64 b */
+ u64 remote_invalid_req_err;
+ /* remote_invalid_req_err is 64 b */
+ u64 remote_access_err;
+ /* remote_access_err is 64 b */
+ u64 remote_op_err;
+ /* remote_op_err is 64 b */
+ u64 dup_req;
+ /* dup_req is 64 b */
+ u64 res_exceed_max;
+ /* res_exceed_max is 64 b */
+ u64 res_length_mismatch;
+ /* res_length_mismatch is 64 b */
+ u64 res_exceeds_wqe;
+ /* res_exceeds_wqe is 64 b */
+ u64 res_opcode_err;
+ /* res_opcode_err is 64 b */
+ u64 res_rx_invalid_rkey;
+ /* res_rx_invalid_rkey is 64 b */
+ u64 res_rx_domain_err;
+ /* res_rx_domain_err is 64 b */
+ u64 res_rx_no_perm;
+ /* res_rx_no_perm is 64 b */
+ u64 res_rx_range_err;
+ /* res_rx_range_err is 64 b */
+ u64 res_tx_invalid_rkey;
+ /* res_tx_invalid_rkey is 64 b */
+ u64 res_tx_domain_err;
+ /* res_tx_domain_err is 64 b */
+ u64 res_tx_no_perm;
+ /* res_tx_no_perm is 64 b */
+ u64 res_tx_range_err;
+ /* res_tx_range_err is 64 b */
+ u64 res_irrq_oflow;
+ /* res_irrq_oflow is 64 b */
+ u64 res_unsup_opcode;
+ /* res_unsup_opcode is 64 b */
+ u64 res_unaligned_atomic;
+ /* res_unaligned_atomic is 64 b */
+ u64 res_rem_inv_err;
+ /* res_rem_inv_err is 64 b */
+ u64 res_mem_error;
+ /* res_mem_error is 64 b */
+ u64 res_srq_err;
+ /* res_srq_err is 64 b */
+ u64 res_cmp_err;
+ /* res_cmp_err is 64 b */
+ u64 res_invalid_dup_rkey;
+ /* res_invalid_dup_rkey is 64 b */
+ u64 res_wqe_format_err;
+ /* res_wqe_format_err is 64 b */
+ u64 res_cq_load_err;
+ /* res_cq_load_err is 64 b */
+ u64 res_srq_load_err;
+ /* res_srq_load_err is 64 b */
+ u64 res_tx_pci_err;
+ /* res_tx_pci_err is 64 b */
+ u64 res_rx_pci_err;
+ /* res_rx_pci_err is 64 b */
+ u64 res_oos_drop_count;
+ /* res_oos_drop_count */
+ u64 active_qp_count_p0;
+ /* port 0 active qps */
+ u64 active_qp_count_p1;
+ /* port 1 active qps */
+ u64 active_qp_count_p2;
+ /* port 2 active qps */
+ u64 active_qp_count_p3;
+ /* port 3 active qps */
+};
+
+struct bnxt_qplib_ext_stat {
+ u64 tx_atomic_req;
+ u64 tx_read_req;
+ u64 tx_read_res;
+ u64 tx_write_req;
+ u64 tx_send_req;
+ u64 tx_roce_pkts;
+ u64 tx_roce_bytes;
+ u64 rx_atomic_req;
+ u64 rx_read_req;
+ u64 rx_read_res;
+ u64 rx_write_req;
+ u64 rx_send_req;
+ u64 rx_roce_pkts;
+ u64 rx_roce_bytes;
+ u64 rx_roce_good_pkts;
+ u64 rx_roce_good_bytes;
+ u64 rx_out_of_buffer;
+ u64 rx_out_of_sequence;
+ u64 tx_cnp;
+ u64 rx_cnp;
+ u64 rx_ecn_marked;
+ u64 seq_err_naks_rcvd;
+ u64 rnr_naks_rcvd;
+ u64 missing_resp;
+ u64 to_retransmits;
+ u64 dup_req;
+ u64 rx_dcn_payload_cut;
+ u64 te_bypassed;
+};
+
+#define BNXT_QPLIB_ACCESS_LOCAL_WRITE (1 << 0)
+#define BNXT_QPLIB_ACCESS_REMOTE_READ (1 << 1)
+#define BNXT_QPLIB_ACCESS_REMOTE_WRITE (1 << 2)
+#define BNXT_QPLIB_ACCESS_REMOTE_ATOMIC (1 << 3)
+#define BNXT_QPLIB_ACCESS_MW_BIND (1 << 4)
+#define BNXT_QPLIB_ACCESS_ZERO_BASED (1 << 5)
+#define BNXT_QPLIB_ACCESS_ON_DEMAND (1 << 6)
+
+int bnxt_qplib_get_sgid(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_sgid_tbl *sgid_tbl, int index,
+ struct bnxt_qplib_gid *gid);
+int bnxt_qplib_del_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,
+ struct bnxt_qplib_gid *gid, u16 vlan_id, bool update);
+int bnxt_qplib_add_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,
+ const union ib_gid *gid, const u8 *mac, u16 vlan_id,
+ bool update, u32 *index);
+int bnxt_qplib_update_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,
+ struct bnxt_qplib_gid *gid, u16 gid_idx, const u8 *smac);
+int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw);
+int bnxt_qplib_set_func_resources(struct bnxt_qplib_res *res);
+int bnxt_qplib_create_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah,
+ bool block);
+int bnxt_qplib_destroy_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah,
+ bool block);
+int bnxt_qplib_alloc_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw);
+int bnxt_qplib_dereg_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw,
+ bool block);
+int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_mrinfo *mrinfo, bool block);
+int bnxt_qplib_free_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr);
+int bnxt_qplib_alloc_fast_reg_mr(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_mrw *mr, int max);
+int bnxt_qplib_alloc_fast_reg_page_list(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_frpl *frpl, int max);
+void bnxt_qplib_free_fast_reg_page_list(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_frpl *frpl);
+int bnxt_qplib_map_tc2cos(struct bnxt_qplib_res *res, u16 *cids);
+int bnxt_qplib_modify_cc(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_cc_param *cc_param);
+int bnxt_qplib_query_cc_param(struct bnxt_qplib_res *res,
+ struct bnxt_qplib_cc_param *cc_param);
+int bnxt_qplib_set_link_aggr_mode(struct bnxt_qplib_res *res,
+ u8 aggr_mode, u8 member_port_map,
+ u8 active_port_map, bool aggr_en,
+ u32 stats_fw_id);
+int bnxt_qplib_get_roce_error_stats(struct bnxt_qplib_rcfw *rcfw,
+ struct bnxt_qplib_roce_stats *stats,
+ struct bnxt_qplib_query_stats_info *sinfo);
+int bnxt_qplib_qext_stat(struct bnxt_qplib_rcfw *rcfw, u32 fid,
+ struct bnxt_qplib_ext_stat *estat,
+ struct bnxt_qplib_query_stats_info *sinfo);
+static inline void bnxt_re_set_max_gid(u16 *max_sgid);
+bool ib_modify_qp_is_ok_compat(enum ib_qp_state cur_state, enum ib_qp_state next_state,
+ enum ib_qp_type type, enum ib_qp_attr_mask mask);
+
+#define BNXT_MAX_SQ_SIZE 0xFFFF
+#define BNXT_MAX_VAR_WQE_SIZE 512
+#define BNXT_SGE_SIZE 16
+
+/* PF defines */
+#define BNXT_RE_MAX_QP_SUPPORTED(chip_gen) \
+ chip_gen == BNXT_RE_DEFAULT ? (64 * 1024) : 0
+
+#define BNXT_RE_MAX_MRW_SUPPORTED(chip_gen) \
+ chip_gen == BNXT_RE_DEFAULT ? (256 * 1024) : 0
+
+#define BNXT_RE_MAX_CQ_SUPPORTED(chip_gen) \
+ chip_gen == BNXT_RE_DEFAULT ? (64 * 1024) : 0
+
+#define BNXT_RE_MAX_SRQ_SUPPORTED(chip_gen) \
+ chip_gen == BNXT_RE_DEFAULT ? (4 * 1024) : 0
+
+#define BNXT_RE_MAX_AH_SUPPORTED(chip_gen) \
+ chip_gen == BNXT_RE_DEFAULT ? (64 * 1024) : 0
+
+/* VF defines */
+#define BNXT_RE_VF_MAX_QP_SUPPORTED(chip_gen) \
+ chip_gen == BNXT_RE_DEFAULT ? (6 * 1024) : 0
+
+#define BNXT_RE_VF_MAX_MRW_SUPPORTED(chip_gen) \
+ chip_gen == BNXT_RE_DEFAULT ? (6 * 1024) : 0
+
+#define BNXT_RE_VF_MAX_CQ_SUPPORTED(chip_gen) \
+ chip_gen == BNXT_RE_DEFAULT ? (6 * 1024) : 0
+
+#define BNXT_RE_VF_MAX_SRQ_SUPPORTED(chip_gen) \
+ chip_gen == BNXT_RE_DEFAULT ? (4 * 1024) : 0
+
+static inline void bnxt_re_set_max_gid(u16 *max_sgid)
+{
+ *max_sgid = max_t(u32, 256, *max_sgid);
+ *max_sgid = min_t(u32, 256, *max_sgid);
+}
+
+#endif
diff --git a/sys/dev/bnxt/bnxt_re/qplib_tlv.h b/sys/dev/bnxt/bnxt_re/qplib_tlv.h
new file mode 100644
index 000000000000..eeaea41a37e6
--- /dev/null
+++ b/sys/dev/bnxt/bnxt_re/qplib_tlv.h
@@ -0,0 +1,187 @@
+/*
+ * Copyright (c) 2017 - 2024, Broadcom. All rights reserved. The term
+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef __QPLIB_TLV_H__
+#define __QPLIB_TLV_H__
+
+struct roce_tlv {
+ struct tlv tlv;
+ u8 total_size;
+ u8 unused[7];
+};
+
+#define CHUNK_SIZE 16
+#define CHUNKS(x) (((x) + CHUNK_SIZE - 1) / CHUNK_SIZE)
+
+#define ROCE_1ST_TLV_PREP(rtlv, tot_chunks, content_bytes, more) \
+ do { \
+ (rtlv)->tlv.cmd_discr = CMD_DISCR_TLV_ENCAP; \
+ (rtlv)->tlv.tlv_type = TLV_TYPE_ROCE_SP_COMMAND; \
+ (rtlv)->tlv.length = (content_bytes); \
+ (rtlv)->tlv.flags = TLV_FLAGS_REQUIRED; \
+ (rtlv)->tlv.flags |= (more) ? TLV_FLAGS_MORE : 0; \
+ (rtlv)->total_size = (tot_chunks); \
+ } while (0)
+
+#define ROCE_EXT_TLV_PREP(rtlv, ext_type, content_bytes, more, reqd) \
+ do { \
+ (rtlv)->tlv.cmd_discr = CMD_DISCR_TLV_ENCAP; \
+ (rtlv)->tlv.tlv_type = (ext_type); \
+ (rtlv)->tlv.length = (content_bytes); \
+ (rtlv)->tlv.flags |= (more) ? TLV_FLAGS_MORE : 0; \
+ (rtlv)->tlv.flags |= (reqd) ? TLV_FLAGS_REQUIRED : 0; \
+ } while (0)
+
+/*
+ * TLV size in units of 16 byte chunks
+ */
+#define TLV_SIZE ((sizeof(struct roce_tlv) + 15) / 16)
+/*
+ * TLV length in bytes
+ */
+#define TLV_BYTES (TLV_SIZE * 16)
+
+#define HAS_TLV_HEADER(msg) (((struct tlv *)(msg))->cmd_discr == CMD_DISCR_TLV_ENCAP)
+#define GET_TLV_DATA(tlv) ((void *)&((uint8_t *)(tlv))[TLV_BYTES])
+
+static inline u8 __get_cmdq_base_opcode(struct cmdq_base *req, u32 size)
+{
+ if (HAS_TLV_HEADER(req) && size > TLV_BYTES)
+ return ((struct cmdq_base *)GET_TLV_DATA(req))->opcode;
+ else
+ return req->opcode;
+}
+
+static inline void __set_cmdq_base_opcode(struct cmdq_base *req,
+ u32 size, u8 val)
+{
+ if (HAS_TLV_HEADER(req) && size > TLV_BYTES)
+ ((struct cmdq_base *)GET_TLV_DATA(req))->opcode = val;
+ else
+ req->opcode = val;
+}
+
+static inline __le16 __get_cmdq_base_cookie(struct cmdq_base *req, u32 size)
+{
+ if (HAS_TLV_HEADER(req) && size > TLV_BYTES)
+ return ((struct cmdq_base *)GET_TLV_DATA(req))->cookie;
+ else
+ return req->cookie;
+}
+
+static inline void __set_cmdq_base_cookie(struct cmdq_base *req,
+ u32 size, __le16 val)
+{
+ if (HAS_TLV_HEADER(req) && size > TLV_BYTES)
+ ((struct cmdq_base *)GET_TLV_DATA(req))->cookie = val;
+ else
+ req->cookie = val;
+}
+
+static inline __le64 __get_cmdq_base_resp_addr(struct cmdq_base *req, u32 size)
+{
+ if (HAS_TLV_HEADER(req) && size > TLV_BYTES)
+ return ((struct cmdq_base *)GET_TLV_DATA(req))->resp_addr;
+ else
+ return req->resp_addr;
+}
+
+static inline void __set_cmdq_base_resp_addr(struct cmdq_base *req,
+ u32 size, __le64 val)
+{
+ if (HAS_TLV_HEADER(req) && size > TLV_BYTES)
+ ((struct cmdq_base *)GET_TLV_DATA(req))->resp_addr = val;
+ else
+ req->resp_addr = val;
+}
+
+static inline u8 __get_cmdq_base_resp_size(struct cmdq_base *req, u32 size)
+{
+ if (HAS_TLV_HEADER(req) && size > TLV_BYTES)
+ return ((struct cmdq_base *)GET_TLV_DATA(req))->resp_size;
+ else
+ return req->resp_size;
+}
+
+static inline void __set_cmdq_base_resp_size(struct cmdq_base *req,
+ u32 size, u8 val)
+{
+ if (HAS_TLV_HEADER(req) && size > TLV_BYTES)
+ ((struct cmdq_base *)GET_TLV_DATA(req))->resp_size = val;
+ else
+ req->resp_size = val;
+}
+
+static inline u8 __get_cmdq_base_cmd_size(struct cmdq_base *req, u32 size)
+{
+ if (HAS_TLV_HEADER(req) && size > TLV_BYTES)
+ return ((struct roce_tlv *)(req))->total_size;
+ else
+ return req->cmd_size;
+}
+
+static inline void __set_cmdq_base_cmd_size(struct cmdq_base *req,
+ u32 size, u8 val)
+{
+ if (HAS_TLV_HEADER(req) && size > TLV_BYTES)
+ ((struct cmdq_base *)GET_TLV_DATA(req))->cmd_size = val;
+ else
+ req->cmd_size = val;
+}
+
+static inline __le16 __get_cmdq_base_flags(struct cmdq_base *req, u32 size)
+{
+ if (HAS_TLV_HEADER(req) && size > TLV_BYTES)
+ return ((struct cmdq_base *)GET_TLV_DATA(req))->flags;
+ else
+ return req->flags;
+}
+
+static inline void __set_cmdq_base_flags(struct cmdq_base *req,
+ u32 size, __le16 val)
+{
+ if (HAS_TLV_HEADER(req) && size > TLV_BYTES)
+ ((struct cmdq_base *)GET_TLV_DATA(req))->flags = val;
+ else
+ req->flags = val;
+}
+
+struct bnxt_qplib_tlv_modify_cc_req {
+ struct roce_tlv tlv_hdr;
+ struct cmdq_modify_roce_cc base_req;
+ __le64 tlvpad;
+ struct cmdq_modify_roce_cc_gen1_tlv ext_req;
+};
+
+struct bnxt_qplib_tlv_query_rcc_sb {
+ struct roce_tlv tlv_hdr;
+ struct creq_query_roce_cc_resp_sb base_sb;
+ struct creq_query_roce_cc_gen1_resp_sb_tlv gen1_sb;
+};
+#endif
diff --git a/sys/dev/bnxt/bnxt_re/stats.c b/sys/dev/bnxt/bnxt_re/stats.c
new file mode 100644
index 000000000000..7b0e6097aae6
--- /dev/null
+++ b/sys/dev/bnxt/bnxt_re/stats.c
@@ -0,0 +1,773 @@
+/*
+ * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term
+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Description: statistics related functions
+ */
+
+#include "bnxt_re.h"
+#include "bnxt.h"
+
+int bnxt_re_get_flow_stats_from_service_pf(struct bnxt_re_dev *rdev,
+ struct bnxt_re_flow_counters *stats,
+ struct bnxt_qplib_query_stats_info *sinfo)
+{
+ struct hwrm_cfa_flow_stats_output resp = {};
+ struct hwrm_cfa_flow_stats_input req = {};
+ struct bnxt_en_dev *en_dev = rdev->en_dev;
+ struct bnxt_fw_msg fw_msg = {};
+ u16 target_id;
+ int rc = 0;
+
+ if (sinfo->function_id == 0xFFFFFFFF)
+ target_id = -1;
+ else
+ target_id = sinfo->function_id + 1;
+
+ /* Issue HWRM cmd to read flow counters for CNP tx and rx */
+ bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_CFA_FLOW_STATS, -1, target_id);
+ req.num_flows = cpu_to_le16(6);
+ req.flow_handle_0 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_CNP_CNT);
+ req.flow_handle_1 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_CNP_CNT |
+ HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_DIR_RX);
+ req.flow_handle_2 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_ROCEV1_CNT);
+ req.flow_handle_3 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_ROCEV1_CNT |
+ HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_DIR_RX);
+ req.flow_handle_4 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_ROCEV2_CNT);
+ req.flow_handle_5 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_ROCEV2_CNT |
+ HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_DIR_RX);
+ bnxt_re_fill_fw_msg(&fw_msg, &req, sizeof(req), &resp,
+ sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev));
+ rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev),
+ "Failed to get CFA Flow stats : rc = 0x%x\n", rc);
+ return rc;
+ }
+
+ stats->cnp_stats.cnp_tx_pkts = le64_to_cpu(resp.packet_0);
+ stats->cnp_stats.cnp_tx_bytes = le64_to_cpu(resp.byte_0);
+ stats->cnp_stats.cnp_rx_pkts = le64_to_cpu(resp.packet_1);
+ stats->cnp_stats.cnp_rx_bytes = le64_to_cpu(resp.byte_1);
+
+ stats->ro_stats.tx_pkts = le64_to_cpu(resp.packet_2) +
+ le64_to_cpu(resp.packet_4);
+ stats->ro_stats.tx_bytes = le64_to_cpu(resp.byte_2) +
+ le64_to_cpu(resp.byte_4);
+ stats->ro_stats.rx_pkts = le64_to_cpu(resp.packet_3) +
+ le64_to_cpu(resp.packet_5);
+ stats->ro_stats.rx_bytes = le64_to_cpu(resp.byte_3) +
+ le64_to_cpu(resp.byte_5);
+
+ return 0;
+}
+
+int bnxt_re_get_qos_stats(struct bnxt_re_dev *rdev)
+{
+ struct bnxt_re_ro_counters roce_only_tmp[2] = {{}, {}};
+ struct bnxt_re_cnp_counters tmp_counters[2] = {{}, {}};
+ struct hwrm_cfa_flow_stats_output resp = {};
+ struct hwrm_cfa_flow_stats_input req = {};
+ struct bnxt_en_dev *en_dev = rdev->en_dev;
+ struct bnxt_fw_msg fw_msg = {};
+ struct bnxt_re_cc_stat *cnps;
+ struct bnxt_re_rstat *dstat;
+ int rc = 0;
+ u64 bytes;
+ u64 pkts;
+
+ /* Issue HWRM cmd to read flow counters for CNP tx and rx */
+ bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_CFA_FLOW_STATS, -1, -1);
+ req.num_flows = cpu_to_le16(6);
+ req.flow_handle_0 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_CNP_CNT);
+ req.flow_handle_1 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_CNP_CNT |
+ HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_DIR_RX);
+ req.flow_handle_2 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_ROCEV1_CNT);
+ req.flow_handle_3 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_ROCEV1_CNT |
+ HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_DIR_RX);
+ req.flow_handle_4 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_ROCEV2_CNT);
+ req.flow_handle_5 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_ROCEV2_CNT |
+ HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_DIR_RX);
+ bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
+ sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev));
+ rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
+ if (rc) {
+ dev_err(rdev_to_dev(rdev),
+ "Failed to get CFA Flow stats : rc = 0x%x\n", rc);
+ goto done;
+ }
+
+ tmp_counters[0].cnp_tx_pkts = le64_to_cpu(resp.packet_0);
+ tmp_counters[0].cnp_tx_bytes = le64_to_cpu(resp.byte_0);
+ tmp_counters[0].cnp_rx_pkts = le64_to_cpu(resp.packet_1);
+ tmp_counters[0].cnp_rx_bytes = le64_to_cpu(resp.byte_1);
+
+ roce_only_tmp[0].tx_pkts = le64_to_cpu(resp.packet_2) +
+ le64_to_cpu(resp.packet_4);
+ roce_only_tmp[0].tx_bytes = le64_to_cpu(resp.byte_2) +
+ le64_to_cpu(resp.byte_4);
+ roce_only_tmp[0].rx_pkts = le64_to_cpu(resp.packet_3) +
+ le64_to_cpu(resp.packet_5);
+ roce_only_tmp[0].rx_bytes = le64_to_cpu(resp.byte_3) +
+ le64_to_cpu(resp.byte_5);
+
+ cnps = &rdev->stats.cnps;
+ dstat = &rdev->stats.dstat;
+ if (!cnps->is_first) {
+ /* First query done.. */
+ cnps->is_first = true;
+ cnps->prev[0].cnp_tx_pkts = tmp_counters[0].cnp_tx_pkts;
+ cnps->prev[0].cnp_tx_bytes = tmp_counters[0].cnp_tx_bytes;
+ cnps->prev[0].cnp_rx_pkts = tmp_counters[0].cnp_rx_pkts;
+ cnps->prev[0].cnp_rx_bytes = tmp_counters[0].cnp_rx_bytes;
+
+ cnps->prev[1].cnp_tx_pkts = tmp_counters[1].cnp_tx_pkts;
+ cnps->prev[1].cnp_tx_bytes = tmp_counters[1].cnp_tx_bytes;
+ cnps->prev[1].cnp_rx_pkts = tmp_counters[1].cnp_rx_pkts;
+ cnps->prev[1].cnp_rx_bytes = tmp_counters[1].cnp_rx_bytes;
+
+ dstat->prev[0].tx_pkts = roce_only_tmp[0].tx_pkts;
+ dstat->prev[0].tx_bytes = roce_only_tmp[0].tx_bytes;
+ dstat->prev[0].rx_pkts = roce_only_tmp[0].rx_pkts;
+ dstat->prev[0].rx_bytes = roce_only_tmp[0].rx_bytes;
+
+ dstat->prev[1].tx_pkts = roce_only_tmp[1].tx_pkts;
+ dstat->prev[1].tx_bytes = roce_only_tmp[1].tx_bytes;
+ dstat->prev[1].rx_pkts = roce_only_tmp[1].rx_pkts;
+ dstat->prev[1].rx_bytes = roce_only_tmp[1].rx_bytes;
+ } else {
+ u64 byte_mask, pkts_mask;
+ u64 diff;
+
+ byte_mask = bnxt_re_get_cfa_stat_mask(rdev->chip_ctx,
+ BYTE_MASK);
+ pkts_mask = bnxt_re_get_cfa_stat_mask(rdev->chip_ctx,
+ PKTS_MASK);
+ /*
+ * Calculate the number of cnp packets and use
+ * the value to calculate the CRC bytes.
+ * Multply pkts with 4 and add it to total bytes
+ */
+ pkts = bnxt_re_stat_diff(tmp_counters[0].cnp_tx_pkts,
+ &cnps->prev[0].cnp_tx_pkts,
+ pkts_mask);
+ cnps->cur[0].cnp_tx_pkts += pkts;
+ diff = bnxt_re_stat_diff(tmp_counters[0].cnp_tx_bytes,
+ &cnps->prev[0].cnp_tx_bytes,
+ byte_mask);
+ bytes = diff + pkts * 4;
+ cnps->cur[0].cnp_tx_bytes += bytes;
+ pkts = bnxt_re_stat_diff(tmp_counters[0].cnp_rx_pkts,
+ &cnps->prev[0].cnp_rx_pkts,
+ pkts_mask);
+ cnps->cur[0].cnp_rx_pkts += pkts;
+ bytes = bnxt_re_stat_diff(tmp_counters[0].cnp_rx_bytes,
+ &cnps->prev[0].cnp_rx_bytes,
+ byte_mask);
+ cnps->cur[0].cnp_rx_bytes += bytes;
+
+ /*
+ * Calculate the number of cnp packets and use
+ * the value to calculate the CRC bytes.
+ * Multply pkts with 4 and add it to total bytes
+ */
+ pkts = bnxt_re_stat_diff(tmp_counters[1].cnp_tx_pkts,
+ &cnps->prev[1].cnp_tx_pkts,
+ pkts_mask);
+ cnps->cur[1].cnp_tx_pkts += pkts;
+ diff = bnxt_re_stat_diff(tmp_counters[1].cnp_tx_bytes,
+ &cnps->prev[1].cnp_tx_bytes,
+ byte_mask);
+ cnps->cur[1].cnp_tx_bytes += diff + pkts * 4;
+ pkts = bnxt_re_stat_diff(tmp_counters[1].cnp_rx_pkts,
+ &cnps->prev[1].cnp_rx_pkts,
+ pkts_mask);
+ cnps->cur[1].cnp_rx_pkts += pkts;
+ bytes = bnxt_re_stat_diff(tmp_counters[1].cnp_rx_bytes,
+ &cnps->prev[1].cnp_rx_bytes,
+ byte_mask);
+ cnps->cur[1].cnp_rx_bytes += bytes;
+
+ pkts = bnxt_re_stat_diff(roce_only_tmp[0].tx_pkts,
+ &dstat->prev[0].tx_pkts,
+ pkts_mask);
+ dstat->cur[0].tx_pkts += pkts;
+ diff = bnxt_re_stat_diff(roce_only_tmp[0].tx_bytes,
+ &dstat->prev[0].tx_bytes,
+ byte_mask);
+ dstat->cur[0].tx_bytes += diff + pkts * 4;
+ pkts = bnxt_re_stat_diff(roce_only_tmp[0].rx_pkts,
+ &dstat->prev[0].rx_pkts,
+ pkts_mask);
+ dstat->cur[0].rx_pkts += pkts;
+
+ bytes = bnxt_re_stat_diff(roce_only_tmp[0].rx_bytes,
+ &dstat->prev[0].rx_bytes,
+ byte_mask);
+ dstat->cur[0].rx_bytes += bytes;
+ pkts = bnxt_re_stat_diff(roce_only_tmp[1].tx_pkts,
+ &dstat->prev[1].tx_pkts,
+ pkts_mask);
+ dstat->cur[1].tx_pkts += pkts;
+ diff = bnxt_re_stat_diff(roce_only_tmp[1].tx_bytes,
+ &dstat->prev[1].tx_bytes,
+ byte_mask);
+ dstat->cur[1].tx_bytes += diff + pkts * 4;
+ pkts = bnxt_re_stat_diff(roce_only_tmp[1].rx_pkts,
+ &dstat->prev[1].rx_pkts,
+ pkts_mask);
+ dstat->cur[1].rx_pkts += pkts;
+ bytes = bnxt_re_stat_diff(roce_only_tmp[1].rx_bytes,
+ &dstat->prev[1].rx_bytes,
+ byte_mask);
+ dstat->cur[1].rx_bytes += bytes;
+ }
+done:
+ return rc;
+}
+
+static void bnxt_re_copy_ext_stats(struct bnxt_re_dev *rdev,
+ u8 indx, struct bnxt_qplib_ext_stat *s)
+{
+ struct bnxt_re_ext_roce_stats *e_errs;
+ struct bnxt_re_cnp_counters *cnp;
+ struct bnxt_re_ext_rstat *ext_d;
+ struct bnxt_re_ro_counters *ro;
+
+ cnp = &rdev->stats.cnps.cur[indx];
+ ro = &rdev->stats.dstat.cur[indx];
+ ext_d = &rdev->stats.dstat.ext_rstat[indx];
+ e_errs = &rdev->stats.dstat.e_errs;
+
+ cnp->cnp_tx_pkts = s->tx_cnp;
+ cnp->cnp_rx_pkts = s->rx_cnp;
+ /* In bonding mode do not duplicate other stats */
+ if (indx)
+ return;
+ cnp->ecn_marked = s->rx_ecn_marked;
+
+ ro->tx_pkts = s->tx_roce_pkts;
+ ro->tx_bytes = s->tx_roce_bytes;
+ ro->rx_pkts = s->rx_roce_pkts;
+ ro->rx_bytes = s->rx_roce_bytes;
+
+ ext_d->tx.atomic_req = s->tx_atomic_req;
+ ext_d->tx.read_req = s->tx_read_req;
+ ext_d->tx.read_resp = s->tx_read_res;
+ ext_d->tx.write_req = s->tx_write_req;
+ ext_d->tx.send_req = s->tx_send_req;
+ ext_d->rx.atomic_req = s->rx_atomic_req;
+ ext_d->rx.read_req = s->rx_read_req;
+ ext_d->rx.read_resp = s->rx_read_res;
+ ext_d->rx.write_req = s->rx_write_req;
+ ext_d->rx.send_req = s->rx_send_req;
+ ext_d->grx.rx_pkts = s->rx_roce_good_pkts;
+ ext_d->grx.rx_bytes = s->rx_roce_good_bytes;
+ ext_d->rx_dcn_payload_cut = s->rx_dcn_payload_cut;
+ ext_d->te_bypassed = s->te_bypassed;
+ e_errs->oob = s->rx_out_of_buffer;
+ e_errs->oos = s->rx_out_of_sequence;
+ e_errs->seq_err_naks_rcvd = s->seq_err_naks_rcvd;
+ e_errs->rnr_naks_rcvd = s->rnr_naks_rcvd;
+ e_errs->missing_resp = s->missing_resp;
+ e_errs->to_retransmits = s->to_retransmits;
+ e_errs->dup_req = s->dup_req;
+}
+
+static int bnxt_re_get_ext_stat(struct bnxt_re_dev *rdev)
+{
+ struct bnxt_qplib_ext_stat estat[2] = {{}, {}};
+ struct bnxt_qplib_query_stats_info sinfo;
+ u32 fid;
+ int rc;
+
+ fid = PCI_FUNC(rdev->en_dev->pdev->devfn);
+ /* Set default values for sinfo */
+ sinfo.function_id = 0xFFFFFFFF;
+ sinfo.collection_id = 0xFF;
+ sinfo.vf_valid = false;
+ rc = bnxt_qplib_qext_stat(&rdev->rcfw, fid, &estat[0], &sinfo);
+ if (rc)
+ goto done;
+ bnxt_re_copy_ext_stats(rdev, 0, &estat[0]);
+
+done:
+ return rc;
+}
+
+static void bnxt_re_copy_rstat(struct bnxt_re_rdata_counters *d,
+ struct ctx_hw_stats_ext *s,
+ bool is_thor)
+{
+ d->tx_ucast_pkts = le64_to_cpu(s->tx_ucast_pkts);
+ d->tx_mcast_pkts = le64_to_cpu(s->tx_mcast_pkts);
+ d->tx_bcast_pkts = le64_to_cpu(s->tx_bcast_pkts);
+ d->tx_discard_pkts = le64_to_cpu(s->tx_discard_pkts);
+ d->tx_error_pkts = le64_to_cpu(s->tx_error_pkts);
+ d->tx_ucast_bytes = le64_to_cpu(s->tx_ucast_bytes);
+ /* Add four bytes of CRC bytes per packet */
+ d->tx_ucast_bytes += d->tx_ucast_pkts * 4;
+ d->tx_mcast_bytes = le64_to_cpu(s->tx_mcast_bytes);
+ d->tx_bcast_bytes = le64_to_cpu(s->tx_bcast_bytes);
+ d->rx_ucast_pkts = le64_to_cpu(s->rx_ucast_pkts);
+ d->rx_mcast_pkts = le64_to_cpu(s->rx_mcast_pkts);
+ d->rx_bcast_pkts = le64_to_cpu(s->rx_bcast_pkts);
+ d->rx_discard_pkts = le64_to_cpu(s->rx_discard_pkts);
+ d->rx_error_pkts = le64_to_cpu(s->rx_error_pkts);
+ d->rx_ucast_bytes = le64_to_cpu(s->rx_ucast_bytes);
+ d->rx_mcast_bytes = le64_to_cpu(s->rx_mcast_bytes);
+ d->rx_bcast_bytes = le64_to_cpu(s->rx_bcast_bytes);
+ if (is_thor) {
+ d->rx_agg_pkts = le64_to_cpu(s->rx_tpa_pkt);
+ d->rx_agg_bytes = le64_to_cpu(s->rx_tpa_bytes);
+ d->rx_agg_events = le64_to_cpu(s->rx_tpa_events);
+ d->rx_agg_aborts = le64_to_cpu(s->rx_tpa_errors);
+ }
+}
+
+static void bnxt_re_get_roce_data_stats(struct bnxt_re_dev *rdev)
+{
+ bool is_thor = _is_chip_gen_p5_p7(rdev->chip_ctx);
+ struct bnxt_re_rdata_counters *rstat;
+
+ rstat = &rdev->stats.dstat.rstat[0];
+ bnxt_re_copy_rstat(rstat, rdev->qplib_res.hctx->stats.dma, is_thor);
+
+}
+
+int bnxt_re_get_device_stats(struct bnxt_re_dev *rdev)
+{
+ struct bnxt_qplib_query_stats_info sinfo;
+ int rc = 0;
+
+ /* Stats are in 1s cadence */
+ if (test_bit(BNXT_RE_FLAG_ISSUE_CFA_FLOW_STATS, &rdev->flags)) {
+ if (bnxt_ext_stats_supported(rdev->chip_ctx, rdev->dev_attr->dev_cap_flags,
+ rdev->is_virtfn))
+ rc = bnxt_re_get_ext_stat(rdev);
+ else
+ rc = bnxt_re_get_qos_stats(rdev);
+
+ if (rc && rc != -ENOMEM)
+ clear_bit(BNXT_RE_FLAG_ISSUE_CFA_FLOW_STATS,
+ &rdev->flags);
+ }
+
+ if (test_bit(BNXT_RE_FLAG_ISSUE_ROCE_STATS, &rdev->flags)) {
+ bnxt_re_get_roce_data_stats(rdev);
+
+ /* Set default values for sinfo */
+ sinfo.function_id = 0xFFFFFFFF;
+ sinfo.collection_id = 0xFF;
+ sinfo.vf_valid = false;
+ rc = bnxt_qplib_get_roce_error_stats(&rdev->rcfw,
+ &rdev->stats.dstat.errs,
+ &sinfo);
+ if (rc && rc != -ENOMEM)
+ clear_bit(BNXT_RE_FLAG_ISSUE_ROCE_STATS,
+ &rdev->flags);
+ }
+
+ return rc;
+}
+
+static const char * const bnxt_re_stat_descs[] = {
+ "link_state",
+ "max_qp",
+ "max_srq",
+ "max_cq",
+ "max_mr",
+ "max_mw",
+ "max_ah",
+ "max_pd",
+ "active_qp",
+ "active_rc_qp",
+ "active_ud_qp",
+ "active_srq",
+ "active_cq",
+ "active_mr",
+ "active_mw",
+ "active_ah",
+ "active_pd",
+ "qp_watermark",
+ "rc_qp_watermark",
+ "ud_qp_watermark",
+ "srq_watermark",
+ "cq_watermark",
+ "mr_watermark",
+ "mw_watermark",
+ "ah_watermark",
+ "pd_watermark",
+ "resize_cq_count",
+ "hw_retransmission",
+ "recoverable_errors",
+ "rx_pkts",
+ "rx_bytes",
+ "tx_pkts",
+ "tx_bytes",
+ "cnp_tx_pkts",
+ "cnp_tx_bytes",
+ "cnp_rx_pkts",
+ "cnp_rx_bytes",
+ "roce_only_rx_pkts",
+ "roce_only_rx_bytes",
+ "roce_only_tx_pkts",
+ "roce_only_tx_bytes",
+ "rx_roce_error_pkts",
+ "rx_roce_discard_pkts",
+ "tx_roce_error_pkts",
+ "tx_roce_discards_pkts",
+ "res_oob_drop_count",
+ "tx_atomic_req",
+ "rx_atomic_req",
+ "tx_read_req",
+ "tx_read_resp",
+ "rx_read_req",
+ "rx_read_resp",
+ "tx_write_req",
+ "rx_write_req",
+ "tx_send_req",
+ "rx_send_req",
+ "rx_good_pkts",
+ "rx_good_bytes",
+ "rx_dcn_payload_cut",
+ "te_bypassed",
+ "rx_ecn_marked_pkts",
+ "max_retry_exceeded",
+ "to_retransmits",
+ "seq_err_naks_rcvd",
+ "rnr_naks_rcvd",
+ "missing_resp",
+ "dup_reqs",
+ "unrecoverable_err",
+ "bad_resp_err",
+ "local_qp_op_err",
+ "local_protection_err",
+ "mem_mgmt_op_err",
+ "remote_invalid_req_err",
+ "remote_access_err",
+ "remote_op_err",
+ "res_exceed_max",
+ "res_length_mismatch",
+ "res_exceeds_wqe",
+ "res_opcode_err",
+ "res_rx_invalid_rkey",
+ "res_rx_domain_err",
+ "res_rx_no_perm",
+ "res_rx_range_err",
+ "res_tx_invalid_rkey",
+ "res_tx_domain_err",
+ "res_tx_no_perm",
+ "res_tx_range_err",
+ "res_irrq_oflow",
+ "res_unsup_opcode",
+ "res_unaligned_atomic",
+ "res_rem_inv_err",
+ "res_mem_error64",
+ "res_srq_err",
+ "res_cmp_err",
+ "res_invalid_dup_rkey",
+ "res_wqe_format_err",
+ "res_cq_load_err",
+ "res_srq_load_err",
+ "res_tx_pci_err",
+ "res_rx_pci_err",
+ "res_oos_drop_count",
+ "num_irq_started",
+ "num_irq_stopped",
+ "poll_in_intr_en",
+ "poll_in_intr_dis",
+ "cmdq_full_dbg_cnt",
+ "fw_service_prof_type_sup",
+ "dbq_int_recv",
+ "dbq_int_en",
+ "dbq_pacing_resched",
+ "dbq_pacing_complete",
+ "dbq_pacing_alerts",
+ "dbq_dbr_fifo_reg"
+
+};
+
+static void bnxt_re_print_ext_stat(struct bnxt_re_dev *rdev,
+ struct rdma_hw_stats *stats)
+{
+ struct bnxt_re_cnp_counters *cnp;
+ struct bnxt_re_ext_rstat *ext_s;
+
+ ext_s = &rdev->stats.dstat.ext_rstat[0];
+ cnp = &rdev->stats.cnps.cur[0];
+
+ stats->value[BNXT_RE_TX_ATOMIC_REQ] = ext_s->tx.atomic_req;
+ stats->value[BNXT_RE_RX_ATOMIC_REQ] = ext_s->rx.atomic_req;
+ stats->value[BNXT_RE_TX_READ_REQ] = ext_s->tx.read_req;
+ stats->value[BNXT_RE_TX_READ_RESP] = ext_s->tx.read_resp;
+ stats->value[BNXT_RE_RX_READ_REQ] = ext_s->rx.read_req;
+ stats->value[BNXT_RE_RX_READ_RESP] = ext_s->rx.read_resp;
+ stats->value[BNXT_RE_TX_WRITE_REQ] = ext_s->tx.write_req;
+ stats->value[BNXT_RE_RX_WRITE_REQ] = ext_s->rx.write_req;
+ stats->value[BNXT_RE_TX_SEND_REQ] = ext_s->tx.send_req;
+ stats->value[BNXT_RE_RX_SEND_REQ] = ext_s->rx.send_req;
+ stats->value[BNXT_RE_RX_GOOD_PKTS] = ext_s->grx.rx_pkts;
+ stats->value[BNXT_RE_RX_GOOD_BYTES] = ext_s->grx.rx_bytes;
+ if (_is_chip_p7(rdev->chip_ctx)) {
+ stats->value[BNXT_RE_RX_DCN_PAYLOAD_CUT] = ext_s->rx_dcn_payload_cut;
+ stats->value[BNXT_RE_TE_BYPASSED] = ext_s->te_bypassed;
+ }
+ stats->value[BNXT_RE_RX_ECN_MARKED_PKTS] = cnp->ecn_marked;
+}
+
+static void bnxt_re_print_roce_only_counters(struct bnxt_re_dev *rdev,
+ struct rdma_hw_stats *stats)
+{
+ struct bnxt_re_ro_counters *roce_only = &rdev->stats.dstat.cur[0];
+
+ stats->value[BNXT_RE_ROCE_ONLY_RX_PKTS] = roce_only->rx_pkts;
+ stats->value[BNXT_RE_ROCE_ONLY_RX_BYTES] = roce_only->rx_bytes;
+ stats->value[BNXT_RE_ROCE_ONLY_TX_PKTS] = roce_only->tx_pkts;
+ stats->value[BNXT_RE_ROCE_ONLY_TX_BYTES] = roce_only->tx_bytes;
+}
+
+static void bnxt_re_print_normal_total_counters(struct bnxt_re_dev *rdev,
+ struct rdma_hw_stats *stats)
+{
+ struct bnxt_re_ro_counters *roce_only;
+ struct bnxt_re_cc_stat *cnps;
+
+ cnps = &rdev->stats.cnps;
+ roce_only = &rdev->stats.dstat.cur[0];
+
+ stats->value[BNXT_RE_RX_PKTS] = cnps->cur[0].cnp_rx_pkts + roce_only->rx_pkts;
+ stats->value[BNXT_RE_RX_BYTES] = cnps->cur[0].cnp_rx_bytes + roce_only->rx_bytes;
+ stats->value[BNXT_RE_TX_PKTS] = cnps->cur[0].cnp_tx_pkts + roce_only->tx_pkts;
+ stats->value[BNXT_RE_TX_BYTES] = cnps->cur[0].cnp_tx_bytes + roce_only->tx_bytes;
+}
+
+static void bnxt_re_print_normal_counters(struct bnxt_re_dev *rdev,
+ struct rdma_hw_stats *rstats)
+{
+ struct bnxt_re_rdata_counters *stats;
+ struct bnxt_re_cc_stat *cnps;
+ bool en_disp;
+
+ stats = &rdev->stats.dstat.rstat[0];
+ cnps = &rdev->stats.cnps;
+ en_disp = !_is_chip_gen_p5_p7(rdev->chip_ctx);
+
+ bnxt_re_print_normal_total_counters(rdev, rstats);
+ if (!rdev->is_virtfn) {
+ rstats->value[BNXT_RE_CNP_TX_PKTS] = cnps->cur[0].cnp_tx_pkts;
+ if (en_disp)
+ rstats->value[BNXT_RE_CNP_TX_BYTES] = cnps->cur[0].cnp_tx_bytes;
+ rstats->value[BNXT_RE_CNP_RX_PKTS] = cnps->cur[0].cnp_rx_pkts;
+ if (en_disp)
+ rstats->value[BNXT_RE_CNP_RX_BYTES] = cnps->cur[0].cnp_rx_bytes;
+ }
+ /* Print RoCE only bytes.. CNP counters include RoCE packets also */
+ bnxt_re_print_roce_only_counters(rdev, rstats);
+
+ rstats->value[BNXT_RE_RX_ROCE_ERROR_PKTS] = stats ? stats->rx_error_pkts : 0;
+ rstats->value[BNXT_RE_RX_ROCE_DISCARD_PKTS] = stats ? stats->rx_discard_pkts : 0;
+ if (!en_disp) {
+ rstats->value[BNXT_RE_TX_ROCE_ERROR_PKTS] = stats ? stats->tx_error_pkts : 0;
+ rstats->value[BNXT_RE_TX_ROCE_DISCARDS_PKTS] = stats ? stats->tx_discard_pkts : 0;
+ }
+
+ if (bnxt_ext_stats_supported(rdev->chip_ctx, rdev->dev_attr->dev_cap_flags,
+ rdev->is_virtfn)) {
+ rstats->value[BNXT_RE_RES_OOB_DROP_COUNT] = rdev->stats.dstat.e_errs.oob;
+ bnxt_re_print_ext_stat(rdev, rstats);
+ }
+}
+
+static void bnxt_re_copy_db_pacing_stats(struct bnxt_re_dev *rdev,
+ struct rdma_hw_stats *stats)
+{
+ struct bnxt_re_dbr_sw_stats *dbr_sw_stats = rdev->dbr_sw_stats;
+
+ stats->value[BNXT_RE_DBQ_PACING_RESCHED] = dbr_sw_stats->dbq_pacing_resched;
+ stats->value[BNXT_RE_DBQ_PACING_CMPL] = dbr_sw_stats->dbq_pacing_complete;
+ stats->value[BNXT_RE_DBQ_PACING_ALERT] = dbr_sw_stats->dbq_pacing_alerts;
+ stats->value[BNXT_RE_DBQ_DBR_FIFO_REG] = readl_fbsd(rdev->en_dev->softc,
+ rdev->dbr_db_fifo_reg_off, 0);
+}
+
+int bnxt_re_get_hw_stats(struct ib_device *ibdev,
+ struct rdma_hw_stats *stats,
+ u8 port, int index)
+{
+ struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
+ struct bnxt_re_ext_roce_stats *e_errs;
+ struct bnxt_re_rdata_counters *rstat;
+ struct bnxt_qplib_roce_stats *errs;
+ unsigned long tstamp_diff;
+ struct pci_dev *pdev;
+ int sched_msec;
+ int rc = 0;
+
+ if (!port || !stats)
+ return -EINVAL;
+
+ if (!rdev)
+ return -ENODEV;
+
+ if (!__bnxt_re_is_rdev_valid(rdev)) {
+ return -ENODEV;
+ }
+
+ pdev = rdev->en_dev->pdev;
+ errs = &rdev->stats.dstat.errs;
+ rstat = &rdev->stats.dstat.rstat[0];
+ e_errs = &rdev->stats.dstat.e_errs;
+#define BNXT_RE_STATS_CTX_UPDATE_TIMER 250
+ sched_msec = BNXT_RE_STATS_CTX_UPDATE_TIMER;
+ tstamp_diff = jiffies - rdev->stats.read_tstamp;
+ if (test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) {
+ if (/* restrict_stats && */ tstamp_diff < msecs_to_jiffies(sched_msec))
+ goto skip_query;
+ rc = bnxt_re_get_device_stats(rdev);
+ if (rc)
+ dev_err(rdev_to_dev(rdev),
+ "Failed to query device stats\n");
+ rdev->stats.read_tstamp = jiffies;
+ }
+
+ if (rdev->dbr_pacing)
+ bnxt_re_copy_db_pacing_stats(rdev, stats);
+
+skip_query:
+
+ if (rdev->netdev)
+ stats->value[BNXT_RE_LINK_STATE] = bnxt_re_link_state(rdev);
+ stats->value[BNXT_RE_MAX_QP] = rdev->dev_attr->max_qp;
+ stats->value[BNXT_RE_MAX_SRQ] = rdev->dev_attr->max_srq;
+ stats->value[BNXT_RE_MAX_CQ] = rdev->dev_attr->max_cq;
+ stats->value[BNXT_RE_MAX_MR] = rdev->dev_attr->max_mr;
+ stats->value[BNXT_RE_MAX_MW] = rdev->dev_attr->max_mw;
+ stats->value[BNXT_RE_MAX_AH] = rdev->dev_attr->max_ah;
+ stats->value[BNXT_RE_MAX_PD] = rdev->dev_attr->max_pd;
+ stats->value[BNXT_RE_ACTIVE_QP] = atomic_read(&rdev->stats.rsors.qp_count);
+ stats->value[BNXT_RE_ACTIVE_RC_QP] = atomic_read(&rdev->stats.rsors.rc_qp_count);
+ stats->value[BNXT_RE_ACTIVE_UD_QP] = atomic_read(&rdev->stats.rsors.ud_qp_count);
+ stats->value[BNXT_RE_ACTIVE_SRQ] = atomic_read(&rdev->stats.rsors.srq_count);
+ stats->value[BNXT_RE_ACTIVE_CQ] = atomic_read(&rdev->stats.rsors.cq_count);
+ stats->value[BNXT_RE_ACTIVE_MR] = atomic_read(&rdev->stats.rsors.mr_count);
+ stats->value[BNXT_RE_ACTIVE_MW] = atomic_read(&rdev->stats.rsors.mw_count);
+ stats->value[BNXT_RE_ACTIVE_AH] = atomic_read(&rdev->stats.rsors.ah_count);
+ stats->value[BNXT_RE_ACTIVE_PD] = atomic_read(&rdev->stats.rsors.pd_count);
+ stats->value[BNXT_RE_QP_WATERMARK] = atomic_read(&rdev->stats.rsors.max_qp_count);
+ stats->value[BNXT_RE_RC_QP_WATERMARK] = atomic_read(&rdev->stats.rsors.max_rc_qp_count);
+ stats->value[BNXT_RE_UD_QP_WATERMARK] = atomic_read(&rdev->stats.rsors.max_ud_qp_count);
+ stats->value[BNXT_RE_SRQ_WATERMARK] = atomic_read(&rdev->stats.rsors.max_srq_count);
+ stats->value[BNXT_RE_CQ_WATERMARK] = atomic_read(&rdev->stats.rsors.max_cq_count);
+ stats->value[BNXT_RE_MR_WATERMARK] = atomic_read(&rdev->stats.rsors.max_mr_count);
+ stats->value[BNXT_RE_MW_WATERMARK] = atomic_read(&rdev->stats.rsors.max_mw_count);
+ stats->value[BNXT_RE_AH_WATERMARK] = atomic_read(&rdev->stats.rsors.max_ah_count);
+ stats->value[BNXT_RE_PD_WATERMARK] = atomic_read(&rdev->stats.rsors.max_pd_count);
+ stats->value[BNXT_RE_RESIZE_CQ_COUNT] = atomic_read(&rdev->stats.rsors.resize_count);
+ stats->value[BNXT_RE_HW_RETRANSMISSION] = BNXT_RE_HW_RETX(rdev->dev_attr->dev_cap_flags) ? 1 : 0;
+ stats->value[BNXT_RE_RECOVERABLE_ERRORS] = rstat ? rstat->tx_bcast_pkts : 0;
+
+ bnxt_re_print_normal_counters(rdev, stats);
+
+
+ stats->value[BNXT_RE_MAX_RETRY_EXCEEDED] = errs->max_retry_exceeded;
+ if (bnxt_ext_stats_supported(rdev->chip_ctx, rdev->dev_attr->dev_cap_flags,
+ rdev->is_virtfn) &&
+ _is_hw_retx_supported(rdev->dev_attr->dev_cap_flags)) {
+ stats->value[BNXT_RE_TO_RETRANSMITS] = e_errs->to_retransmits;
+ stats->value[BNXT_RE_SEQ_ERR_NAKS_RCVD] = e_errs->seq_err_naks_rcvd;
+ stats->value[BNXT_RE_RNR_NAKS_RCVD] = e_errs->rnr_naks_rcvd;
+ stats->value[BNXT_RE_MISSING_RESP] = e_errs->missing_resp;
+ stats->value[BNXT_RE_DUP_REQS] = e_errs->dup_req;
+ } else {
+ stats->value[BNXT_RE_TO_RETRANSMITS] = errs->to_retransmits;
+ stats->value[BNXT_RE_SEQ_ERR_NAKS_RCVD] = errs->seq_err_naks_rcvd;
+ stats->value[BNXT_RE_RNR_NAKS_RCVD] = errs->rnr_naks_rcvd;
+ stats->value[BNXT_RE_MISSING_RESP] = errs->missing_resp;
+ stats->value[BNXT_RE_DUP_REQS] = errs->dup_req;
+ }
+
+ stats->value[BNXT_RE_UNRECOVERABLE_ERR] = errs->unrecoverable_err;
+ stats->value[BNXT_RE_BAD_RESP_ERR] = errs->bad_resp_err;
+ stats->value[BNXT_RE_LOCAL_QP_OP_ERR] = errs->local_qp_op_err;
+ stats->value[BNXT_RE_LOCAL_PROTECTION_ERR] = errs->local_protection_err;
+ stats->value[BNXT_RE_MEM_MGMT_OP_ERR] = errs->mem_mgmt_op_err;
+ stats->value[BNXT_RE_REMOTE_INVALID_REQ_ERR] = errs->remote_invalid_req_err;
+ stats->value[BNXT_RE_REMOTE_ACCESS_ERR] = errs->remote_access_err;
+ stats->value[BNXT_RE_REMOTE_OP_ERR] = errs->remote_op_err;
+ stats->value[BNXT_RE_RES_EXCEED_MAX] = errs->res_exceed_max;
+ stats->value[BNXT_RE_RES_LENGTH_MISMATCH] = errs->res_length_mismatch;
+ stats->value[BNXT_RE_RES_EXCEEDS_WQE] = errs->res_exceeds_wqe;
+ stats->value[BNXT_RE_RES_OPCODE_ERR] = errs->res_opcode_err;
+ stats->value[BNXT_RE_RES_RX_INVALID_RKEY] = errs->res_rx_invalid_rkey;
+ stats->value[BNXT_RE_RES_RX_DOMAIN_ERR] = errs->res_rx_domain_err;
+ stats->value[BNXT_RE_RES_RX_NO_PERM] = errs->res_rx_no_perm;
+ stats->value[BNXT_RE_RES_RX_RANGE_ERR] = errs->res_rx_range_err;
+ stats->value[BNXT_RE_RES_TX_INVALID_RKEY] = errs->res_tx_invalid_rkey;
+ stats->value[BNXT_RE_RES_TX_DOMAIN_ERR] = errs->res_tx_domain_err;
+ stats->value[BNXT_RE_RES_TX_NO_PERM] = errs->res_tx_no_perm;
+ stats->value[BNXT_RE_RES_TX_RANGE_ERR] = errs->res_tx_range_err;
+ stats->value[BNXT_RE_RES_IRRQ_OFLOW] = errs->res_irrq_oflow;
+ stats->value[BNXT_RE_RES_UNSUP_OPCODE] = errs->res_unsup_opcode;
+ stats->value[BNXT_RE_RES_UNALIGNED_ATOMIC] = errs->res_unaligned_atomic;
+ stats->value[BNXT_RE_RES_REM_INV_ERR] = errs->res_rem_inv_err;
+ stats->value[BNXT_RE_RES_MEM_ERROR64] = errs->res_mem_error;
+ stats->value[BNXT_RE_RES_SRQ_ERR] = errs->res_srq_err;
+ stats->value[BNXT_RE_RES_CMP_ERR] = errs->res_cmp_err;
+ stats->value[BNXT_RE_RES_INVALID_DUP_RKEY] = errs->res_invalid_dup_rkey;
+ stats->value[BNXT_RE_RES_WQE_FORMAT_ERR] = errs->res_wqe_format_err;
+ stats->value[BNXT_RE_RES_CQ_LOAD_ERR] = errs->res_cq_load_err;
+ stats->value[BNXT_RE_RES_SRQ_LOAD_ERR] = errs->res_srq_load_err;
+ stats->value[BNXT_RE_RES_TX_PCI_ERR] = errs->res_tx_pci_err;
+ stats->value[BNXT_RE_RES_RX_PCI_ERR] = errs->res_rx_pci_err;
+
+
+ if (bnxt_ext_stats_supported(rdev->chip_ctx, rdev->dev_attr->dev_cap_flags,
+ rdev->is_virtfn)) {
+ stats->value[BNXT_RE_RES_OOS_DROP_COUNT] = e_errs->oos;
+ } else {
+ /* Display on function 0 as OOS counters are chip-wide */
+ if (PCI_FUNC(pdev->devfn) == 0)
+ stats->value[BNXT_RE_RES_OOS_DROP_COUNT] = errs->res_oos_drop_count;
+ }
+ stats->value[BNXT_RE_NUM_IRQ_STARTED] = rdev->rcfw.num_irq_started;
+ stats->value[BNXT_RE_NUM_IRQ_STOPPED] = rdev->rcfw.num_irq_stopped;
+ stats->value[BNXT_RE_POLL_IN_INTR_EN] = rdev->rcfw.poll_in_intr_en;
+ stats->value[BNXT_RE_POLL_IN_INTR_DIS] = rdev->rcfw.poll_in_intr_dis;
+ stats->value[BNXT_RE_CMDQ_FULL_DBG_CNT] = rdev->rcfw.cmdq_full_dbg;
+ if (!rdev->is_virtfn)
+ stats->value[BNXT_RE_FW_SERVICE_PROF_TYPE_SUP] = is_qport_service_type_supported(rdev);
+
+ return ARRAY_SIZE(bnxt_re_stat_descs);
+}
+
+struct rdma_hw_stats *bnxt_re_alloc_hw_port_stats(struct ib_device *ibdev,
+ u8 port_num)
+{
+ return rdma_alloc_hw_stats_struct(bnxt_re_stat_descs,
+ ARRAY_SIZE(bnxt_re_stat_descs),
+ RDMA_HW_STATS_DEFAULT_LIFESPAN);
+}
diff --git a/sys/dev/bnxt/bnxt_re/stats.h b/sys/dev/bnxt/bnxt_re/stats.h
new file mode 100644
index 000000000000..748d8165947b
--- /dev/null
+++ b/sys/dev/bnxt/bnxt_re/stats.h
@@ -0,0 +1,353 @@
+/*
+ * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term
+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Description: statistics related data structures
+ */
+
+#ifndef __STATS_H__
+#define __STATS_H__
+
+#define BNXT_RE_CFA_STAT_BYTES_MASK 0xFFFFFFFFF
+#define BNXT_RE_CFA_STAT_PKTS_MASK 0xFFFFFFF
+enum {
+ BYTE_MASK = 0,
+ PKTS_MASK = 1
+};
+
+struct bnxt_re_cnp_counters {
+ u64 cnp_tx_pkts;
+ u64 cnp_tx_bytes;
+ u64 cnp_rx_pkts;
+ u64 cnp_rx_bytes;
+ u64 ecn_marked;
+};
+
+struct bnxt_re_ro_counters {
+ u64 tx_pkts;
+ u64 tx_bytes;
+ u64 rx_pkts;
+ u64 rx_bytes;
+};
+
+struct bnxt_re_flow_counters {
+ struct bnxt_re_ro_counters ro_stats;
+ struct bnxt_re_cnp_counters cnp_stats;
+};
+
+struct bnxt_re_ext_cntr {
+ u64 atomic_req;
+ u64 read_req;
+ u64 read_resp;
+ u64 write_req;
+ u64 send_req;
+};
+
+struct bnxt_re_ext_good {
+ u64 rx_pkts;
+ u64 rx_bytes;
+};
+
+struct bnxt_re_ext_rstat {
+ struct bnxt_re_ext_cntr tx;
+ struct bnxt_re_ext_cntr rx;
+ struct bnxt_re_ext_good grx;
+ u64 rx_dcn_payload_cut;
+ u64 te_bypassed;
+};
+
+struct bnxt_re_rdata_counters {
+ u64 tx_ucast_pkts;
+ u64 tx_mcast_pkts;
+ u64 tx_bcast_pkts;
+ u64 tx_discard_pkts;
+ u64 tx_error_pkts;
+ u64 tx_ucast_bytes;
+ u64 tx_mcast_bytes;
+ u64 tx_bcast_bytes;
+ u64 rx_ucast_pkts;
+ u64 rx_mcast_pkts;
+ u64 rx_bcast_pkts;
+ u64 rx_discard_pkts;
+ u64 rx_error_pkts;
+ u64 rx_ucast_bytes;
+ u64 rx_mcast_bytes;
+ u64 rx_bcast_bytes;
+ u64 rx_agg_pkts;
+ u64 rx_agg_bytes;
+ u64 rx_agg_events;
+ u64 rx_agg_aborts;
+};
+
+struct bnxt_re_cc_stat {
+ struct bnxt_re_cnp_counters prev[2];
+ struct bnxt_re_cnp_counters cur[2];
+ bool is_first;
+};
+
+struct bnxt_re_ext_roce_stats {
+ u64 oob;
+ u64 oos;
+ u64 seq_err_naks_rcvd;
+ u64 rnr_naks_rcvd;
+ u64 missing_resp;
+ u64 to_retransmits;
+ u64 dup_req;
+};
+
+struct bnxt_re_rstat {
+ struct bnxt_re_ro_counters prev[2];
+ struct bnxt_re_ro_counters cur[2];
+ struct bnxt_re_rdata_counters rstat[2];
+ struct bnxt_re_ext_rstat ext_rstat[2];
+ struct bnxt_re_ext_roce_stats e_errs;
+ struct bnxt_qplib_roce_stats errs;
+ unsigned long long prev_oob;
+};
+
+struct bnxt_re_res_cntrs {
+ atomic_t qp_count;
+ atomic_t rc_qp_count;
+ atomic_t ud_qp_count;
+ atomic_t cq_count;
+ atomic_t srq_count;
+ atomic_t mr_count;
+ atomic_t mw_count;
+ atomic_t ah_count;
+ atomic_t pd_count;
+ atomic_t resize_count;
+ atomic_t max_qp_count;
+ atomic_t max_rc_qp_count;
+ atomic_t max_ud_qp_count;
+ atomic_t max_cq_count;
+ atomic_t max_srq_count;
+ atomic_t max_mr_count;
+ atomic_t max_mw_count;
+ atomic_t max_ah_count;
+ atomic_t max_pd_count;
+};
+
+struct bnxt_re_device_stats {
+ struct bnxt_re_rstat dstat;
+ struct bnxt_re_res_cntrs rsors;
+ struct bnxt_re_cc_stat cnps;
+ unsigned long read_tstamp;
+ /* To be used in case to disable stats query from worker or change
+ * query interval. 0 means stats_query disabled.
+ */
+ u32 stats_query_sec;
+ /* A free running counter to be used along with stats_query_sec to
+ * decide whether to issue the command to FW.
+ */
+ u32 stats_query_counter;
+};
+
+static inline u64 bnxt_re_get_cfa_stat_mask(struct bnxt_qplib_chip_ctx *cctx,
+ bool type)
+{
+ u64 mask;
+
+ if (type == BYTE_MASK) {
+ mask = BNXT_RE_CFA_STAT_BYTES_MASK; /* 36 bits */
+ if (_is_chip_gen_p5_p7(cctx))
+ mask >>= 0x01; /* 35 bits */
+ } else {
+ mask = BNXT_RE_CFA_STAT_PKTS_MASK; /* 28 bits */
+ if (_is_chip_gen_p5_p7(cctx))
+ mask |= (0x10000000ULL); /* 29 bits */
+ }
+
+ return mask;
+}
+
+static inline u64 bnxt_re_stat_diff(u64 cur, u64 *prev, u64 mask)
+{
+ u64 diff;
+
+ if (!cur)
+ return 0;
+ diff = (cur - *prev) & mask;
+ if (diff)
+ *prev = cur;
+ return diff;
+}
+
+static inline void bnxt_re_clear_rsors_stat(struct bnxt_re_res_cntrs *rsors)
+{
+ atomic_set(&rsors->qp_count, 0);
+ atomic_set(&rsors->cq_count, 0);
+ atomic_set(&rsors->srq_count, 0);
+ atomic_set(&rsors->mr_count, 0);
+ atomic_set(&rsors->mw_count, 0);
+ atomic_set(&rsors->ah_count, 0);
+ atomic_set(&rsors->pd_count, 0);
+ atomic_set(&rsors->resize_count, 0);
+ atomic_set(&rsors->max_qp_count, 0);
+ atomic_set(&rsors->max_cq_count, 0);
+ atomic_set(&rsors->max_srq_count, 0);
+ atomic_set(&rsors->max_mr_count, 0);
+ atomic_set(&rsors->max_mw_count, 0);
+ atomic_set(&rsors->max_ah_count, 0);
+ atomic_set(&rsors->max_pd_count, 0);
+ atomic_set(&rsors->max_rc_qp_count, 0);
+ atomic_set(&rsors->max_ud_qp_count, 0);
+}
+
+enum bnxt_re_hw_stats {
+ BNXT_RE_LINK_STATE,
+ BNXT_RE_MAX_QP,
+ BNXT_RE_MAX_SRQ,
+ BNXT_RE_MAX_CQ,
+ BNXT_RE_MAX_MR,
+ BNXT_RE_MAX_MW,
+ BNXT_RE_MAX_AH,
+ BNXT_RE_MAX_PD,
+ BNXT_RE_ACTIVE_QP,
+ BNXT_RE_ACTIVE_RC_QP,
+ BNXT_RE_ACTIVE_UD_QP,
+ BNXT_RE_ACTIVE_SRQ,
+ BNXT_RE_ACTIVE_CQ,
+ BNXT_RE_ACTIVE_MR,
+ BNXT_RE_ACTIVE_MW,
+ BNXT_RE_ACTIVE_AH,
+ BNXT_RE_ACTIVE_PD,
+ BNXT_RE_QP_WATERMARK,
+ BNXT_RE_RC_QP_WATERMARK,
+ BNXT_RE_UD_QP_WATERMARK,
+ BNXT_RE_SRQ_WATERMARK,
+ BNXT_RE_CQ_WATERMARK,
+ BNXT_RE_MR_WATERMARK,
+ BNXT_RE_MW_WATERMARK,
+ BNXT_RE_AH_WATERMARK,
+ BNXT_RE_PD_WATERMARK,
+ BNXT_RE_RESIZE_CQ_COUNT,
+ BNXT_RE_HW_RETRANSMISSION,
+ BNXT_RE_RECOVERABLE_ERRORS,
+ BNXT_RE_RX_PKTS,
+ BNXT_RE_RX_BYTES,
+ BNXT_RE_TX_PKTS,
+ BNXT_RE_TX_BYTES,
+ BNXT_RE_CNP_TX_PKTS,
+ BNXT_RE_CNP_TX_BYTES,
+ BNXT_RE_CNP_RX_PKTS,
+ BNXT_RE_CNP_RX_BYTES,
+ BNXT_RE_ROCE_ONLY_RX_PKTS,
+ BNXT_RE_ROCE_ONLY_RX_BYTES,
+ BNXT_RE_ROCE_ONLY_TX_PKTS,
+ BNXT_RE_ROCE_ONLY_TX_BYTES,
+ BNXT_RE_RX_ROCE_ERROR_PKTS,
+ BNXT_RE_RX_ROCE_DISCARD_PKTS,
+ BNXT_RE_TX_ROCE_ERROR_PKTS,
+ BNXT_RE_TX_ROCE_DISCARDS_PKTS,
+ BNXT_RE_RES_OOB_DROP_COUNT,
+ BNXT_RE_TX_ATOMIC_REQ,
+ BNXT_RE_RX_ATOMIC_REQ,
+ BNXT_RE_TX_READ_REQ,
+ BNXT_RE_TX_READ_RESP,
+ BNXT_RE_RX_READ_REQ,
+ BNXT_RE_RX_READ_RESP,
+ BNXT_RE_TX_WRITE_REQ,
+ BNXT_RE_RX_WRITE_REQ,
+ BNXT_RE_TX_SEND_REQ,
+ BNXT_RE_RX_SEND_REQ,
+ BNXT_RE_RX_GOOD_PKTS,
+ BNXT_RE_RX_GOOD_BYTES,
+ BNXT_RE_RX_DCN_PAYLOAD_CUT,
+ BNXT_RE_TE_BYPASSED,
+ BNXT_RE_RX_ECN_MARKED_PKTS,
+ BNXT_RE_MAX_RETRY_EXCEEDED,
+ BNXT_RE_TO_RETRANSMITS,
+ BNXT_RE_SEQ_ERR_NAKS_RCVD,
+ BNXT_RE_RNR_NAKS_RCVD,
+ BNXT_RE_MISSING_RESP,
+ BNXT_RE_DUP_REQS,
+ BNXT_RE_UNRECOVERABLE_ERR,
+ BNXT_RE_BAD_RESP_ERR,
+ BNXT_RE_LOCAL_QP_OP_ERR,
+ BNXT_RE_LOCAL_PROTECTION_ERR,
+ BNXT_RE_MEM_MGMT_OP_ERR,
+ BNXT_RE_REMOTE_INVALID_REQ_ERR,
+ BNXT_RE_REMOTE_ACCESS_ERR,
+ BNXT_RE_REMOTE_OP_ERR,
+ BNXT_RE_RES_EXCEED_MAX,
+ BNXT_RE_RES_LENGTH_MISMATCH,
+ BNXT_RE_RES_EXCEEDS_WQE,
+ BNXT_RE_RES_OPCODE_ERR,
+ BNXT_RE_RES_RX_INVALID_RKEY,
+ BNXT_RE_RES_RX_DOMAIN_ERR,
+ BNXT_RE_RES_RX_NO_PERM,
+ BNXT_RE_RES_RX_RANGE_ERR,
+ BNXT_RE_RES_TX_INVALID_RKEY,
+ BNXT_RE_RES_TX_DOMAIN_ERR,
+ BNXT_RE_RES_TX_NO_PERM,
+ BNXT_RE_RES_TX_RANGE_ERR,
+ BNXT_RE_RES_IRRQ_OFLOW,
+ BNXT_RE_RES_UNSUP_OPCODE,
+ BNXT_RE_RES_UNALIGNED_ATOMIC,
+ BNXT_RE_RES_REM_INV_ERR,
+ BNXT_RE_RES_MEM_ERROR64,
+ BNXT_RE_RES_SRQ_ERR,
+ BNXT_RE_RES_CMP_ERR,
+ BNXT_RE_RES_INVALID_DUP_RKEY,
+ BNXT_RE_RES_WQE_FORMAT_ERR,
+ BNXT_RE_RES_CQ_LOAD_ERR,
+ BNXT_RE_RES_SRQ_LOAD_ERR,
+ BNXT_RE_RES_TX_PCI_ERR,
+ BNXT_RE_RES_RX_PCI_ERR,
+ BNXT_RE_RES_OOS_DROP_COUNT,
+ BNXT_RE_NUM_IRQ_STARTED,
+ BNXT_RE_NUM_IRQ_STOPPED,
+ BNXT_RE_POLL_IN_INTR_EN,
+ BNXT_RE_POLL_IN_INTR_DIS,
+ BNXT_RE_CMDQ_FULL_DBG_CNT,
+ BNXT_RE_FW_SERVICE_PROF_TYPE_SUP,
+ BNXT_RE_DBQ_INT_RECV,
+ BNXT_RE_DBQ_INT_EN,
+ BNXT_RE_DBQ_PACING_RESCHED,
+ BNXT_RE_DBQ_PACING_CMPL,
+ BNXT_RE_DBQ_PACING_ALERT,
+ BNXT_RE_DBQ_DBR_FIFO_REG,
+ BNXT_RE_DBQ_NUM_EXT_COUNTERS
+};
+
+#define BNXT_RE_NUM_STD_COUNTERS (BNXT_RE_OUT_OF_SEQ_ERR + 1)
+
+struct bnxt_re_stats {
+ struct bnxt_qplib_roce_stats errs;
+ struct bnxt_qplib_ext_stat ext_stat;
+};
+
+struct rdma_hw_stats *bnxt_re_alloc_hw_port_stats(struct ib_device *ibdev,
+ u8 port_num);
+int bnxt_re_get_hw_stats(struct ib_device *ibdev,
+ struct rdma_hw_stats *stats,
+ u8 port, int index);
+int bnxt_re_get_device_stats(struct bnxt_re_dev *rdev);
+int bnxt_re_get_flow_stats_from_service_pf(struct bnxt_re_dev *rdev,
+ struct bnxt_re_flow_counters *stats,
+ struct bnxt_qplib_query_stats_info *sinfo);
+int bnxt_re_get_qos_stats(struct bnxt_re_dev *rdev);
+#endif /* __STATS_H__ */
diff --git a/sys/modules/bnxt/bnxt_re/Makefile b/sys/modules/bnxt/bnxt_re/Makefile
new file mode 100644
index 000000000000..d6aa701dbae6
--- /dev/null
+++ b/sys/modules/bnxt/bnxt_re/Makefile
@@ -0,0 +1,22 @@
+.PATH: ${SRCTOP}/sys/dev/bnxt/bnxt_re
+
+KMOD=bnxt_re
+SRCS += ib_verbs.c ib_verbs.h
+SRCS += qplib_fp.c qplib_fp.h
+SRCS += qplib_sp.c qplib_sp.h
+SRCS += qplib_res.c qplib_res.h
+SRCS += qplib_rcfw.c qplib_rcfw.h
+SRCS += stats.c stats.h
+SRCS += main.c bnxt_re.h
+SRCS += opt_inet.h opt_inet6.h opt_ratelimit.h
+SRCS += ${LINUXKPI_GENSRCS}
+
+CFLAGS+= -I${SRCTOP}/sys/dev/bnxt/bnxt_en
+CFLAGS+= -I${SRCTOP}/sys/ofed/include
+CFLAGS+= -I${SRCTOP}/sys/ofed/include/uapi
+CFLAGS+= ${LINUXKPI_INCLUDES}
+CFLAGS+= -DCONFIG_INFINIBAND_USER_MEM
+
+.include <bsd.kmod.mk>
+
+CFLAGS+= -Wno-cast-qual -Wno-pointer-arith ${GCC_MS_EXTENSIONS}